Landmarks stored and used as floating point numbers (deepfakes#928)

* remove and fix int adjustments
* masking rounding

lib/aligner.py

@@ -38,17 +38,15 @@ def transform(self, image, mat, size, padding=0):
         logger.trace("matrix: %s, size: %s. padding: %s", mat, size, padding)
         matrix = self.transform_matrix(mat, size, padding)
         interpolators = get_matrix_scaling(matrix)
-        retval = cv2.warpAffine(image,  # pylint: disable=no-member
-                                matrix, (size, size), flags=interpolators[0])
+        retval = cv2.warpAffine(image, matrix, (size, size), flags=interpolators[0])
         return retval
 
     def transform_points(self, points, mat, size, padding=0):
         """ Transform points along matrix """
         logger.trace("points: %s, matrix: %s, size: %s. padding: %s", points, mat, size, padding)
         matrix = self.transform_matrix(mat, size, padding)
         points = np.expand_dims(points, axis=1)
-        points = cv2.transform(points,  # pylint: disable=no-member
-                               matrix, points.shape)
+        points = cv2.transform(points, matrix, points.shape)
         retval = np.squeeze(points)
         logger.trace("Returning: %s", retval)
         return retval
@@ -59,9 +57,9 @@ def get_original_roi(self, mat, size, padding=0):
         matrix = self.transform_matrix(mat, size, padding)
         points = np.array([[0, 0], [0, size - 1], [size - 1, size - 1], [size - 1, 0]], np.int32)
         points = points.reshape((-1, 1, 2))
-        matrix = cv2.invertAffineTransform(matrix)  # pylint: disable=no-member
+        matrix = cv2.invertAffineTransform(matrix)
         logger.trace("Returning: (points: %s, matrix: %s", points, matrix)
-        return cv2.transform(points, matrix)  # pylint: disable=no-member
+        return cv2.transform(points, matrix)
 
     @staticmethod
     def get_feature_mask(aligned_landmarks_68, size, padding=0, dilation=30):
@@ -72,7 +70,7 @@ def get_feature_mask(aligned_landmarks_68, size, padding=0, dilation=30):
         translation = padding
         pad_mat = np.matrix([[scale, 0.0, translation], [0.0, scale, translation]])
         aligned_landmarks_68 = np.expand_dims(aligned_landmarks_68, axis=1)
-        aligned_landmarks_68 = cv2.transform(aligned_landmarks_68,  # pylint: disable=no-member
+        aligned_landmarks_68 = cv2.transform(aligned_landmarks_68,
                                              pad_mat,
                                              aligned_landmarks_68.shape)
         aligned_landmarks_68 = np.squeeze(aligned_landmarks_68)
@@ -85,26 +83,22 @@ def get_feature_mask(aligned_landmarks_68, size, padding=0, dilation=30):
         mouth_points = aligned_landmarks_68[48:68].tolist()
         # TODO remove excessive reshapes and flattens
 
-        l_eye = np.array(l_eye_points + l_brow_points).reshape((-1, 2)).astype(int).flatten()
-        r_eye = np.array(r_eye_points + r_brow_points).reshape((-1, 2)).astype(int).flatten()
+        l_eye = np.array(l_eye_points + l_brow_points).reshape((-1, 2)).astype('int32').flatten()
+        r_eye = np.array(r_eye_points + r_brow_points).reshape((-1, 2)).astype('int32').flatten()
         mouth = np.array(mouth_points + nose_points + chin_points)
-        mouth = mouth.reshape((-1, 2)).astype(int).flatten()
-        l_eye_hull = cv2.convexHull(l_eye.reshape((-1, 2)))  # pylint: disable=no-member
-        r_eye_hull = cv2.convexHull(r_eye.reshape((-1, 2)))  # pylint: disable=no-member
-        mouth_hull = cv2.convexHull(mouth.reshape((-1, 2)))  # pylint: disable=no-member
+        mouth = mouth.reshape((-1, 2)).astype('int32').flatten()
+        l_eye_hull = cv2.convexHull(l_eye.reshape((-1, 2)))
+        r_eye_hull = cv2.convexHull(r_eye.reshape((-1, 2)))
+        mouth_hull = cv2.convexHull(mouth.reshape((-1, 2)))
 
         mask = np.zeros((size, size, 3), dtype=float)
-        cv2.fillConvexPoly(mask,  # pylint: disable=no-member
-                           l_eye_hull, (1, 1, 1))
-        cv2.fillConvexPoly(mask,  # pylint: disable=no-member
-                           r_eye_hull, (1, 1, 1))
-        cv2.fillConvexPoly(mask,  # pylint: disable=no-member
-                           mouth_hull, (1, 1, 1))
+        cv2.fillConvexPoly(mask, l_eye_hull, (1, 1, 1))
+        cv2.fillConvexPoly(mask, r_eye_hull, (1, 1, 1))
+        cv2.fillConvexPoly(mask, mouth_hull, (1, 1, 1))
 
         if dilation > 0:
             kernel = np.ones((dilation, dilation), np.uint8)
-            mask = cv2.dilate(mask,  # pylint: disable=no-member
-                              kernel, iterations=1)
+            mask = cv2.dilate(mask, kernel, iterations=1)
 
         logger.trace("Returning: %s", mask)
         return mask
@@ -116,9 +110,9 @@ def get_matrix_scaling(mat):
     y_scale = (mat[0, 0] * mat[1, 1] - mat[0, 1] * mat[1, 0]) / x_scale
     avg_scale = (x_scale + y_scale) * 0.5
     if avg_scale >= 1.:
-        interpolators = cv2.INTER_CUBIC, cv2.INTER_AREA  # pylint: disable=no-member
+        interpolators = cv2.INTER_CUBIC, cv2.INTER_AREA
     else:
-        interpolators = cv2.INTER_AREA, cv2.INTER_CUBIC  # pylint: disable=no-member
+        interpolators = cv2.INTER_AREA, cv2.INTER_CUBIC
     logger.trace("interpolator: %s, inverse interpolator: %s", interpolators[0], interpolators[1])
     return interpolators
 

lib/faces_detect.py

@@ -193,7 +193,7 @@ def from_alignment(self, alignment, image=None):
         self.h = alignment["h"]
         landmarks = alignment["landmarks_xy"]
         if not isinstance(landmarks, np.ndarray):
-            landmarks = np.array(landmarks, dtype="int32")
+            landmarks = np.array(landmarks, dtype="float32")
         self.landmarks_xy = landmarks
         # Manual tool does not know the final hash so default to None
         self.hash = alignment.get("hash", None)

lib/model/masks.py

@@ -41,7 +41,7 @@ class Mask():
     def __init__(self, landmarks, face, channels=4):
         logger.trace("Initializing %s: (face_shape: %s, channels: %s, landmarks: %s)",
                      self.__class__.__name__, face.shape, channels, landmarks)
-        self.landmarks = landmarks
+        self.landmarks = np.rint(landmarks).astype("int32")
         self.face = face
         self.dtype = face.dtype
         self.threshold = 255 if self.dtype == "uint8" else 255.0

lib/training_data.py

@@ -689,10 +689,11 @@ def _random_warp_landmarks(self, batch, batch_src_points, batch_dst_points):
         slices = self._constants["tgt_slices"]
 
         batch_dst = (batch_dst_points + np.random.normal(size=batch_dst_points.shape,
-                                                         scale=2.0)).astype("int32")
+                                                         scale=2.0))
 
         face_cores = [cv2.convexHull(np.concatenate([src[17:], dst[17:]], axis=0))
-                      for src, dst in zip(batch_src_points, batch_dst)]
+                      for src, dst in zip(batch_src_points.astype("int32"),
+                                          batch_dst.astype("int32"))]
 
         batch_src = np.append(batch_src_points, edge_anchors, axis=1)
         batch_dst = np.append(batch_dst, edge_anchors, axis=1)

plugins/extract/align/_base.py

@@ -186,7 +186,7 @@ def finalize(self, batch):
         for face, landmarks in zip(batch["detected_faces"], batch["landmarks"]):
             if not isinstance(landmarks, np.ndarray):
                 landmarks = np.array(landmarks)
-            face.landmarks_xy = np.rint(landmarks).astype("int32")
+            face.landmarks_xy = landmarks
         self._remove_invalid_keys(batch, ("detected_faces", "filename", "image"))
         logger.trace("Item out: %s", {key: val
                                       for key, val in batch.items()

plugins/extract/mask/components.py

@@ -32,8 +32,8 @@ def predict(self, batch):
         for mask, face in zip(batch["feed"], batch["detected_faces"]):
             parts = self.parse_parts(np.array(face.feed_landmarks))
             for item in parts:
-                item = np.concatenate(item)
-                hull = cv2.convexHull(item).astype("int32")  # pylint: disable=no-member
+                item = np.rint(np.concatenate(item)).astype("int32")
+                hull = cv2.convexHull(item)
                 cv2.fillConvexPoly(mask, hull, 1.0, lineType=cv2.LINE_AA)
         batch["prediction"] = batch["feed"]
         return batch

plugins/extract/mask/extended.py

@@ -32,8 +32,8 @@ def predict(self, batch):
         for mask, face in zip(batch["feed"], batch["detected_faces"]):
             parts = self.parse_parts(np.array(face.feed_landmarks))
             for item in parts:
-                item = np.concatenate(item)
-                hull = cv2.convexHull(item).astype("int32")  # pylint: disable=no-member
+                item = np.rint(np.concatenate(item)).astype("int32")
+                hull = cv2.convexHull(item)
                 cv2.fillConvexPoly(mask, hull, 1.0, lineType=cv2.LINE_AA)
         batch["prediction"] = batch["feed"]
         return batch

plugins/extract/mask/unet_dfl.py

@@ -19,7 +19,7 @@
 
 
 class Mask(Masker):
-    """ Perform transformation to align and get landmarks """
+    """ Neural network to process face image into a segmentation mask of the face """
     def __init__(self, **kwargs):
         git_model_id = 6
         model_filename = "DFL_256_sigmoid_v1.h5"

plugins/extract/mask/vgg_clear.py

@@ -20,7 +20,7 @@
 
 
 class Mask(Masker):
-    """ Perform transformation to align and get landmarks """
+    """ Neural network to process face image into a segmentation mask of the face """
     def __init__(self, **kwargs):
         git_model_id = 8
         model_filename = "Nirkin_300_softmax_v1.h5"

plugins/extract/mask/vgg_obstructed.py

@@ -20,7 +20,7 @@
 
 
 class Mask(Masker):
-    """ Perform transformation to align and get landmarks """
+    """ Neural network to process face image into a segmentation mask of the face """
     def __init__(self, **kwargs):
         git_model_id = 5
         model_filename = "Nirkin_500_softmax_v1.h5"

scripts/fsmedia.py

@@ -236,8 +236,7 @@ def get_items(self):
         """ Set the post processing actions """
         postprocess_items = dict()
         # Debug Landmarks
-        if (hasattr(self.args, 'debug_landmarks')
-                and self.args.debug_landmarks):
+        if (hasattr(self.args, 'debug_landmarks') and self.args.debug_landmarks):
             postprocess_items["DebugLandmarks"] = None
 
         # Face Filter post processing

tools/lib_alignments/annotate.py

@@ -42,8 +42,7 @@ def draw_bounding_box(self, color_id=1, thickness=1):
             bottom_right = (alignment["x"] + alignment["w"], alignment["y"] + alignment["h"])
             logger.trace("Drawing bounding box: (top_left: %s, bottom_right: %s, color: %s, "
                          "thickness: %s)", top_left, bottom_right, color, thickness)
-            cv2.rectangle(self.image,  # pylint: disable=no-member
-                          top_left, bottom_right, color, thickness)
+            cv2.rectangle(self.image, top_left, bottom_right, color, thickness)
 
     def draw_extract_box(self, color_id=2, thickness=1):
         """ Draw the extracted face box """
@@ -54,25 +53,24 @@ def draw_extract_box(self, color_id=2, thickness=1):
             logger.trace("Drawing Extract Box: (idx: %s, roi: %s)", idx, roi)
             top_left = [point for point in roi.squeeze()[0]]
             top_left = (top_left[0], top_left[1] - 10)
-            cv2.putText(self.image,  # pylint: disable=no-member
+            cv2.putText(self.image,
                         str(idx),
                         top_left,
-                        cv2.FONT_HERSHEY_DUPLEX,  # pylint: disable=no-member
+                        cv2.FONT_HERSHEY_DUPLEX,
                         1.0,
                         color,
                         thickness)
-            cv2.polylines(self.image, [roi], True, color, thickness)  # pylint: disable=no-member
+            cv2.polylines(self.image, [roi], True, color, thickness)
 
     def draw_landmarks(self, color_id=3, radius=1):
         """ Draw the facial landmarks """
         color = self.colors[color_id]
         for alignment in self.alignments:
-            landmarks = alignment["landmarks_xy"]
+            landmarks = alignment["landmarks_xy"].astype("int32")
             logger.trace("Drawing Landmarks: (landmarks: %s, color: %s, radius: %s)",
                          landmarks, color, radius)
             for (pos_x, pos_y) in landmarks:
-                cv2.circle(self.image,  # pylint: disable=no-member
-                           (pos_x, pos_y), radius, color, -1)
+                cv2.circle(self.image, (pos_x, pos_y), radius, color, -1)
 
     def draw_landmarks_mesh(self, color_id=4, thickness=1):
         """ Draw the facial landmarks """
@@ -92,8 +90,7 @@ def draw_landmarks_mesh(self, color_id=4, thickness=1):
             for key, val in facial_landmarks_idxs.items():
                 points = np.array([landmarks[val[0]:val[1]]], np.int32)
                 fill_poly = bool(key in ("right_eye", "left_eye", "mouth"))
-                cv2.polylines(self.image,  # pylint: disable=no-member
-                              points, fill_poly, color, thickness)
+                cv2.polylines(self.image, points, fill_poly, color, thickness)
 
     def draw_grey_out_faces(self, live_face):
         """ Grey out all faces except target """
@@ -104,7 +101,6 @@ def draw_grey_out_faces(self, live_face):
         for idx, roi in enumerate(self.roi):
             if idx != int(live_face):
                 logger.trace("Greying out face: (idx: %s, roi: %s)", idx, roi)
-                cv2.fillPoly(overlay, roi, (0, 0, 0))  # pylint: disable=no-member
+                cv2.fillPoly(overlay, roi, (0, 0, 0))
 
-        cv2.addWeighted(overlay,  # pylint: disable=no-member
-                        alpha, self.image, 1. - alpha, 0., self.image)
+        cv2.addWeighted(overlay, alpha, self.image, 1. - alpha, 0., self.image)

tools/lib_alignments/jobs.py

@@ -337,7 +337,7 @@ def convert_dfl_alignment(dfl_alignments, f_hash, alignments):
                      "y": top,
                      "h": bottom - top,
                      "hash": f_hash,
-                     "landmarks_xy": np.array(dfl_alignments["source_landmarks"], dtype="uint8")}
+                     "landmarks_xy": np.array(dfl_alignments["source_landmarks"], dtype="float32")}
         logger.trace("Adding alignment: (frame: '%s', alignment: %s", sourcefile, alignment)
         alignments.setdefault(sourcefile, list()).append(alignment)
 
@@ -951,7 +951,7 @@ def update_alignments(self, landmarks):
         logger.debug("Update alignments")
         for idx, frame in tqdm(self.mappings.items(), desc="Updating"):
             logger.trace("Updating: (frame: %s)", frame)
-            landmarks_update = landmarks[:, :, idx].astype(int)
+            landmarks_update = landmarks[:, :, idx]
             landmarks_xy = landmarks_update.reshape(68, 2).tolist()
             self.alignments.data[frame][0]["landmarks_xy"] = landmarks_xy
             logger.trace("Updated: (frame: '%s', landmarks: %s)", frame, landmarks_xy)