From 7b12021dc19e3939096a8ebe797d040bd3503432 Mon Sep 17 00:00:00 2001 From: torzdf <36920800+torzdf@users.noreply.github.com> Date: Tue, 14 May 2019 16:06:15 +0000 Subject: [PATCH] Fixups vgg_face: to own module vgg_face: correct output and normalization Extract: more aggressive queue sizes sys_info: Fix cudnn check error on Linux --- lib/face_filter.py | 62 +++-------------------------- lib/sysinfo.py | 2 + lib/vgg_face.py | 79 +++++++++++++++++++++++++++++++++++++ plugins/extract/pipeline.py | 6 +-- tools/sort.py | 62 +---------------------------- 5 files changed, 90 insertions(+), 121 deletions(-) create mode 100644 lib/vgg_face.py diff --git a/lib/face_filter.py b/lib/face_filter.py index 4dd92e3617..5c11e71f71 100644 --- a/lib/face_filter.py +++ b/lib/face_filter.py @@ -2,15 +2,12 @@ """ Face Filterer for extraction in faceswap.py """ import logging -import os -import sys import cv2 -import numpy as np from lib.faces_detect import DetectedFace from lib.logger import get_loglevel -from lib.utils import GetModel +from lib.vgg_face import VGGFace from plugins.extract.pipeline import Extractor logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -31,29 +28,14 @@ def __init__(self, reference_file_paths, nreference_file_paths, detector, aligne "detector: %s, aligner: %s. loglevel: %s, multiprocess: %s, threshold: %s)", self.__class__.__name__, reference_file_paths, nreference_file_paths, detector, aligner, loglevel, multiprocess, threshold) - git_model_id = 7 - model_filename = ["vgg_face_v1.caffemodel", "vgg_face_v1.prototxt"] - - self.input_size = 224 self.numeric_loglevel = get_loglevel(loglevel) - self.vgg_face = self.get_model(git_model_id, model_filename) + self.vgg_face = VGGFace() self.filters = self.load_images(reference_file_paths, nreference_file_paths) self.align_faces(detector, aligner, loglevel, multiprocess) self.get_filter_encodings() self.threshold = threshold logger.debug("Initialized %s", self.__class__.__name__) - # <<< GET MODEL >>> # - @staticmethod - def get_model(git_model_id, model_filename): - """ Check if model is available, if not, download and unzip it """ - root_path = os.path.abspath(os.path.dirname(sys.argv[0])) - cache_path = os.path.join(root_path, "plugins", "extract", ".cache") - model = GetModel(model_filename, cache_path, git_model_id).model_path - vgg_face = cv2.dnn.readNetFromCaffe(model[1], model[0]) # pylint: disable=no-member - vgg_face.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) # pylint: disable=no-member - return vgg_face - @staticmethod def load_images(reference_file_paths, nreference_file_paths): """ Load the images """ @@ -136,44 +118,18 @@ def get_filter_encodings(self): """ Return filter face encodings from Keras VGG Face """ for filename, face in self.filters.items(): logger.debug("Getting encodings for: '%s'", filename) - encodings = self.predict(face["face"]) + encodings = self.vgg_face.predict(face["face"]) logger.debug("Filter Filename: %s, encoding shape: %s", filename, encodings.shape) face["encoding"] = encodings del face["face"] - def predict(self, face): - """ Return encodings for given image from vgg_face """ - if face.shape[0] != self.input_size: - face = self.resize_face(face) - blob = cv2.dnn.blobFromImage(face, # pylint: disable=no-member - 1.0, - (self.input_size, self.input_size), - [104, 117, 123], - False, - False) - self.vgg_face.setInput(blob) - preds = self.vgg_face.forward()[0, :] - return preds - - def resize_face(self, face): - """ Resize incoming face to model_input_size """ - if face.shape[0] < self.input_size: - interpolation = cv2.INTER_CUBIC # pylint:disable=no-member - else: - interpolation = cv2.INTER_AREA # pylint:disable=no-member - - face = cv2.resize(face, # pylint:disable=no-member - dsize=(self.input_size, self.input_size), - interpolation=interpolation) - return face - def check(self, detected_face): """ Check the extracted Face """ logger.trace("Checking face with FaceFilter") distances = {"filter": list(), "nfilter": list()} - encodings = self.predict(detected_face.aligned_face) + encodings = self.vgg_face.predict(detected_face.aligned_face) for filt in self.filters.values(): - similarity = self.find_cosine_similiarity(filt["encoding"], encodings) + similarity = self.vgg_face.find_cosine_similiarity(filt["encoding"], encodings) distances[filt["type"]].append(similarity) avgs = {key: avg(val) if val else None for key, val in distances.items()} @@ -217,11 +173,3 @@ def check(self, detected_face): logger.trace("Accepted face: (similarity: %s, threshold: %s)", distances, self.threshold) return retval - - @staticmethod - def find_cosine_similiarity(source_face, test_face): - """ Find the cosine similarity between a source face and a test face """ - var_a = np.matmul(np.transpose(source_face), test_face) - var_b = np.sum(np.multiply(source_face, source_face)) - var_c = np.sum(np.multiply(test_face, test_face)) - return 1 - (var_a / (np.sqrt(var_b) * np.sqrt(var_c))) diff --git a/lib/sysinfo.py b/lib/sysinfo.py index d6d04b336f..e5809f51ff 100644 --- a/lib/sysinfo.py +++ b/lib/sysinfo.py @@ -217,6 +217,8 @@ def cudnn_version(self): def cudnn_checkfiles_linux(): """ Return the checkfile locations for linux """ chk = os.popen("ldconfig -p | grep -P \"libcudnn.so.\\d+\" | head -n 1").read() + if "libcudnn.so." not in chk: + return list() chk = chk.strip().replace("libcudnn.so.", "") cudnn_vers = chk[0] cudnn_path = chk[chk.find("=>") + 3:chk.find("libcudnn") - 1] diff --git a/lib/vgg_face.py b/lib/vgg_face.py new file mode 100644 index 0000000000..b30c39c171 --- /dev/null +++ b/lib/vgg_face.py @@ -0,0 +1,79 @@ +#!/usr/bin python3 +""" VGG_Face inference using OpenCV-DNN +Model from: https://www.robots.ox.ac.uk/~vgg/software/vgg_face/ + +Licensed under Creative Commons Attribution License. +https://creativecommons.org/licenses/by-nc/4.0/ +""" + +import logging +import sys +import os + +import cv2 +import numpy as np + +from lib.utils import GetModel + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + + +class VGGFace(): + """ VGG Face feature extraction. + Input images should be in BGR Order """ + + def __init__(self): + logger.debug("Initializing %s", self.__class__.__name__) + git_model_id = 7 + model_filename = ["vgg_face_v1.caffemodel", "vgg_face_v1.prototxt"] + self.input_size = 224 + # Average image provided in http://www.robots.ox.ac.uk/~vgg/software/vgg_face/ + self.average_img = [129.1863, 104.7624, 93.5940] + + self.model = self.get_model(git_model_id, model_filename) + logger.debug("Initialized %s", self.__class__.__name__) + + # <<< GET MODEL >>> # + @staticmethod + def get_model(git_model_id, model_filename): + """ Check if model is available, if not, download and unzip it """ + root_path = os.path.abspath(os.path.dirname(sys.argv[0])) + cache_path = os.path.join(root_path, "plugins", "extract", ".cache") + model = GetModel(model_filename, cache_path, git_model_id).model_path + model = cv2.dnn.readNetFromCaffe(model[1], model[0]) # pylint: disable=no-member + model.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) # pylint: disable=no-member + return model + + def predict(self, face): + """ Return encodings for given image from vgg_face """ + if face.shape[0] != self.input_size: + face = self.resize_face(face) + blob = cv2.dnn.blobFromImage(face, # pylint: disable=no-member + 1.0, + (self.input_size, self.input_size), + self.average_img, + False, + False) + self.model.setInput(blob) + preds = self.model.forward("fc7")[0, :] + return preds + + def resize_face(self, face): + """ Resize incoming face to model_input_size """ + if face.shape[0] < self.input_size: + interpolation = cv2.INTER_CUBIC # pylint:disable=no-member + else: + interpolation = cv2.INTER_AREA # pylint:disable=no-member + + face = cv2.resize(face, # pylint:disable=no-member + dsize=(self.input_size, self.input_size), + interpolation=interpolation) + return face + + @staticmethod + def find_cosine_similiarity(source_face, test_face): + """ Find the cosine similarity between a source face and a test face """ + var_a = np.matmul(np.transpose(source_face), test_face) + var_b = np.sum(np.multiply(source_face, source_face)) + var_c = np.sum(np.multiply(test_face, test_face)) + return 1 - (var_a / (np.sqrt(var_b) * np.sqrt(var_c))) diff --git a/plugins/extract/pipeline.py b/plugins/extract/pipeline.py index ae764870b5..fac7357dfd 100644 --- a/plugins/extract/pipeline.py +++ b/plugins/extract/pipeline.py @@ -118,11 +118,11 @@ def add_queues(self): """ Add the required processing queues to Queue Manager """ queues = dict() for task in ("extract_detect_in", "extract_align_in", "extract_align_out"): - size = 0 + # Limit queue size to avoid stacking ram + size = 32 if task == "extract_detect_in" or (not self.is_parallel and task == "extract_align_in"): - # Limit queue size on input queues to avoid stacking ram - size = 100 + size = 64 queue_manager.add_queue(task, maxsize=size) queues[task] = queue_manager.get_queue(task) logger.debug("Queues: %s", queues) diff --git a/tools/sort.py b/tools/sort.py index 0b133b3a8c..8d3682982d 100644 --- a/tools/sort.py +++ b/tools/sort.py @@ -18,7 +18,7 @@ from lib.faces_detect import DetectedFace from lib.multithreading import SpawnProcess from lib.queue_manager import queue_manager, QueueEmpty -from lib.utils import GetModel +from lib.vgg_face import VGGFace from plugins.plugin_loader import PluginLoader from . import cli @@ -895,63 +895,3 @@ def bad_args(args): # pylint: disable=unused-argument PARSER.set_defaults(func=bad_args) ARGUMENTS = PARSER.parse_args() ARGUMENTS.func(ARGUMENTS) - - -class VGGFace(): - """ Temporary drop-in replacement for face_recognition.face_encodings """ - # TODO: Replace with GPU bound keras-vgg-face - def __init__(self): - git_model_id = 7 - model_filename = ["vgg_face_v1.caffemodel", "vgg_face_v1.prototxt"] - self.input_size = 224 - self.model = self.get_model(git_model_id, model_filename) - - @staticmethod - def get_model(git_model_id, model_filename): - """ Check if model is available, if not, download and unzip it """ - root_path = os.path.abspath(os.path.dirname(sys.argv[0])) - cache_path = os.path.join(root_path, "plugins", "extract", ".cache") - model = GetModel(model_filename, cache_path, git_model_id).model_path - vgg_face = cv2.dnn.readNetFromCaffe(model[1], model[0]) # pylint: disable=no-member - vgg_face.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) # pylint: disable=no-member - return vgg_face - - def predict(self, face): - """ Return face encodings from vgg_face """ - if face.shape[0] != self.input_size: - face = self.resize_face(face) - blob = cv2.dnn.blobFromImage(face, # pylint: disable=no-member - 1.0, - (self.input_size, self.input_size), - [104, 117, 123], - False, - False) - logger.trace("vgg_face input shape: %s", blob.shape) - self.model.setInput(blob) - preds = self.model.forward()[0, :] - logger.trace("vgg_face encoding shape: %s", preds.shape) - return preds - - def resize_face(self, face): - """ Resize incoming face to model_input_size """ - if face.shape[0] < self.input_size: - interpolation = cv2.INTER_CUBIC # pylint:disable=no-member - else: - interpolation = cv2.INTER_AREA # pylint:disable=no-member - - face = cv2.resize(face, # pylint:disable=no-member - dsize=(self.input_size, self.input_size), - interpolation=interpolation) - return face - - @staticmethod - def find_cosine_similiarity(source_face, test_face): - """ Find the cosine similarity between a source face and a test face """ - logger.trace("source_face shape: %s, test_face shape: %s", - source_face.shape, test_face.shape) - var_a = np.matmul(np.transpose(source_face), test_face) - var_b = np.sum(np.multiply(source_face, source_face)) - var_c = np.sum(np.multiply(test_face, test_face)) - retval = 1 - (var_a / (np.sqrt(var_b) * np.sqrt(var_c))) - logger.debug("Similarity: %s", retval) - return retval