Source code for pyVHR.extraction.sig_processing

import cv2
import mediapipe as mp
import numpy as np
from pyVHR.extraction.utils import *
from pyVHR.extraction.skin_extraction_methods import *
from pyVHR.extraction.sig_extraction_methods import *

"""
This module defines classes or methods used for Signal extraction and processing.
"""


[docs]class SignalProcessing(): """ This class performs offline signal extraction with different methods: - holistic. - squared / rectangular patches. """ def __init__(self): # Common parameters # self.tot_frames = None self.visualize_skin_collection = [] self.skin_extractor = SkinExtractionConvexHull() # Patches parameters # high_prio_ldmk_id, mid_prio_ldmk_id = get_magic_landmarks() self.ldmks = high_prio_ldmk_id + mid_prio_ldmk_id self.square = None self.rects = None self.visualize_skin = False self.visualize_landmarks = False self.visualize_landmarks_number = False self.visualize_patch = False self.font_size = 0.3 self.font_color = (255, 0, 0, 255) self.visualize_skin_collection = [] self.visualize_landmarks_collection = []
[docs] def set_total_frames(self, n): """ Set the total frames to be processed; if you want to process all the possible frames use n = 0. Args: n (int): number of frames to be processed. """ if n < 0: print("[ERROR] n must be a positive number!") self.tot_frames = int(n)
[docs] def set_skin_extractor(self, extractor): """ Set the skin extractor that will be used for skin extraction. Args: extractor: instance of a skin_extraction class (see :py:mod:`pyVHR.extraction.skin_extraction_methods`). """ self.skin_extractor = extractor
[docs] def set_visualize_skin_and_landmarks(self, visualize_skin=False, visualize_landmarks=False, visualize_landmarks_number=False, visualize_patch=False): """ Set visualization parameters. You can retrieve visualization output with the methods :py:meth:`pyVHR.extraction.sig_processing.SignalProcessing.get_visualize_skin` and :py:meth:`pyVHR.extraction.sig_processing.SignalProcessing.get_visualize_patches` Args: visualize_skin (bool): The skin and the patches will be visualized. visualize_landmarks (bool): The landmarks (centers of patches) will be visualized. visualize_landmarks_number (bool): The landmarks number will be visualized. visualize_patch (bool): The patches outline will be visualized. """ self.visualize_skin = visualize_skin self.visualize_landmarks = visualize_landmarks self.visualize_landmarks_number = visualize_landmarks_number self.visualize_patch = visualize_patch
[docs] def get_visualize_skin(self): """ Get the skin images produced by the last processing. Remember to set :py:meth:`pyVHR.extraction.sig_processing.SignalProcessing.set_visualize_skin_and_landmarks` correctly. Returns: list of ndarray: list of cv2 images; each image is a ndarray with shape [rows, columns, rgb_channels]. """ return self.visualize_skin_collection
[docs] def get_visualize_patches(self): """ Get the 'skin+patches' images produced by the last processing. Remember to set :py:meth:`pyVHR.extraction.sig_processing.SignalProcessing.set_visualize_skin_and_landmarks` correctly. Returns: list of ndarray: list of cv2 images; each image is a ndarray with shape [rows, columns, rgb_channels]. """ return self.visualize_landmarks_collection
### HOLISTIC METHODS ###
[docs] def extract_raw_holistic(self, videoFileName): """ Locates the skin pixels in each frame. This method is intended for rPPG methods that use raw video signal. Args: videoFileName (str): video file name or path. Returns: float32 ndarray: raw signal as float32 ndarray with shape [num_frames, rows, columns, rgb_channels]. """ skin_ex = self.skin_extractor mp_drawing = mp.solutions.drawing_utils mp_face_mesh = mp.solutions.face_mesh PRESENCE_THRESHOLD = 0.5 VISIBILITY_THRESHOLD = 0.5 sig = [] processed_frames_count = 0 with mp_face_mesh.FaceMesh( max_num_faces=1, min_detection_confidence=0.5, min_tracking_confidence=0.5) as face_mesh: for frame in extract_frames_yield(videoFileName): # convert the BGR image to RGB. image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) processed_frames_count += 1 width = image.shape[1] height = image.shape[0] # [landmarks, info], with info->x_center ,y_center, r, g, b ldmks = np.zeros((468, 5), dtype=np.float32) ldmks[:, 0] = -1.0 ldmks[:, 1] = -1.0 ### face landmarks ### results = face_mesh.process(image) if results.multi_face_landmarks: face_landmarks = results.multi_face_landmarks[0] landmarks = [l for l in face_landmarks.landmark] for idx in range(len(landmarks)): landmark = landmarks[idx] if not ((landmark.HasField('visibility') and landmark.visibility < VISIBILITY_THRESHOLD) or (landmark.HasField('presence') and landmark.presence < PRESENCE_THRESHOLD)): coords = mp_drawing._normalized_to_pixel_coordinates( landmark.x, landmark.y, width, height) if coords: ldmks[idx, 0] = coords[1] ldmks[idx, 1] = coords[0] ### skin extraction ### cropped_skin_im, full_skin_im = skin_ex.extract_skin( image, ldmks) else: cropped_skin_im = np.zeros_like(image) full_skin_im = np.zeros_like(image) if self.visualize_skin == True: self.visualize_skin_collection.append(full_skin_im) ### sig computing ### sig.append(full_skin_im) ### loop break ### if self.tot_frames is not None and self.tot_frames > 0 and processed_frames_count >= self.tot_frames: break sig = np.array(sig, dtype=np.float32) return sig
[docs] def extract_holistic(self, videoFileName): """ This method compute the RGB-mean signal using the whole skin (holistic); Args: videoFileName (str): video file name or path. Returns: float32 ndarray: RGB signal as ndarray with shape [num_frames, 1, rgb_channels]. The second dimension is 1 because the whole skin is considered as one estimators. """ self.visualize_skin_collection = [] skin_ex = self.skin_extractor mp_drawing = mp.solutions.drawing_utils mp_face_mesh = mp.solutions.face_mesh PRESENCE_THRESHOLD = 0.5 VISIBILITY_THRESHOLD = 0.5 sig = [] processed_frames_count = 0 with mp_face_mesh.FaceMesh( max_num_faces=1, min_detection_confidence=0.5, min_tracking_confidence=0.5) as face_mesh: for frame in extract_frames_yield(videoFileName): # convert the BGR image to RGB. image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) processed_frames_count += 1 width = image.shape[1] height = image.shape[0] # [landmarks, info], with info->x_center ,y_center, r, g, b ldmks = np.zeros((468, 5), dtype=np.float32) ldmks[:, 0] = -1.0 ldmks[:, 1] = -1.0 ### face landmarks ### results = face_mesh.process(image) if results.multi_face_landmarks: face_landmarks = results.multi_face_landmarks[0] landmarks = [l for l in face_landmarks.landmark] for idx in range(len(landmarks)): landmark = landmarks[idx] if not ((landmark.HasField('visibility') and landmark.visibility < VISIBILITY_THRESHOLD) or (landmark.HasField('presence') and landmark.presence < PRESENCE_THRESHOLD)): coords = mp_drawing._normalized_to_pixel_coordinates( landmark.x, landmark.y, width, height) if coords: ldmks[idx, 0] = coords[1] ldmks[idx, 1] = coords[0] ### skin extraction ### cropped_skin_im, full_skin_im = skin_ex.extract_skin( image, ldmks) else: cropped_skin_im = np.zeros_like(image) full_skin_im = np.zeros_like(image) if self.visualize_skin == True: self.visualize_skin_collection.append(full_skin_im) ### sig computing ### sig.append(holistic_mean( cropped_skin_im, np.int32(SignalProcessingParams.RGB_LOW_TH), np.int32(SignalProcessingParams.RGB_HIGH_TH))) ### loop break ### if self.tot_frames is not None and self.tot_frames > 0 and processed_frames_count >= self.tot_frames: break sig = np.array(sig, dtype=np.float32) return sig
### PATCHES METHODS ###
[docs] def set_landmarks(self, landmarks_list): """ Set the patches centers (landmarks) that will be used for signal processing. There are 468 facial points you can choose; for visualizing their identification number please use :py:meth:`pyVHR.plot.visualize.visualize_landmarks_list`. Args: landmarks_list (list): list of positive integers between 0 and 467 that identify patches centers (landmarks). """ if not isinstance(landmarks_list, list): print("[ERROR] landmarks_set must be a list!") return self.ldmks = landmarks_list
[docs] def set_square_patches_side(self, square_side): """ Set the dimension of the square patches that will be used for signal processing. There are 468 facial points you can choose; for visualizing their identification number please use :py:meth:`pyVHR.plot.visualize.visualize_landmarks_list`. Args: square_side (float): positive float that defines the length of the square patches. """ if not isinstance(square_side, float) or square_side <= 0.0: print("[ERROR] square_side must be a positive float!") return self.square = square_side
[docs] def set_rect_patches_sides(self, rects_dim): """ Set the dimension of each rectangular patch. There are 468 facial points you can choose; for visualizing their identification number please use :py:meth:`pyVHR.plot.visualize.visualize_landmarks_list`. Args: rects_dim (float32 ndarray): positive float32 np.ndarray of shape [num_landmarks, 2]. If the list of used landmarks is [1,2,3] and rects_dim is [[10,20],[12,13],[40,40]] then the landmark number 2 will have a rectangular patch of xy-dimension 12x13. """ if type(rects_dim) != type(np.array([])): print("[ERROR] rects_dim must be an np.ndarray!") return if rects_dim.shape[0] != len(self.ldmks) and rects_dim.shape[1] != 2: print("[ERROR] incorrect rects_dim shape!") return self.rects = rects_dim
[docs] def extract_patches(self, videoFileName, region_type, sig_extraction_method): """ This method compute the RGB-mean signal using specific skin regions (patches). Args: videoFileName (str): video file name or path. region_type (str): patches types can be "squares" or "rects". sig_extraction_method (str): RGB signal can be computed with "mean" or "median". We recommend to use mean. Returns: float32 ndarray: RGB signal as ndarray with shape [num_frames, num_patches, rgb_channels]. """ if self.square is None and self.rects is None: print( "[ERROR] Use set_landmarks_squares or set_landmarkds_rects before calling this function!") return None if region_type != "squares" and region_type != "rects": print("[ERROR] Invalid landmarks region type!") return None if sig_extraction_method != "mean" and sig_extraction_method != "median": print("[ERROR] Invalid signal extraction method!") return None ldmks_regions = None if region_type == "squares": ldmks_regions = np.float32(self.square) elif region_type == "rects": ldmks_regions = np.float32(self.rects) sig_ext_met = None if sig_extraction_method == "mean": if region_type == "squares": sig_ext_met = landmarks_mean elif region_type == "rects": sig_ext_met = landmarks_mean_custom_rect elif sig_extraction_method == "median": if region_type == "squares": sig_ext_met = landmarks_median elif region_type == "rects": sig_ext_met = landmarks_median_custom_rect self.visualize_skin_collection = [] self.visualize_landmarks_collection = [] skin_ex = self.skin_extractor mp_drawing = mp.solutions.drawing_utils mp_face_mesh = mp.solutions.face_mesh PRESENCE_THRESHOLD = 0.5 VISIBILITY_THRESHOLD = 0.5 sig = [] processed_frames_count = 0 with mp_face_mesh.FaceMesh( max_num_faces=1, min_detection_confidence=0.5, min_tracking_confidence=0.5) as face_mesh: for frame in extract_frames_yield(videoFileName): # convert the BGR image to RGB. image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) processed_frames_count += 1 width = image.shape[1] height = image.shape[0] # [landmarks, info], with info->x_center ,y_center, r, g, b ldmks = np.zeros((468, 5), dtype=np.float32) ldmks[:, 0] = -1.0 ldmks[:, 1] = -1.0 magic_ldmks = [] ### face landmarks ### results = face_mesh.process(image) if results.multi_face_landmarks: face_landmarks = results.multi_face_landmarks[0] landmarks = [l for l in face_landmarks.landmark] for idx in range(len(landmarks)): landmark = landmarks[idx] if not ((landmark.HasField('visibility') and landmark.visibility < VISIBILITY_THRESHOLD) or (landmark.HasField('presence') and landmark.presence < PRESENCE_THRESHOLD)): coords = mp_drawing._normalized_to_pixel_coordinates( landmark.x, landmark.y, width, height) if coords: ldmks[idx, 0] = coords[1] ldmks[idx, 1] = coords[0] ### skin extraction ### cropped_skin_im, full_skin_im = skin_ex.extract_skin( image, ldmks) else: cropped_skin_im = np.zeros_like(image) full_skin_im = np.zeros_like(image) ### sig computing ### for idx in self.ldmks: magic_ldmks.append(ldmks[idx]) magic_ldmks = np.array(magic_ldmks, dtype=np.float32) temp = sig_ext_met(magic_ldmks, full_skin_im, ldmks_regions, np.int32(SignalProcessingParams.RGB_LOW_TH), np.int32(SignalProcessingParams.RGB_HIGH_TH)) sig.append(temp) # visualize patches and skin if self.visualize_skin == True: self.visualize_skin_collection.append(full_skin_im) if self.visualize_landmarks == True: annotated_image = full_skin_im.copy() color = np.array([self.font_color[0], self.font_color[1], self.font_color[2]], dtype=np.uint8) for idx in self.ldmks: cv2.circle( annotated_image, (int(ldmks[idx, 1]), int(ldmks[idx, 0])), radius=0, color=self.font_color, thickness=-1) if self.visualize_landmarks_number == True: cv2.putText(annotated_image, str(idx), (int(ldmks[idx, 1]), int(ldmks[idx, 0])), cv2.FONT_HERSHEY_SIMPLEX, self.font_size, self.font_color, 1) if self.visualize_patch == True: if region_type == "squares": sides = np.array([self.square] * len(magic_ldmks)) annotated_image = draw_rects( annotated_image, np.array(magic_ldmks[:, 1]), np.array(magic_ldmks[:, 0]), sides, sides, color) elif region_type == "rects": annotated_image = draw_rects( annotated_image, np.array(magic_ldmks[:, 1]), np.array(magic_ldmks[:, 0]), np.array(self.rects[:, 0]), np.array(self.rects[:, 1]), color) self.visualize_landmarks_collection.append( annotated_image) ### loop break ### if self.tot_frames is not None and self.tot_frames > 0 and processed_frames_count >= self.tot_frames: break sig = np.array(sig, dtype=np.float32) return np.copy(sig[:, :, 2:])