Untitled
plain_text
a month ago
31 kB
6
Indexable
Never
import cv2 as cv import mediapipe as mp import scipy.io as sio import numpy as np import glob # import natsort import os from pathlib import Path import timeit import scipy import scipy.ndimage import PIL import PIL.Image from PIL import Image # import tensorflow as tf import torch import skimage.metrics import lpips import matplotlib.pyplot as plt import sys import shutil import json from sklearn.preprocessing import normalize sys.path.append("/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/eg3d/dataset_preprocessing/ffhq/Deep3DFaceRecon_pytorch/insightface/recognition/") from arcface_torch.backbones import get_model MN = { 'silhouette': [ 10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377, 152, 148, 176, 149, 150, 136, 172, 58, 132, 93, 234, 127, 162, 21, 54, 103, 67, 109 ], 'lipsUpperOuter': [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291], 'lipsLowerOuter': [146, 91, 181, 84, 17, 314, 405, 321, 375, 291], 'lipsUpperInner': [78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308], 'lipsLowerInner': [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308], 'rightEyeUpper0': [246, 161, 160, 159, 158, 157, 173], 'rightEyeLower0': [33, 7, 163, 144, 145, 153, 154, 155, 133], 'rightEyeUpper1': [247, 30, 29, 27, 28, 56, 190], 'rightEyeLower1': [130, 25, 110, 24, 23, 22, 26, 112, 243], 'rightEyeUpper2': [113, 225, 224, 223, 222, 221, 189], 'rightEyeLower2': [226, 31, 228, 229, 230, 231, 232, 233, 244], 'rightEyeLower3': [143, 111, 117, 118, 119, 120, 121, 128, 245], 'rightEyebrowUpper': [156, 70, 63, 105, 66, 107, 55, 193], 'rightEyebrowLower': [35, 124, 46, 53, 52, 65], 'rightEyeIris': [473, 474, 475, 476, 477], 'leftEyeUpper0': [466, 388, 387, 386, 385, 384, 398], 'leftEyeLower0': [263, 249, 390, 373, 374, 380, 381, 382, 362], 'leftEyeUpper1': [467, 260, 259, 257, 258, 286, 414], 'leftEyeLower1': [359, 255, 339, 254, 253, 252, 256, 341, 463], 'leftEyeUpper2': [342, 445, 444, 443, 442, 441, 413], 'leftEyeLower2': [446, 261, 448, 449, 450, 451, 452, 453, 464], 'leftEyeLower3': [372, 340, 346, 347, 348, 349, 350, 357, 465], 'leftEyebrowUpper': [383, 300, 293, 334, 296, 336, 285, 417], 'leftEyebrowLower': [265, 353, 276, 283, 282, 295], 'leftEyeIris': [468, 469, 470, 471, 472], 'midwayBetweenEyes': [168], 'noseTip': [1], 'noseBottom': [2], 'noseRightCorner': [98], 'noseLeftCorner': [327], 'rightCheek': [205], 'leftCheek': [425] }; def calc_alignment_coefficients(pa, pb): matrix = [] for p1, p2 in zip(pa, pb): matrix.append([p1[0], p1[1], 1, 0, 0, 0, -p2[0] * p1[0], -p2[0] * p1[1]]) matrix.append([0, 0, 0, p1[0], p1[1], 1, -p2[1] * p1[0], -p2[1] * p1[1]]) a = np.matrix(matrix, dtype=float) b = np.array(pb).reshape(8) res = np.dot(np.linalg.inv(a.T * a) * a.T, b) return np.array(res).reshape(8) def perspectiveTransform(perspectiveMatrix, sourceCorners, sourcePoints): ''' perspectiveMatrix as above sourcePoints has shape (n,2) ''' augment = np.ones((sourcePoints.shape[0], 1)) projective_corners = np.concatenate( (sourcePoints, augment), axis=1).T # projective_points has shape 3xn projective_points = perspectiveMatrix.dot(projective_corners) # obtain the target_points by dividing the projective_points # by its last row (where it is non-zero) # target_points has shape (3,n). target_points = np.true_divide(projective_points, projective_points[-1,:]) # so we want return points in row form return target_points[:2,:].T def align_face(lm, img, output_size, file_name=None): """ :param filepath: str :return: PIL Image """ # lm_chin = lm[0: 17] # left-right # lm_eyebrow_left = lm[17: 22] # left-right # lm_eyebrow_right = lm[22: 27] # left-right # lm_nose = lm[27: 31] # top-down # lm_nostrils = lm[31: 36] # top-down # left_eye, right_eye = face_mesh.calc_around_eye_bbox(lm) # print(face_mesh.calc_eye_dist(face_result)) # # 虹彩検出 # left_iris, right_iris = detect_iris(image, iris_detector, left_eye, # right_eye) # # 虹彩の外接円を計算 # left_center, left_radius = calc_min_enc_losingCircle(left_iris) # right_center, right_radius = calc_min_enc_losingCircle(right_iris) # # # lm_eye_right = left_eye.copy() # lm_eye_left = right_eye.copy() lm_eye_right = np.concatenate((lm[MN['leftEyeLower1'], :], lm[MN['leftEyeUpper1'], :]), axis=0) # left-clockwise lm_eye_left = np.concatenate((lm[MN['rightEyeLower1'], :], lm[MN['rightEyeUpper1'], :]), axis=0) # left-clockwise lm_mouth_outer = np.concatenate((lm[MN['lipsLowerOuter'], :], lm[MN['lipsUpperOuter'], :]), axis=0) # left-clockwise lm_mouth_inner = np.concatenate((lm[MN['lipsLowerInner'], :], lm[MN['lipsUpperInner'], :]), axis=0) # left-clockwise # Calculate auxiliary vectors. eye_left = np.mean(lm_eye_left, axis=0) # print(eye_left) eye_right = np.mean(lm_eye_right, axis=0) # print(eye_right) eye_avg = (eye_left + eye_right) * 0.5 eye_to_eye = eye_right - eye_left # print(eye_to_eye) mouth_left = lm_mouth_outer[10] mouth_right = lm_mouth_outer[-1] mouth_avg = (mouth_left + mouth_right) * 0.5 eye_to_mouth = mouth_avg - eye_avg # print(eye_to_mouth) # Choose oriented crop rectangle. x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1] x /= np.hypot(*x) x *= max(np.hypot(*eye_to_eye) * 2.0, np.hypot(*eye_to_mouth) * 1.8) y = np.flipud(x) * [-1, 1] c = eye_avg + eye_to_mouth * 0.1 quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y]) qsize = np.hypot(*x) * 2 # read image # img = PIL.Image.open(filepath) transform_size = output_size enable_padding = False # annotated_image = np.array(img.copy()) # for i in range(lm_eye_right.shape[0]): # annotated_image = cv.circle(annotated_image, (int(lm_eye_right[i,0]), int(lm_eye_right[i,1])), radius=0, color=(0, 0, 255), thickness=10) # for i in range(lm_eye_left.shape[0]): # annotated_image = cv.circle(annotated_image, (int(lm_eye_left[i,0]), int(lm_eye_left[i,1])), radius=0, color=(0, 0, 255), thickness=10) # for i in range(lm_mouth_outer.shape[0]): # annotated_image = cv.circle(annotated_image, (int(lm_mouth_outer[i,0]), int(lm_mouth_outer[i,1])), radius=0, color=(0, 0, 255), thickness=10) # # annotated_image = cv.circle(annotated_image, (int(lm_eye_left[i,0]), int(lm_eye_left[i,1])), radius=0, color=(0, 0, 255), thickness=10) # # # annotated_image = cv.circle(annotated_image, (int(eye_left[0]), int(eye_left[1])), radius=0, color=(0, 255, 0), thickness=10) # annotated_image = cv.circle(annotated_image, (int(eye_right[0]), int(eye_right[1])), radius=0, color=(0, 255, 0), thickness=10) # annotated_image = cv.circle(annotated_image, (int(mouth_left[0]), int(mouth_left[1])), radius=0, color=(0, 255, 0), thickness=10) # annotated_image = cv.circle(annotated_image, (int(mouth_right[0]), int(mouth_right[1])), radius=0, color=(0, 255, 0), thickness=10) # # # annotated_image = cv.circle(annotated_image, (int(lm_mouth_outer[10,0]), int(lm_mouth_outer[10,1])), radius=0, color=(0, 255, 0), thickness=10) # # annotated_image = cv.circle(annotated_image, (int(lm_mouth_outer[-1,0]), int(lm_mouth_outer[-1,1])), radius=0, color=(0, 255, 0), thickness=10) # # name = '/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output/landmark/' + file_name + '_debug1.jpg' # print(name) # cv.imwrite(name, annotated_image) # Shrink. shrink = int(np.floor(qsize / output_size * 0.5)) if shrink > 1: rsize = (int(np.rint(float(img.size[0]) / shrink)), int(np.rint(float(img.size[1]) / shrink))) img = img.resize(rsize, PIL.Image.ANTIALIAS) quad /= shrink qsize /= shrink lm /= shrink # Crop. border = max(int(np.rint(qsize * 0.1)), 3) crop = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))), int(np.ceil(max(quad[:, 1])))) crop = (max(crop[0] - border, 0), max(crop[1] - border, 0), min(crop[2] + border, img.size[0]), min(crop[3] + border, img.size[1])) if crop[2] - crop[0] < img.size[0] or crop[3] - crop[1] < img.size[1]: img = img.crop(crop) quad -= crop[0:2] # print(lm[1,:]) lm -= np.array(crop[0:2]) # print(lm[1,:]) # Pad. pad = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))), int(np.ceil(max(quad[:, 1])))) pad = (max(-pad[0] + border, 0), max(-pad[1] + border, 0), max(pad[2] - img.size[0] + border, 0), max(pad[3] - img.size[1] + border, 0)) if enable_padding and max(pad) > border - 4: pad = np.maximum(pad, int(np.rint(qsize * 0.3))) img = np.pad(np.float32(img), ((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)), 'reflect') h, w, _ = img.shape y, x, _ = np.ogrid[:h, :w, :1] mask = np.maximum(1.0 - np.minimum(np.float32(x) / pad[0], np.float32(w - 1 - x) / pad[2]), 1.0 - np.minimum(np.float32(y) / pad[1], np.float32(h - 1 - y) / pad[3])) blur = qsize * 0.02 img += (scipy.ndimage.gaussian_filter(img, [blur, blur, 0]) - img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0) img += (np.median(img, axis=(0, 1)) - img) * np.clip(mask, 0.0, 1.0) img = PIL.Image.fromarray(np.uint8(np.clip(np.rint(img), 0, 255)), 'RGB') quad += pad[:2] lm += np.array(pad[:2]) annotated_image = np.array(img.copy()) for i in range(lm.shape[0]): annotated_image = cv.circle(annotated_image, (int(lm[i,0]), int(lm[i,1])), radius=0, color=(255, 0, 0), thickness=10) # annotated_image = cv.circle(np.array(img.copy()), (int(face_landmarks.landmark[159].x*w), int(face_landmarks.landmark[159].y*h)), radius=0, color=(0, 255, 0), thickness=10) # cv.imwrite(os.path.splitext(file)[0] + '_debug.jpg', annotated_image) # print(quad + 0.5) # Transform. img = img.transform((transform_size, transform_size), PIL.Image.QUAD, (quad + 0.5).flatten(), PIL.Image.BILINEAR) forward_transforms = calc_alignment_coefficients(quad + 0.5, [[0, 0], [0, transform_size], [transform_size, transform_size], [transform_size, 0]]) forward_transforms = np.concatenate((forward_transforms, np.array((1,)))) lm = perspectiveTransform(np.array(forward_transforms).reshape((3,3)), np.array([[0, 0], [0, transform_size], [transform_size, transform_size], [transform_size, 0]]), lm) if 'ours' in file_name: our_512_box = perspectiveTransform(np.array(forward_transforms).reshape((3,3)), np.array([[0, 0], [0, transform_size], [transform_size, transform_size], [transform_size, 0]]), np.array([[0, 0], [0, 511], [511, 511], [511, 0]])) our_512_box = our_512_box.clip(0, 511) name = os.path.join(align_folder, file_name + '.png') cv.imwrite(name, np.array(img.copy())) # annotated_image = np.array(img.copy()) # for i in range(lm.shape[0]): # annotated_image = cv.circle(annotated_image, (int(lm[i,0]), int(lm[i,1])), radius=0, color=(255, 0, 0), thickness=10) # # annotated_image = cv.circle(np.array(img.copy()), (int(face_landmarks.landmark[159].x*w), int(face_landmarks.landmark[159].y*h)), radius=0, color=(0, 255, 0), thickness=10) # # print(os.path.splitext(os.path.basename(file_name))[0]) # name = '/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output/landmark/' + file_name + '_debug2.jpg' # print(name) # cv.imwrite(name, annotated_image) if output_size < transform_size: img = img.resize((output_size, output_size), PIL.Image.ANTIALIAS) mask = np.zeros((512,512)) import cv2 if 'ours' in file_name: return lm, cv2.fillConvexPoly(mask, (our_512_box.astype(np.int64)), 1) # import pdb; pdb.set_trace() # Return aligned image. else: return lm def extract_foreground(image, matte): # calculate display resolution w, h = image.width, image.height # rw, rh = 800, int(h * 800 / (3 * w)) # obtain predicted foreground image = np.asarray(image) if len(image.shape) == 2: image = image[:, :, None] if image.shape[2] == 1: image = np.repeat(image, 3, axis=2) elif image.shape[2] == 4: image = image[:, :, 0:3] matte = np.repeat(np.asarray(matte)[:, :, None], 3, axis=2) / 255 foreground = image * matte + np.full(image.shape, 255) * (1 - matte) # combine image, foreground, and alpha into one line # combined = np.concatenate((image, foreground, matte * 255), axis=1) # combined = Image.fromarray(np.uint8(combined)).resize((rw, rh)) return foreground def get_face_feat(img, net): img = cv.resize(img, (112, 112)) img = np.transpose(img, (2, 0, 1)) img = torch.from_numpy(img).unsqueeze(0).float() img.div_(255).sub_(0.5).div_(0.5) net.eval() feat = net(img).detach().numpy() feat = normalize(feat) return feat def get_faces_sim(img1, img2, net): feat1 = get_face_feat(img1, net) feat2 = get_face_feat(img2, net) sim = np.dot(feat1, feat2.T) return sim # specify filenames here file_extension = 'png' # results_folder = 'inversion/output/landmark/' # end net = get_model('r50', fp16=False) net.load_state_dict(torch.load('/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/eg3d/backbone.pth')) net.eval() mp_drawing = mp.solutions.drawing_utils mp_face_mesh = mp.solutions.face_mesh # Path(results_folder).mkdir(parents=True, exist_ok=True) max_numoffaces = 3 bool_notracking = True pts3D = np.zeros((468, 3)) # # # For static images: # filenames = glob.glob('test5.' + file_extension) # IMAGE_FILES = natsort.natsorted(filenames) loss_fn = lpips.LPIPS(net='alex') ref_path_list = [] # ref_list = ['2', '3', '4', '6', '8', '12', '16'] # ref_list = ['2', '3', '4', '6', '8', '12', '16'] # dist = np.array([0.6096, 0.9144, 1.2192, 1.8288, 2.4384, 3.6576, 4.8768]) ref_list = ['16'] dist = np.array([4.8768]) # %28 40 65 100 130 200 300 # %1000* 0.6096 0.9144 1.2192 1.8288 2.4384 3.6576 4.8768 # face_names = ['62RT-060329'] # model_list = ['ours', 'fried', 'Chen+ours'] # model_list = ['ours'] # model_list = [sys.argv[1]] # print(model_list) # file_list = glob('/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/PoseEstimate/inputs/tmp/crop/**.png') # file_list_name = [os.path.splitext(os.path.basename(f))[0] for f in file_list] indir = '/net/per610a/export/das18a/satoh-lab/wangzx/dataset/Perspective/' file_list = glob.glob(indir + 'CMDP_*/**/*_2.jpg') with open('/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output/landmark/sigresult/mapping.json', 'r') as f: dict_sig = json.load(f) # import pdb; pdb.set_trace() inver_dict_sig = {} for key_ in dict_sig: inver_dict_sig[dict_sig[key_]] = key_ face_names = [os.path.splitext(os.path.basename(dict_sig[f]))[0] for f in dict_sig] # face_names = ['X5FH-050622'] # # face_names = ['8S7R-060329', 'VND0-060329', 'AMOH-060329', 'XWQM-060208', 'P1EV-060208', # '9QAD-050615', 'KKMI-060329', 'J6XX-060208', 'QZGT-060329', '5ZAZ-060208', # 'X5FH-050622', '92RT-050419', 'QQXY-060329', '3XKB-060208', 'VIMC-050615'] # face_names = ['3XKB-060208'] # face_names = ['62RT-060329', 'X5FH-050622'] # model_list = ['fried'] theirs_folder = '/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output/landmark/sigresult/theirs' ours_folder = '/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output_compare/config43/dolly-slider-single' chen_folder = '/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output_compare/config43' pti_folder = '/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output_compare/config44' wacv_folder = '/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output_compare/config44' imp_folder = '/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output/landmark/sigresult/imp' #implemetation our_model_name = 'ours' our_transfered_model_name = 'ours_transferred' chen_model_name = 'original_pti+chen' wacv_model_name = 'wacv' pti_model_name = 'original_pti' # our_new_folder = '' main_result_folder = os.path.join('/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output/landmark/sigresult/eval4') os.makedirs(main_result_folder, exist_ok=True) target_folder_dict = {os.path.splitext(os.path.basename(f))[0]: '/'.join(f.split('/')[-3:-1]) + '/' for f in file_list} # target_folder_dict = {'62RT-060329': 'CMDP_1/15_62RT/', # 'X5FH-050622': 'CMDP_2/29_X5FH'} model_list = ['input', 'sig', 'imp', 'ours', 'ours_trans', 'chen', 'wacv', 'pti'] # model_list = ['sig'] psnr = np.zeros((len(model_list), len(face_names), len(ref_list))) ssim = np.zeros((len(model_list), len(face_names), len(ref_list))) face_id = np.zeros((len(model_list), len(face_names), len(ref_list))) lpips_score = np.zeros((len(model_list), len(face_names), len(ref_list))) lmks = np.zeros((len(model_list), len(face_names), len(ref_list))) drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1) res = {} # import pdb; pdb.set_trace() # for k in range(len(model_list)): # model_name = model_list[k] for j in range(len(face_names)): face_name = face_names[j] + '_2' face_result_folder = os.path.join(main_result_folder, face_name) + '/' target_folder = '/net/per610a/export/das18a/satoh-lab/wangzx/dataset/Perspective/' + target_folder_dict[face_name] IMAGE_FILES = {} for ref_id in range(len(dist)): IMAGE_FILES['sig' + ref_list[ref_id]] = os.path.join(theirs_folder, inver_dict_sig[face_name[:-2]] + '.jpg') IMAGE_FILES['imp' + ref_list[ref_id]] = os.path.join(imp_folder, inver_dict_sig[face_name[:-2]] + f'_{int(ref_list[ref_id]) // 2}_out.jpg') IMAGE_FILES['reference' + ref_list[ref_id]] = os.path.join(target_folder, face_name[:-2] + '_' + ref_list[ref_id] + '.jpg') IMAGE_FILES['input' + ref_list[ref_id]] = os.path.join(target_folder, face_name[:-2] + '_2.jpg') chen_old = os.path.join(chen_folder, face_name, '_'.join((face_name, chen_model_name, ref_list[ref_id] + '.png'))) IMAGE_FILES['chen' + ref_list[ref_id]] = chen_old wacv_old = os.path.join(wacv_folder, face_name, '_'.join((face_name, wacv_model_name, ref_list[ref_id] + '.png'))) IMAGE_FILES['wacv' + ref_list[ref_id]] = wacv_old pti_old = os.path.join(pti_folder, face_name, '_'.join((face_name, pti_model_name, ref_list[ref_id] + '.png'))) IMAGE_FILES['pti' + ref_list[ref_id]] = pti_old # our_old = os.path.join(ours_folder, face_name, '_'.join((face_name, our_model_name, ref_list[ref_id] + '.png'))) our_old = os.path.join(ours_folder, '_'.join((face_name, our_model_name+ '.png'))) # our_new = os.path.join(our_new_folder, '_'.join((face_name, model_name, ref_list[ref_id] + '.png'))) # shutil.copy(our_old, our_new) IMAGE_FILES['ours' + ref_list[ref_id]] = our_old IMAGE_FILES['ours_trans' + ref_list[ref_id]] = os.path.join(ours_folder, '_'.join((face_name, our_transfered_model_name+ '.png'))) align_folder = os.path.join(face_result_folder, 'align/') if not os.path.exists(align_folder): os.makedirs(align_folder) matting_folder = os.path.join(face_result_folder, 'matting/') if not os.path.exists(matting_folder): os.makedirs(matting_folder) fore_folder = os.path.join(face_result_folder, 'fore/') if not os.path.exists(fore_folder): os.makedirs(fore_folder) with mp_face_mesh.FaceMesh( static_image_mode=bool_notracking, max_num_faces=max_numoffaces, min_detection_confidence=0.1, min_tracking_confidence=0.1) as face_mesh: for idx, file_name in enumerate(IMAGE_FILES): print(IMAGE_FILES[file_name]) # import pdb; pdb.set_trace() image = cv.imread(IMAGE_FILES[file_name]) # Convert the BGR image to RGB before processing. start = timeit.default_timer() results = face_mesh.process(cv.cvtColor(image, cv.COLOR_BGR2RGB)) stop = timeit.default_timer() print('Time: ', stop - start) # # Print and draw face mesh landmarks on the image. if not results.multi_face_landmarks: continue annotated_image = image.copy() num_face = len(results.multi_face_landmarks) PupilDistance_faces = np.zeros((num_face,)) for i in range(num_face): face_landmarks = results.multi_face_landmarks[i] PupilDistance_faces[i] = face_landmarks.landmark[386].x - face_landmarks.landmark[159].x # find the max face # print(PupilDistance_faces) ind = np.argmax(PupilDistance_faces) face_landmarks = results.multi_face_landmarks[ind] # print(len(face_landmarks.landmark)) for i in range(468): pts3D[i, 0] = face_landmarks.landmark[i].x pts3D[i, 1] = face_landmarks.landmark[i].y pts3D[i, 2] = face_landmarks.landmark[i].z h, w = annotated_image.shape[0], annotated_image.shape[1] pts3D_rescale = pts3D[:,:2].copy() pts3D_rescale[:, 0] = pts3D[:, 0] * w pts3D_rescale[:, 1] = pts3D[:, 1] * h im_pil = PIL.Image.fromarray(image.copy()) if 'our' in file_name: lm, out_box = align_face(pts3D_rescale, im_pil, 512, file_name=file_name) else: lm = align_face(pts3D_rescale, im_pil, 512, file_name=file_name) res[file_name] = lm for i, xk in enumerate(ref_list): for kk in range(len(model_list)): lmks[kk, j, i] = np.linalg.norm(res[model_list[kk] + xk] - res['reference' + xk]) / 512 # lmks[1, j, i] = np.linalg.norm(res['ours' + xk] - res['reference' + xk]) / 512 # lmks[1, j, i] = np.linalg.norm(res['ours' + xk] - res['reference' + xk]) / 512 nm = model_list[kk] print(f"{nm} normalized landmark error:", lmks[kk, j, i]) # print("our normalized landmark error:", lmks[1, j, i]) os.chdir('/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/MODNet') os.system('python -m demo.image_matting.colab.inference --input-path %s --output-path %s --ckpt-path ./pretrained/modnet_photographic_portrait_matting.ckpt' % ( align_folder, matting_folder )) os.chdir('/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/scripts') for k in range(len(model_list)): model_name = model_list[k] for i, xk in enumerate(ref_list): input_image_name = os.path.join(align_folder, model_name + xk + '.png') input_image = Image.open(input_image_name) input_matte_name = os.path.join(matting_folder, model_name + xk + '.png') print(input_matte_name) input_matte = Image.open(input_matte_name) input_fore = extract_foreground(input_image, input_matte)*(np.repeat(out_box[:,:,None], 3, axis=2) > 0) + (np.repeat(out_box[:,:,None], 3, axis=2) == 0)*255 # import pdb; pdb.set_trace() Image.fromarray(np.uint8(input_fore)).save(os.path.join(fore_folder, 'input' + xk + '_fore.png')) ref_image_name = os.path.join(align_folder, 'reference' + xk + '.png') ref_image = Image.open(ref_image_name) ref_matte_name = os.path.join(matting_folder, 'reference' + xk + '.png') ref_matte = Image.open(ref_matte_name) ref_fore = extract_foreground(ref_image, ref_matte)*(np.repeat(out_box[:,:,None], 3, axis=2) > 0) + (np.repeat(out_box[:,:,None], 3, axis=2) == 0)*255 Image.fromarray(np.uint8(ref_fore)).save(os.path.join(fore_folder, 'reference' + xk + '_fore.png')) # print(ref_fore.shape) face_id[k, j, i] = get_faces_sim((np.uint8(Image.fromarray(np.uint8(ref_fore)))), np.uint8(Image.fromarray(np.uint8(input_fore))), net) # import pdb; pdb.set_trace() psnr[k, j, i] = skimage.metrics.peak_signal_noise_ratio(np.uint8(ref_fore), np.uint8(input_fore)) ssim[k, j, i] = skimage.metrics.structural_similarity(np.uint8(ref_fore), np.uint8(input_fore), multichannel = True) print("psnr:", psnr[k, j, i]) print("ssim:", ssim[k, j, i]) print("face_id:", face_id[k, j, i]) ref_fore_torch = torch.tensor(ref_fore/255.).float() input_fore_torch = torch.tensor(input_fore/255.).float() d = loss_fn.forward(ref_fore_torch.permute([2,0,1]).unsqueeze(0), input_fore_torch.permute([2,0,1]).unsqueeze(0)) lpips_score[k, j, i] = d.squeeze().detach().cpu().numpy() print("lpips:", lpips_score[k, j, i]) # sio.savemat('/net/per610a/export/das18a/satoh-lab/wangzx/src/Perspective/HFGI3D/inversion/output/landmark/irirs_dist.mat', {'dist1': dist1, # 'dist2': dist2, # 'z_mean': z_mean}) # with open(os.path.join(main_result_folder, model_list[k] + '_face_names.txt'), 'w') as fp: # fp.write("\n".join(str(item) for item in face_names)) # all_in_one{} # all_in_one['face_id'] = face_id # all_in_one['lmk'] = lmks # all_in_one['psnr'] = psnr # all_in_one['ssim'] = ssim # all_in_one['lpips'] = lpips_score for k in range(len(model_list)): np.save(os.path.join(main_result_folder, model_list[k] + '_lmk.npy'), lmks[k,:,:]) np.save(os.path.join(main_result_folder, model_list[k] + '_psnr.npy'), psnr[k,:,:]) np.save(os.path.join(main_result_folder, model_list[k] + '_ssim.npy'), ssim[k,:,:]) np.save(os.path.join(main_result_folder, model_list[k] + '_face_id.npy'), face_id[k,:,:]) np.save(os.path.join(main_result_folder, model_list[k] + '_lpips.npy'), lpips_score[k,:,:]) print(model_list[k] + 'lmk:', lmks[k,:,:].mean()) print(model_list[k] + 'psnr:', psnr[k,:,:].mean()) print(model_list[k] + 'face_id:', face_id[k,:,:].mean()) print(model_list[k] + 'ssmi:', ssim[k,:,:].mean()) print(model_list[k] + 'lpips:', lpips_score[k,:,:].mean()) print("----------------------------------") for k in range(len(model_list)): print("{} & {:.3f} & {:.3f} & {:.3f} & {:.3f} & {:.3f}".format(model_list[k], lmks[k,:,:].mean(), psnr[k,:,:].mean(), ssim[k,:,:].mean(), lpips_score[k,:,:].mean(), face_id[k,:,:].mean())) # print("----------------------------------") # # for j in range(len(face_names)): # face_result_folder = os.path.join(main_result_folder, face_names[j]) + '/' # # plt.plot(dist, lmks[:, j,:].T) # plt.xlabel('distance [m]') # plt.ylabel('normalized landmark error') # plt.legend(model_list) # plt.savefig(os.path.join(face_result_folder, face_names[j] + '_' + model_name[k] + '_lmk.png')) # plt.close() # # plt.plot(dist, psnr[:,j,:].T) # plt.xlabel('distance [m]') # plt.ylabel('PSNR [dB]') # plt.legend(model_list) # plt.savefig(os.path.join(face_result_folder, face_names[j] + '_' + model_name[k] + '_psnr.png')) # plt.close() # # plt.plot(dist, ssim[:, j,:].T) # plt.xlabel('distance [m]') # plt.ylabel('SSIM') # plt.legend(model_list) # plt.savefig(os.path.join(face_result_folder, face_names[j] + '_' + model_name[k] + '_ssim.png')) # plt.close() # # plt.plot(dist, lpips_score[:, j,:].T) # plt.xlabel('distance [m]') # plt.ylabel('LPIPS') # plt.legend(model_list) # plt.savefig(os.path.join(face_result_folder, face_names[j] + '_' + model_name[k] + '_lpips.png')) # plt.close() line_color = ['black', 'red'] fig, ax = plt.subplots(figsize=(7, 4)) for i in range((lmks.shape[0])): print(dist.shape, lmks[i, ...].mean(axis=1).squeeze().T.shape) plt.errorbar(dist, lmks[i, ...].mean(axis=0).T, yerr=lmks[i, ...].std(axis=0).squeeze().T, fmt='o', color=line_color[i], ecolor='lightgray', elinewidth=3, capsize=0) plt.legend(model_list) plt.xlabel('distance [m]') plt.ylabel('normalized landmark error') plt.savefig(os.path.join(main_result_folder, model_name + '_all_lmk.png')) plt.close() fig, ax = plt.subplots(figsize=(7, 4)) for i in range((psnr.shape[0])): plt.errorbar(dist, psnr[i, ...].mean(axis=0).squeeze().T, yerr=psnr[i, ...].std(axis=0).squeeze().T, fmt='o', color=line_color[i], ecolor='lightgray', elinewidth=3, capsize=0) plt.xlabel('distance [m]') plt.ylabel('PSNR [dB]') plt.savefig(os.path.join(main_result_folder, model_name + '_all_psnr.png')) plt.close() fig, ax = plt.subplots(figsize=(7, 4)) for i in range((ssim.shape[0])): plt.errorbar(dist, ssim[i, ...].mean(axis=0).squeeze().T, yerr=ssim[i, ...].std(axis=0).squeeze().T, fmt='o', color=line_color[i], ecolor='lightgray', elinewidth=3, capsize=0) plt.legend(model_list) plt.xlabel('distance [m]') plt.ylabel('SSIM') plt.savefig(os.path.join(main_result_folder, model_name + '_all_ssim.png')) plt.close() fig, ax = plt.subplots(figsize=(7, 4)) for i in range((lpips_score.shape[0])): plt.errorbar(dist, lpips_score[i, ...].mean(axis=0).squeeze().T, yerr=lpips_score[i, ...].std(axis=0).squeeze().T, fmt='o', color=line_color[i], ecolor='lightgray', elinewidth=3, capsize=0) plt.legend(model_list) plt.xlabel('distance [m]') plt.ylabel('LPIPS') plt.savefig(os.path.join(main_result_folder, model_name + '_all_lpips.png')) plt.close()