[ORT][TRT] support FaceFusion

cmake/opencv.cmake

@@ -19,9 +19,9 @@ link_directories(${OpenCV_DIR}/lib)
 
 if(NOT WIN32)
     if(ENABLE_OPENCV_VIDEOIO OR ENABLE_TEST)
-        set(OpenCV_LIBS opencv_core opencv_imgproc opencv_imgcodecs opencv_video opencv_videoio)
+        set(OpenCV_LIBS opencv_core opencv_imgproc opencv_imgcodecs opencv_video opencv_videoio opencv_calib3d)
     else()
-        set(OpenCV_LIBS opencv_core opencv_imgproc opencv_imgcodecs) # no videoio, video module
+        set(OpenCV_LIBS opencv_core opencv_imgproc opencv_imgcodecs opencv_calib3d) # no videoio, video module
     endif()
 else()
     set(OpenCV_LIBS opencv_world490)

examples/lite/CMakeLists.txt

@@ -103,5 +103,10 @@ add_lite_executable(lite_face_parsing_bisenet_dyn cv)
 add_lite_executable(lite_yolov8face cv)
 add_lite_executable(lite_lightenhance cv)
 add_lite_executable(lite_realesrgan cv)
+add_lite_executable(lite_face_68landmarks cv)
+add_lite_executable(lite_face_recognizer cv)
+add_lite_executable(lite_face_swap cv)
+add_lite_executable(lite_face_restoration cv)
+add_lite_executable(lite_facefusion_pipeline cv)
 add_lite_executable(lite_yolov8 cv)
 add_lite_executable(lite_sd_pipeline sd)

examples/lite/cv/test_lite_face_68landmarks.cpp

@@ -0,0 +1,35 @@
+//
+// Created by wangzijian on 11/1/24.
+//
+#include "lite/lite.h"
+
+static void test_default()
+{
+#ifdef ENABLE_ONNXRUNTIME
+    std::string onnx_path = "../../../examples/hub/onnx/cv/2dfan4.onnx";
+    std::string test_img_path = "../../../examples/lite/resources/test_lite_facefusion_pipeline_source.jpg";
+
+    // 1. Test Default Engine ONNXRuntime
+    lite::cv::faceid::Face_68Landmarks *face68Landmarks = new lite::cv::faceid::Face_68Landmarks(onnx_path);
+
+    lite::types::BoundingBoxType<float, float> bbox;
+    bbox.x1 = 487;
+    bbox.y1 = 236;
+    bbox.x2 = 784;
+    bbox.y2 = 624;
+
+    cv::Mat img_bgr = cv::imread(test_img_path);
+    std::vector<cv::Point2f> face_landmark_5of68;
+    face68Landmarks->detect(img_bgr, bbox, face_landmark_5of68);
+
+    std::cout<<"face id detect done!"<<std::endl;
+
+    delete face68Landmarks;
+#endif
+}
+
+int main(__unused int argc, __unused char *argv[])
+{
+    test_default();
+    return 0;
+}

examples/lite/cv/test_lite_face_recognizer.cpp

@@ -0,0 +1,39 @@
+//
+// Created by wangzijian on 11/5/24.
+//
+#include "lite/lite.h"
+
+static void test_default()
+{
+#ifdef ENABLE_ONNXRUNTIME
+    std::string onnx_path = "../../../examples/hub/onnx/cv/arcface_w600k_r50.onnx";
+    std::string test_img_path = "../../../examples/lite/resources/test_lite_facefusion_pipeline_source.jpg";
+
+    // 1. Test Default Engine ONNXRuntime
+    lite::cv::faceid::Face_Recognizer *face_recognizer = new lite::cv::faceid::Face_Recognizer(onnx_path);
+
+    std::vector<cv::Point2f> face_landmark_5 = {
+            cv::Point2f(568.2485f, 398.9512f),
+            cv::Point2f(701.7346f, 399.64795f),
+            cv::Point2f(634.2213f, 482.92694f),
+            cv::Point2f(583.5656f, 543.10187f),
+            cv::Point2f(684.52405f, 543.125f)
+    };
+    cv::Mat img_bgr = cv::imread(test_img_path);
+
+    std::vector<float> source_image_embeding;
+
+    face_recognizer->detect(img_bgr,face_landmark_5,source_image_embeding);
+
+
+    std::cout<<"face id detect done!"<<std::endl;
+
+    delete face_recognizer;
+#endif
+}
+
+int main(__unused int argc, __unused char *argv[])
+{
+    test_default();
+    return 0;
+}

examples/lite/cv/test_lite_face_restoration.cpp

@@ -0,0 +1,38 @@
+//
+// Created by wangzijian on 11/7/24.
+//
+#include "lite/lite.h"
+
+static void test_default()
+{
+#ifdef ENABLE_ONNXRUNTIME
+    std::string onnx_path = "../../../examples/hub/onnx/cv/gfpgan_1.4.onnx";
+    std::string test_img_path = "../../../examples/lite/resources/test_lite_facefusion_pipeline_source.jpg";
+    std::string save_img_path = "../../../examples/lite/resources/test_lite_facefusion_pipeline_target.jpg";
+
+    // 1. Test Default Engine ONNXRuntime
+    lite::cv::face::restoration::GFPGAN *face_restoration = new  lite::cv::face::restoration::GFPGAN(onnx_path);
+
+    std::vector<cv::Point2f> face_landmark_5 = {
+            cv::Point2f(569.092041f, 398.845886f),
+            cv::Point2f(701.891724f, 399.156677f),
+            cv::Point2f(634.767212f, 482.927216f),
+            cv::Point2f(584.270996f, 543.294617f),
+            cv::Point2f(684.877991f, 543.067078f)
+    };
+    cv::Mat img_bgr = cv::imread(test_img_path);
+
+    face_restoration->detect(img_bgr,face_landmark_5,save_img_path);
+
+
+    std::cout<<"face restoration detect done!"<<std::endl;
+
+    delete face_restoration;
+#endif
+}
+
+int main(__unused int argc, __unused char *argv[])
+{
+    test_default();
+    return 0;
+}

examples/lite/cv/test_lite_face_swap.cpp

@@ -0,0 +1,55 @@
+//
+// Created by wangzijian on 11/5/24.
+//
+#include "lite/lite.h"
+
+static void test_default()
+{
+#ifdef ENABLE_ONNXRUNTIME
+    std::string face_swap_onnx_path = "../../../examples/hub/onnx/cv/inswapper_128.onnx";
+    std::string face_detect_onnx_path = "../../../examples/hub/onnx/cv/yoloface_8n.onnx";
+    std::string face_landmarks_68 = "../../../examples/hub/onnx/cv/2dfan4.onnx";
+    std::string face_recognizer_onnx_path = "../../../examples/hub/onnx/cv/arcface_w600k_r50.onnx";;
+
+    std::string source_image_path = "../../../examples/lite/resources/test_lite_facefusion_pipeline_source.jpg";
+    std::string target_image_path = "../../../examples/lite/resources/test_lite_facefusion_pipeline_target.jpg";
+
+    lite::cv::face::detect::YOLOV8Face *yolov8_face = new lite::cv::face::detect::YOLOV8Face(face_detect_onnx_path);
+    lite::cv::faceid::Face_68Landmarks *face68Landmarks = new lite::cv::faceid::Face_68Landmarks(face_landmarks_68);
+    lite::cv::faceid::Face_Recognizer *face_recognizer = new lite::cv::faceid::Face_Recognizer(face_recognizer_onnx_path);
+    lite::cv::face::swap::InSwapper *face_swap = new lite::cv::face::swap::InSwapper(face_swap_onnx_path);
+
+
+    std::vector<lite::types::Boxf> detected_boxes;
+    cv::Mat img_bgr = cv::imread(source_image_path);
+    yolov8_face->detect(img_bgr, detected_boxes);
+    int position = 0; // 0号位置的脸部
+    auto test_bounding_box = detected_boxes[0];
+    std::vector<cv::Point2f> face_landmark_5of68;
+    face68Landmarks->detect(img_bgr, test_bounding_box, face_landmark_5of68);
+    std::vector<float> source_image_embeding;
+    face_recognizer->detect(img_bgr,face_landmark_5of68,source_image_embeding);
+
+    // 上面是source的 现在下面是target的
+    std::vector<lite::types::Boxf> target_detected_boxes;
+    cv::Mat target_img_bgr = cv::imread(target_image_path);
+    yolov8_face->detect(target_img_bgr, target_detected_boxes);
+    auto target_test_bounding_box = target_detected_boxes[0];
+    std::vector<cv::Point2f> target_face_landmark_5of68;
+    face68Landmarks->detect(target_img_bgr, target_test_bounding_box,target_face_landmark_5of68);
+
+    cv::Mat face_swap_image;
+    face_swap->detect(target_img_bgr,source_image_embeding,target_face_landmark_5of68,face_swap_image);
+
+    delete yolov8_face;
+    delete face68Landmarks;
+    delete face_swap;
+    delete face_recognizer;
+#endif
+}
+
+int main(__unused int argc, __unused char *argv[])
+{
+    test_default();
+    return 0;
+}

examples/lite/cv/test_lite_facefusion_pipeline.cpp

@@ -0,0 +1,45 @@
+//
+// Created by wangzijian on 11/7/24.
+//
+#include "lite/lite.h"
+static void test_default()
+{
+#ifdef ENABLE_ONNXRUNTIME
+    std::string face_swap_onnx_path = "../../../examples/hub/onnx/cv/inswapper_128.onnx";
+    std::string face_detect_onnx_path = "../../../examples/hub/onnx/cv/yoloface_8n.onnx";
+    std::string face_landmarks_68 = "../../../examples/hub/onnx/cv/2dfan4.onnx";
+    std::string face_recognizer_onnx_path = "../../../examples/hub/onnx/cv/arcface_w600k_r50.onnx";
+    std::string face_restoration_onnx_path = "../../../examples/hub/onnx/cv/gfpgan_1.4.onnx";
+
+    auto pipeLine =  lite::cv::face::swap::facefusion::PipeLine(
+            face_detect_onnx_path,
+            face_landmarks_68,
+            face_recognizer_onnx_path,
+            face_swap_onnx_path,
+            face_restoration_onnx_path
+            );
+
+    std::string source_image_path = "../../../examples/lite/resources/test_lite_facefusion_pipeline_source.jpg";
+    std::string target_image_path = "../../../examples/lite/resources/test_lite_facefusion_pipeline_target.jpg";
+    std::string save_image_path = "../../../examples/logs/test_lite_facefusion_pipeline_result.jpg";
+
+
+    // 写一个测试时间的代码
+    auto start = std::chrono::high_resolution_clock::now();
+
+
+
+    pipeLine.detect(source_image_path,target_image_path,save_image_path);
+    auto end = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<double> diff = end-start;
+    std::cout << "Time: " << diff.count() << " s\n";
+
+
+#endif
+}
+
+int main()
+{
+
+    test_default();
+}

lite/models.h

@@ -115,6 +115,11 @@
 #include "lite/ort/cv/yolofacev8.h"
 #include "lite/ort/cv/light_enhance.h"
 #include "lite/ort/cv/real_esr_gan.h"
+#include "lite/ort/cv/face_68landmarks.h"
+#include "lite/ort/cv/face_recognizer.h"
+#include "lite/ort/cv/face_swap.h"
+#include "lite/ort/cv/face_restoration.h"
+#include "lite/ort/cv/face_fusion_pipeline.h"
 #include "lite/ort/sd/clip.h"
 #include "lite/ort/sd/unet.h"
 #include "lite/ort/sd/vae.h"
@@ -502,6 +507,11 @@ namespace lite
       typedef ortcv::YoloFaceV8 _ONNXYOLOFaceNet;
       typedef ortcv::LightEnhance _ONNXLightEnhance;
       typedef ortcv::RealESRGAN _ONNXRealESRGAN;
+      typedef ortcv::Face_68Landmarks _ONNXFace_68Landmarks;
+      typedef ortcv::Face_Recognizer _ONNXFace_Recognizer;
+      typedef ortcv::Face_Swap _ONNXFace_Swap;
+      typedef ortcv::Face_Restoration _ONNXFace_Restoration;
+      typedef ortcv::Face_Fusion_Pipeline _ONNXFace_Fusion_Pipeline;
 
       // 1. classification
       namespace classification
@@ -580,6 +590,25 @@ namespace lite
 
         }
 
+        namespace swap
+        {
+            namespace facefusion
+            {
+                typedef _ONNXYOLOFaceNet YOLOV8Face;
+                typedef _ONNXFace_Swap InSwapper;
+                typedef _ONNXFace_Restoration GFPGAN;
+                typedef _ONNXFace_68Landmarks Face_68Landmarks;
+                typedef _ONNXFace_Recognizer Face_Recognizer;
+                typedef _ONNXFace_Fusion_Pipeline PipeLine;
+            }
+            typedef _ONNXFace_Swap InSwapper;
+        }
+
+        namespace restoration
+        {
+            typedef _ONNXFace_Restoration GFPGAN;
+        }
+
         namespace pose
         {
           typedef _ONNXFSANet FSANet; // head pose estimation.
@@ -618,7 +647,8 @@ namespace lite
         typedef _ONNXCavaGhostArcFace CavaGhostArcFace;
         typedef _ONNXCavaCombinedFace CavaCombinedFace;
         typedef _ONNXMobileSEFocalFace MobileSEFocalFace;
-
+        typedef _ONNXFace_68Landmarks Face_68Landmarks;
+        typedef _ONNXFace_Recognizer Face_Recognizer;
       }
       // 5. segmentation
       namespace segmentation