Mediapipe 实现3D人脸检测

下载源码

git clone https://github.com/google/mediapipe.git

安装Bazelisk

./bazel-5.2.0-installer-linux-x86_64.sh --user

设置环境变量

export PATH="$PATH:$HOME/bin"

编译并运行face_mesh样例CPU

bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/face_mesh:face_mesh_cpu

export GLOG_logtostderr=1

bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_cpu --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt

编译并运行face_mesh样例GPU

bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 mediapipe/examples/desktop/face_mesh:face_mesh_gpu

export GLOG_logtostderr=1

bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_gpu --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt --input_video_path=testdir/test2.mp4 --output_video_path=testdir/test2out.mp4

编译成3D人脸特征点的动态库

配置face_detection

/home/liangbaikai/mediapipe/mediapipe/modules/face_detection/face_detection_short_range.pbtxt  

下修改

model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"

修改landmark 模型路径

/home/liangbaikai/mediapipe/mediapipe/modules/face_landmark/face_landmarks_model_loader.pbtxt

node {
  calculator: "SwitchContainer"
  input_side_packet: "ENABLE:with_attention"
  output_side_packet: "PACKET:model_path"
  options: {
    [mediapipe.SwitchContainerOptions.ext] {
      contained_node: {
        calculator: "ConstantSidePacketCalculator"
        options: {
          [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
            packet {
              string_value: "mediapipe/modules/face_landmark/face_landmark.tflite"
            }
          }
        }
      }
      contained_node: {
        calculator: "ConstantSidePacketCalculator"
        options: {
          [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
            packet {
              string_value: "/home/liangbaikai/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite"
            }
          }
        }
      }
    }
  }
}

1.在mediapipe/example/desktop/下新建文件夹myface_mesh_so

2.在myface_mesh_so文件夹新建 BUILD文件和face_landmark_gpu.cpp文件

BUILD文件：

licenses(["notice"])
package(default_visibility = ["//mediapipe/examples:__subpackages__"])

# Linux only
cc_binary(
    name = "face_landmark_gpu",
    srcs = ["face_landmark_gpu.cpp"],
    linkshared = True,
    deps = [
        "//mediapipe/graphs/face_mesh:desktop_live_gpu_calculators",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame",
        "//mediapipe/framework/formats:image_frame_opencv",
        "//mediapipe/framework/port:file_helpers",
        "//mediapipe/framework/port:opencv_highgui",
        "//mediapipe/framework/port:opencv_imgproc",
        "//mediapipe/framework/port:opencv_video",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/port:status",
        "//mediapipe/gpu:gl_calculator_helper",
        "//mediapipe/gpu:gpu_buffer",
        "//mediapipe/gpu:gpu_shared_data_internal",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/flags:parse",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/formats:rect_cc_proto",
    ],
)

cc_binary(
    name = "face_landmark_cpu",
    srcs = ["face_landmark_cpu.cpp"],
    # linkshared = True,
    deps = [
        "//mediapipe/graphs/face_mesh:desktop_live_calculators",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame",
        "//mediapipe/framework/formats:image_frame_opencv",
        "//mediapipe/framework/port:file_helpers",
        "//mediapipe/framework/port:opencv_highgui",
        "//mediapipe/framework/port:opencv_imgproc",
        "//mediapipe/framework/port:opencv_video",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/port:status",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/flags:parse",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/formats:rect_cc_proto",
    ],
)

face_landmark_gpu.cpp

#include <cstdlib>

#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_video_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"

#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"


namespace LIANGBAIKAI_3DFACE_LANDMARKS{


absl::Status Face3D_Landmarks_init_(std::string pbfilepath,
                        mediapipe::CalculatorGraph &graph,
                        mediapipe::GlCalculatorHelper &gpu_helper,
                        std::unique_ptr<mediapipe::OutputStreamPoller> &poller,
                        std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks)
{
    absl::Status run_status;
    std::cout << "GetContents ..."<< std::endl;
    std::string calculator_graph_config_contents;
    MP_RETURN_IF_ERROR(mediapipe::file::GetContents(pbfilepath,&calculator_graph_config_contents));

    // std::cout << "Get calculator graph config contents: " << calculator_graph_config_contents << std::endl;
    mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(calculator_graph_config_contents);

    std::cout << "Initialize the calculator graph."<< std::endl;
    MP_RETURN_IF_ERROR(graph.Initialize(config));
    
    std::cout << "Initialize the GPU."<< std::endl;
    ASSIGN_OR_RETURN(auto gpu_resources, mediapipe::GpuResources::Create());
    MP_RETURN_IF_ERROR(graph.SetGpuResources(std::move(gpu_resources)));
    gpu_helper.InitializeForTest(graph.GetGpuResources().get());

    // 添加视频输出流
    std::cout << "Start running the calculator graph." << std::endl;
    mediapipe::StatusOrPoller sop = graph.AddOutputStreamPoller("output_video");
    assert(sop.ok());
    poller = std::make_unique<mediapipe::OutputStreamPoller>(std::move(sop.value()));
    // 添加landmarks输出流
    mediapipe::StatusOrPoller sop_landmark = graph.AddOutputStreamPoller("multi_face_landmarks");
    assert(sop_landmark.ok());
    poller_landmarks = std::make_unique<mediapipe::OutputStreamPoller>(std::move(sop_landmark.value()));

    std::cout << "StartRun"<< std::endl;
    MP_RETURN_IF_ERROR(graph.StartRun({}));
    return absl::OkStatus(); 
}


absl::Status Face3D_Landmarks_run_(cv::Mat &rgba_mat,std::vector<cv::Point3f> &point_v,
                    mediapipe::CalculatorGraph &graph,
                    mediapipe::GlCalculatorHelper &gpu_helper,
                    std::unique_ptr<mediapipe::OutputStreamPoller> &poller,
                    std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks)
{    
    // Wrap Mat into an ImageFrame.
    auto input_frame = absl::make_unique<mediapipe::ImageFrame>(mediapipe::ImageFormat::SRGBA, rgba_mat.cols, rgba_mat.rows,mediapipe::ImageFrame::kGlDefaultAlignmentBoundary);

    cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get());
    rgba_mat.copyTo(input_frame_mat);

    // Prepare and add graph input packet.
    size_t frame_timestamp_us =(double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;

    MP_RETURN_IF_ERROR(gpu_helper.RunInGlContext(
        [&input_frame, &frame_timestamp_us, &graph,&gpu_helper]() -> absl::Status
        {
            // Convert ImageFrame to GpuBuffer.
            auto texture = gpu_helper.CreateSourceTexture(*input_frame.get());
            auto gpu_frame = texture.GetFrame<mediapipe::GpuBuffer>();
            glFlush();
            texture.Release();
            // Send GPU image packet into the graph.
            MP_RETURN_IF_ERROR(graph.AddPacketToInputStream("input_video", mediapipe::Adopt(gpu_frame.release()).At(mediapipe::Timestamp(frame_timestamp_us))));
            return absl::OkStatus(); 
        }
    ));

    mediapipe::Packet packet;
    if (!poller->Next(&packet)){
        std::cout << "no output video" << std::endl;
    }

    mediapipe::Packet packet_landmarks;
    if(poller_landmarks->QueueSize() > 0){
        if (poller_landmarks->Next(&packet_landmarks))
        {
            std::vector<mediapipe::NormalizedLandmarkList> output_landmarks = packet_landmarks.Get<std::vector<mediapipe::NormalizedLandmarkList>>();
            if(1 == output_landmarks.size()){
                mediapipe::NormalizedLandmarkList single_face_LandmarkList = output_landmarks[0];
                point_v.clear();
                for (int i = 0; i < single_face_LandmarkList.landmark_size(); ++i){
                    const mediapipe::NormalizedLandmark landmark = single_face_LandmarkList.landmark(i);
                    // std::cout << "num : " <<  i  << std::endl;
                    // std::cout << "x : " <<  landmark.x() * rgba_mat.cols  << std::endl;
                    // std::cout << "y : " <<  landmark.y() * rgba_mat.cols  << std::endl;
                    // std::cout << "z : " <<  landmark.z() * rgba_mat.cols  << std::endl;
                    point_v.push_back(cv::Point3f(landmark.x() * rgba_mat.cols,landmark.y() * rgba_mat.cols,landmark.z() * rgba_mat.cols));
                }
            }
        }
    }

    return absl::OkStatus();
}


absl::Status Face3D_Landmarks_stop_(mediapipe::CalculatorGraph &graph){
    MP_RETURN_IF_ERROR(graph.CloseInputStream("input_video"));
    return graph.WaitUntilDone();
}


int Face3D_Landmarks_init(std::string pbfilepath,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks){
    absl::Status run_status = Face3D_Landmarks_init_(pbfilepath,graph,gpu_helper,poller,poller_landmarks);
    if (!run_status.ok()){
        LOG(ERROR) << "Failed to init " << run_status.message();
        return 1;
    }
    return 0;
}

int Face3D_Landmarks_run(cv::Mat &rgba_mat,std::vector<cv::Point3f> &point_v,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks){
    absl::Status run_status = Face3D_Landmarks_run_(rgba_mat,point_v,graph,gpu_helper,poller,poller_landmarks);
    if (!run_status.ok()){
        LOG(ERROR) << "Failed to run " << run_status.message();
        return 1;
    }
    return 0;
}

int Face3D_Landmarks_stop(mediapipe::CalculatorGraph &graph){
    absl::Status run_status = Face3D_Landmarks_stop_(graph);
    if (!run_status.ok()){
        LOG(ERROR) << "Failed to stop " << run_status.message();
        return 1;
    }
    return 0;
}

}



// int main(int argc, char **argv)
// {
//     mediapipe::CalculatorGraph graph;
//     mediapipe::GlCalculatorHelper gpu_helper;
//     std::unique_ptr<mediapipe::OutputStreamPoller> poller;
//     std::unique_ptr<mediapipe::OutputStreamPoller> poller_landmarks;

//     std::string pbfile = "/home/caijun/mediapipe/mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt"; //GPU
//     LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_init(pbfile,graph,gpu_helper ,poller,poller_landmarks);

//     cv::Mat src = cv::imread("/home/caijun/mediapipe/mediapipe/examples/desktop/myface_mesh_so/test/huge.jpg");
//     cv::Mat rgba_mat;
//     cv::cvtColor(src, rgba_mat, cv::COLOR_BGR2RGBA);
//     std::vector<cv::Point3f> point_v;
//     LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_run(rgba_mat,point_v,graph,gpu_helper,poller,poller_landmarks);

//     // for(auto &i : point_v){
//     //     std::cout << i << std::endl;
//     // }

//     for(unsigned i = 0; i < point_v.size();i ++){
//         std::cout << point_v[i].x << std::endl;

//     }

//     LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_stop(graph);
//     return 0;
// }

3.编译

bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 mediapipe/examples/desktop/myface_mesh_so:face_landmark_gpu

成功后会在mediapipe/bazel-bin/mediapipe/examples/desktop/myface_mesh_so/路径下生成动态库文件

4.动态库使用样例

#include<iostream>
#include<string>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/gpu/gl_calculator_helper.h"

#include <sys/time.h>
double const lbk_get_time_ms(){
    double t;
    struct timeval tv;
    gettimeofday(&tv, NULL);
    t =  tv.tv_sec*1000. + tv.tv_usec/1000.;
return t;
}

namespace LIANGBAIKAI_3DFACE_LANDMARKS{
    int Face3D_Landmarks_init(std::string pbfilepath,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks);
    int Face3D_Landmarks_run(cv::Mat &rgba_mat,std::vector<cv::Point3f> &point_v,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks);
    int Face3D_Landmarks_stop(mediapipe::CalculatorGraph &graph);

}

int main(int argc, char **argv)
{
    mediapipe::CalculatorGraph graph;
    mediapipe::GlCalculatorHelper gpu_helper;
    std::unique_ptr<mediapipe::OutputStreamPoller> poller;
    std::unique_ptr<mediapipe::OutputStreamPoller> poller_landmarks;

    std::string pbfile = "/home/liangbaikai/mediapipe/mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt"; //GPU
    LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_init(pbfile,graph,gpu_helper ,poller,poller_landmarks);

    cv::Mat src = cv::imread("/home/liangbaikai/mediapipe/mediapipe/examples/desktop/myface_mesh_so/test/huge.jpg");
    cv::Mat rgba_mat;
    cv::cvtColor(src, rgba_mat, cv::COLOR_BGR2RGBA);
    std::vector<cv::Point3f> point_v;
    LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_run(rgba_mat,point_v,graph,gpu_helper,poller,poller_landmarks);

    for(unsigned i = 0; i < point_v.size();i ++){
        std::cout << point_v[i].x << std::endl;
    }

    LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_stop(graph);
    return 0;
}

cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
message("-- project name : test")
project(test)

set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -Wall -fPIC")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3 -Wall -fPIC -g")

set(CMAKE_BUILD_TYPE Release)
set(TARGET test)


set(OpenCV_DIR "/usr/local/share/OpenCV")
find_package(OpenCV REQUIRED)

include_directories(
    ${PROJECT_SOURCE_DIR}/include
    /home/caijun/mediapipe/ 
    /root/anaconda3/pkgs/abseil-cpp-20211102.0-hd4dd3e8_0/include/ 
    /root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/host/bin/ 
    /root/anaconda3/include 
    /root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/k8-opt/bin/external/com_github_glog_glog/src/ 
    /root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/k8-opt/bin/external/com_github_gflags_gflags/_virtual_includes/gflags/ 
    /root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/k8-opt/bin/external/com_github_glog_glog/_virtual_includes/default_glog_headers/
)

link_directories(
    /home/liangbaikai/mediapipe/bazel-bin/mediapipe/examples/desktop/myface_mesh_so
    /root/anaconda3/pkgs/abseil-cpp-20211102.0-hd4dd3e8_0/lib
)


set(SRC_LIST 
    ${PROJECT_SOURCE_DIR}/src/main.cpp
)

# add_library(${TARGET} SHARED ${SRC_LIST})
add_executable(${TARGET}  ${SRC_LIST})              #test demo


target_link_libraries(${TARGET} 
    ${OpenCV_LIBS} 
    face_landmark_gpu
    # face_landmark_cpu
    absl_raw_hash_set
)