eye_closure_detection.h
#ifndef _EYE_CLOSURE_DETECTION_H
#define _EYE_CLOSURE_DETECTION_H
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <vector>
#include <iostream>
using namespace cv;
using namespace std;
extern void HorizonProjection(const Mat& src, Mat& dst);
extern void VerticalProjection(const Mat& src, Mat& dst);
extern void Diff(const Mat& src, Mat& dst);
extern void FindTopTwoPeakPos(const Mat& src, int& top1_pos, int& top2_pos);
extern Mat DrawProjectionMat(const Mat& projection);
extern Mat VPF_Horizon(const Mat& src, const Mat& IPF);
#endif //_EYE_CLOSURE_DETECTION_H
#include "eye_closure_detection.h"
#define _DEBUG_INFO
void HorizonProjection(const Mat& src, Mat& dst)
{
// accept only char type matrices
CV_Assert(src.depth() != sizeof(uchar));
dst.create(src.rows, 1, CV_32F);
int i, j;
const uchar* p;
float* p_dst;
for(i = 0; i < src.rows; i++){
p = src.ptr<uchar>(i);
p_dst = dst.ptr<float>(i);
p_dst[0] = 0;
for(j = 0; j < src.cols; j++){
p_dst[0] += p[j];
}
p_dst[0] /= src.cols;
}
}
void VerticalProjection(const Mat& src, Mat& dst)
{
// accept only char type matrices
CV_Assert(src.depth() != sizeof(uchar));
dst.create(1, src.cols, CV_32F);
int i, j;
const uchar* p;
float* p_dst = dst.ptr<float>(0);
for(j = 0; j < src.cols; j++){
p_dst[j] = 0;
for(i = 0; i < src.rows; i++){
p = src.ptr<uchar>(i);
p_dst[j] += p[j];
}
p_dst[j] /= src.rows;
}
}
void Diff(const Mat& src, Mat& dst)
{
CV_Assert(src.type() == CV_32F);
dst.create(src.size(), src.type());
int total_cnt = src.total();
dst.at<float>(0) = 0;
for(int i = 1; i < total_cnt; i++){
dst.at<float>(i) = abs(src.at<float>(i) - src.at<float>(i-1));
}
}
void FindTopTwoPeakPos(const Mat& src, int& top1_pos, int& top2_pos)
{
CV_Assert(src.type() == CV_32F);
vector<float> vec_val;
vector<int> vec_pos;
int total_cnt = src.total();
for(int i = 1; i < total_cnt-1; i++){
if(src.at<float>(i) > src.at<float>(i-1)
&& src.at<float>(i) > src.at<float>(i+1)){
vec_val.push_back(src.at<float>(i));
vec_pos.push_back(i);
}
}
float top1 = -1, top2 = -1;
int vec_pos1 = 0, vec_pos2 = 0;
for(int i = 0; i < vec_val.size(); i++){
if(top1 < vec_val[i]){
top1 = vec_val[i];
vec_pos1 = i;
}
#ifdef _DEBUG_INFO
printf("val(%.1f), pos(%d)\n", vec_val[i], vec_pos[i]);
#endif //_DEBUG_INFO
}
for(int i = 0; i < vec_val.size(); i++){
if(top2 < vec_val[i] && vec_pos1 != i){
top2 = vec_val[i];
vec_pos2 = i;
}
}
top1_pos = vec_pos[vec_pos1];
top2_pos = vec_pos[vec_pos2];
}
Mat DrawProjectionMat(const Mat& projection)
{
CV_Assert(projection.type() == CV_32F);
int total_cnt = projection.total();
const int BIN_WIDTH = 10;
const int BIN_HEIGHT = 256;
Mat histImage( BIN_HEIGHT, BIN_WIDTH*total_cnt, CV_8UC3, Scalar( 0,0,0) );
for(int i = 1; i < total_cnt; i++){
line( histImage, Point( BIN_WIDTH*(i-1), BIN_HEIGHT - cvRound(projection.at<float>(i-1)) ) ,
Point( BIN_WIDTH*(i), BIN_HEIGHT - cvRound(projection.at<float>(i)) ),
Scalar( 255, 0, 0), 2, 8, 0 );
}
return histImage;
/// Display
/*namedWindow("calcHist Demo", CV_WINDOW_AUTOSIZE );
imshow("calcHist Demo", histImage );
waitKey(0);*/
}
Mat VPF_Horizon(const Mat& src, const Mat& IPF)
{
// accept only char type matrices
CV_Assert(src.depth() != sizeof(uchar));
Mat VPF;
VPF.create(src.rows, 1, CV_32F);
int i, j;
const uchar* p;
float* p_vpf;
const float* p_ipf;
for(i = 0; i < src.rows; i++){
p = src.ptr<uchar>(i);
p_vpf = VPF.ptr<float>(i);
p_ipf = IPF.ptr<float>(i);
p_vpf[0] = 0;
for(j = 0; j < src.cols; j++){
p_vpf[0] += (p[j]-p_ipf[0])*(p[j]-p_ipf[0]);
//p_vpf[0] += (p[j]-p_ipf[0]);
}
p_vpf[0] /= src.cols;
}
return VPF;
}
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* Written (W) 2012 Michal Uricar
* Copyright (C) 2012 Michal Uricar
*/
//#include <cv.h>
//#include <cvaux.h>
//#include <highgui.h>
#include <opencv2/core/core.hpp>
#include "opencv2/imgproc/imgproc.hpp"
#include <opencv2/highgui/highgui.hpp>
#include "opencv2/objdetect/objdetect.hpp"
using namespace cv;
#include <cstring>
#include <cmath>
#include "../libflandmark/flandmark_detector.h"
#include "eye_closure_detection.h"
//#define _DEBUG_INFO
int detectFaceInImage(IplImage *orig, IplImage* input, CvHaarClassifierCascade* cascade, FLANDMARK_Model *model, int *bbox, double *landmarks)
{
int ret = 0;
// Smallest face size.
CvSize minFeatureSize = cvSize(40, 40);
int flags = CV_HAAR_DO_CANNY_PRUNING;
// How detailed should the search be.
float search_scale_factor = 1.1f;
CvMemStorage* storage;
CvSeq* rects;
int nFaces;
storage = cvCreateMemStorage(0);
cvClearMemStorage(storage);
// Detect all the faces in the greyscale image.
rects = cvHaarDetectObjects(input, cascade, storage, search_scale_factor, 2, flags, minFeatureSize);
nFaces = rects->total;
double t = (double)cvGetTickCount();
for (int iface = 0; iface < (rects ? nFaces : 0); ++iface)
{
CvRect *r = (CvRect*)cvGetSeqElem(rects, iface);
bbox[0] = r->x;
bbox[1] = r->y;
bbox[2] = r->x + r->width;
bbox[3] = r->y + r->height;
ret = flandmark_detect(input, bbox, model, landmarks);
// display landmarks
cvRectangle(orig, cvPoint(bbox[0], bbox[1]), cvPoint(bbox[2], bbox[3]), CV_RGB(255,0,0) );
cvRectangle(orig, cvPoint(model->bb[0], model->bb[1]), cvPoint(model->bb[2], model->bb[3]), CV_RGB(0,0,255) );
cvCircle(orig, cvPoint((int)landmarks[0], (int)landmarks[1]), 3, CV_RGB(0, 0,255), CV_FILLED);
for (int i = 2; i < 2*model->data.options.M; i += 2)
{
cvCircle(orig, cvPoint(int(landmarks[i]), int(landmarks[i+1])), 3, CV_RGB(255,0,0), CV_FILLED);
}
}
t = (double)cvGetTickCount() - t;
int ms = cvRound( t / ((double)cvGetTickFrequency() * 1000.0) );
if (nFaces > 0)
{
printf("Faces detected: %d; Detection of facial landmark on all faces took %d ms\n", nFaces, ms);
} else {
printf("NO Face\n");
ret = -1;
}
cvReleaseMemStorage(&storage);
return ret;
}
void extractEyes(Mat image, double* landmarks, Mat& left_eye, Mat& right_eye)
{
/*
* 5 1 2 6
*
*
* 0/7
*
*
* 3 4
*
*/
//Mat g_left_eye;
//Mat g_right_eye;
Point pt_left_up;
Point pt_right_down;
Point left_eye_center((landmarks[5*2]+landmarks[1*2])/2, (landmarks[5*2+1]+landmarks[1*2+1])/2);
Point right_eye_center((landmarks[2*2]+landmarks[6*2])/2, (landmarks[2*2+1]+landmarks[6*2+1])/2);
float eye_dist = right_eye_center.x - left_eye_center.x;
const float EYE_WIDTH_RATIO = 0.8;
const float EYE_HEIGHT_RATIO = 0.4;
// left eye
pt_left_up = Point(left_eye_center.x - eye_dist*EYE_WIDTH_RATIO/2, left_eye_center.y - eye_dist*EYE_HEIGHT_RATIO/2);
pt_right_down = Point(left_eye_center.x + eye_dist*EYE_WIDTH_RATIO/2, left_eye_center.y + eye_dist*EYE_HEIGHT_RATIO/2);
if(pt_left_up.x < 0)
pt_left_up.x = 0;
if(pt_left_up.y < 0)
pt_left_up.y = 0;
if(pt_right_down.x >= image.cols)
pt_right_down.x = image.cols-1;
if(pt_right_down.y >= image.rows)
pt_right_down.y = image.rows-1;
#ifdef _DEBUG_INFO
printf("pt_left_up(%d,%d), pt_right_down(%d,%d)\n", pt_left_up.x, pt_left_up.y,
pt_right_down.x, pt_right_down.y);
#endif // _DEBUG_INFO
left_eye = image(Rect(pt_left_up, pt_right_down));
#ifdef _DEBUG_INFO
// show left eye area
rectangle(image, Rect(pt_left_up, pt_right_down), CV_RGB(0,255,0), 1);
#endif // _DEBUG_INFO
// right eye
pt_left_up = Point(right_eye_center.x - eye_dist*EYE_WIDTH_RATIO/2, right_eye_center.y - eye_dist*EYE_HEIGHT_RATIO/2);
pt_right_down = Point(right_eye_center.x + eye_dist*EYE_WIDTH_RATIO/2, right_eye_center.y + eye_dist*EYE_HEIGHT_RATIO/2);
if(pt_left_up.x < 0)
pt_left_up.x = 0;
if(pt_left_up.y < 0)
pt_left_up.y = 0;
if(pt_right_down.x >= image.cols)
pt_right_down.x = image.cols-1;
if(pt_right_down.y >= image.rows)
pt_right_down.y = image.rows-1;
right_eye = image(Rect(pt_left_up, pt_right_down));
#ifdef _DEBUG_INFO
// show right eye area
rectangle(image, Rect(pt_left_up, pt_right_down), CV_RGB(0,255,0), 1);
imshow("image", image);
waitKey(0);
#endif // _DEBUG_INFO
}
int main( int argc, char** argv )
{
char flandmark_window[] = "flandmark_example1";
double t;
int ms;
if (argc < 2)
{
fprintf(stderr, "Usage: flandmark_1 <path_to_input_image> [<path_to_output_image>]\n");
exit(1);
}
cvNamedWindow(flandmark_window, CV_WINDOW_AUTOSIZE);
// Haar Cascade file, used for Face Detection.
char faceCascadeFilename[] = "haarcascade_frontalface_alt.xml";
// Load the HaarCascade classifier for face detection.
CvHaarClassifierCascade* faceCascade;
faceCascade = (CvHaarClassifierCascade*)cvLoad(faceCascadeFilename, 0, 0, 0);
if( !faceCascade )
{
printf("Couldnt load Face detector '%s'\n", faceCascadeFilename);
exit(1);
}
// ------------- begin flandmark load model
t = (double)cvGetTickCount();
FLANDMARK_Model * model = flandmark_init("flandmark_model.dat");
if (model == 0)
{
printf("Structure model wasn't created. Corrupted file flandmark_model.dat?\n");
exit(1);
}
t = (double)cvGetTickCount() - t;
ms = cvRound( t / ((double)cvGetTickFrequency() * 1000.0) );
printf("Structure model loaded in %d ms.\n", ms);
// ------------- end flandmark load model
// input image
IplImage *frame = cvLoadImage(argv[1]);
if (frame == NULL)
{
fprintf(stderr, "Cannot open image %s. Exiting...\n", argv[1]);
exit(1);
}
// convert image to grayscale
IplImage *frame_bw = cvCreateImage(cvSize(frame->width, frame->height), IPL_DEPTH_8U, 1);
cvConvertImage(frame, frame_bw);
int *bbox = (int*)malloc(4*sizeof(int));
double *landmarks = (double*)malloc(2*model->data.options.M*sizeof(double));
int ret = detectFaceInImage(frame, frame_bw, faceCascade, model, bbox, landmarks);
cvShowImage(flandmark_window, frame);
cvWaitKey(0);
if(0 != ret){
printf("Landmark not detected!\n");
return -1;
}
if (argc == 3)
{
printf("Saving image to file %s...\n", argv[2]);
cvSaveImage(argv[2], frame);
}
// extract eyes
Mat image_gray(frame_bw);
Mat left_eye, right_eye;
extractEyes(image_gray, landmarks, left_eye, right_eye);
// eye closure detection
Mat hp_l, hp_r, hp_l_diff;
HorizonProjection(left_eye, hp_l);
Diff(hp_l, hp_l_diff);
Mat hist_projection, hist_projection_diff;
hist_projection = DrawProjectionMat(hp_l);
hist_projection_diff = DrawProjectionMat(hp_l_diff);
/// Display
// namedWindow("hist_projection", CV_WINDOW_AUTOSIZE );
// imshow("hist_projection", hist_projection );
//namedWindow("hist_projection_diff", CV_WINDOW_AUTOSIZE );
// imshow("hist_projection_diff", hist_projection_diff );
// waitKey(0);
//int top1, top2;
//FindTopTwoPeakPos(hp_l_diff, top1, top2);
//line(left_eye, Point(0, top1), Point(left_eye.cols-1, top1), Scalar( 255, 0, 0), 1, 8, 0);
//line(left_eye, Point(0, top2), Point(left_eye.cols-1, top2), Scalar( 255, 0, 0), 1, 8, 0);
//cout << top1 << " " << top2 << endl;
Mat vpf = VPF_Horizon(left_eye, hp_l);
Mat vpf_diff;
Diff(vpf, vpf_diff);
Mat vpf_hist, vpf_diff_hist;
vpf_hist = DrawProjectionMat(vpf);
vpf_diff_hist = DrawProjectionMat(vpf_diff);
cout << hp_l << endl;
cout << vpf.size() << endl;
cout << vpf << endl;
/// Display
namedWindow("vpf_hist", CV_WINDOW_AUTOSIZE );
imshow("vpf_hist", vpf_hist );
namedWindow("vpf_diff_hist", CV_WINDOW_AUTOSIZE );
imshow("vpf_diff_hist", vpf_diff_hist );
waitKey(0);
int top1, top2;
FindTopTwoPeakPos(vpf_diff, top1, top2);
line(left_eye, Point(0, top1), Point(left_eye.cols-1, top1), Scalar( 255, 0, 0), 1, 8, 0);
line(left_eye, Point(0, top2), Point(left_eye.cols-1, top2), Scalar( 255, 0, 0), 1, 8, 0);
cout << top1 << " " << top2 << endl;
/// Display
namedWindow("left_eye", CV_WINDOW_AUTOSIZE );
imshow("left_eye", left_eye);
waitKey(0);
// cleanup
free(bbox);
free(landmarks);
cvDestroyWindow(flandmark_window);
cvReleaseImage(&frame);
cvReleaseImage(&frame_bw);
cvReleaseHaarClassifierCascade(&faceCascade);
flandmark_free(model);
}