main.cpp

#include <opencv2/opencv.hpp>
#include <iostream>
#include <vector>
#include <algorithm>
#include <X11/Xlib.h>
#include <X11/Xutil.h>
using namespace std;
using namespace cv;


//This function returns the square of the euclidean distance between 2 points.
double dist(Point x,Point y)
{
	return ( x.x - y.x ) * ( x.x - y.x ) + ( x.y - y.y ) * ( x.y - y.y );
}


//This function returns the radius and the center of the circle given 3 points
//If a circle cannot be formed , it returns a zero radius circle centered at (0,0)
pair < Point, double > circleFromPoints( Point p1, Point p2, Point p3 )
{
	double offset = pow( p2.x, 2 ) + pow( p2.y, 2 );
	double bc = ( pow( p1.x, 2 ) + pow( p1.y, 2 ) - offset ) / 2.0;
	double cd = ( offset - pow( p3.x, 2) - pow( p3.y, 2) ) / 2.0;
	double det = ( p1.x - p2.x ) * ( p2.y - p3.y ) - ( p2.x - p3.x ) * ( p1.y - p2.y ); 
	double TOL = 0.0000001;
	if( abs( det ) < TOL ) 
	{
		cout << "POINTS TOO CLOSE" << endl;
		return make_pair( Point( 0, 0 ), 0 );
	}
	double idet = 1 / det;
	double centerx = ( bc * ( p2.y - p3.y ) - cd * ( p1.y - p2.y ) ) * idet;
	double centery = ( cd * ( p1.x - p2.x ) - bc * ( p2.x - p3.x ) ) * idet;
	double radius = sqrt( pow( p2.x - centerx, 2 ) + pow( p2.y - centery, 2) );

	return make_pair( Point( centerx, centery ), radius );
}


//When called , it simulates a click at the curent mouse cursor location
void mouseClick()
{
	int button = Button1;
	Display *display = XOpenDisplay( NULL );
	XEvent event;
	if( display == NULL )
	{
		cout << "Error connecting to display" << endl;
		exit( EXIT_FAILURE );
	}
	memset( &event, 0x00, sizeof( event ) );
	event.type = ButtonPress;
	event.xbutton.button = button;
	event.xbutton.same_screen = True;
	XQueryPointer( display, RootWindow( display, DefaultScreen( display ) ), &event.xbutton.root, &event.xbutton.window, &event.xbutton.x_root, &event.xbutton.y_root, &event.xbutton.x, &event.xbutton.y, &event.xbutton.state );
	event.xbutton.subwindow = event.xbutton.window;
	while( event.xbutton.subwindow )
	{
		event.xbutton.window = event.xbutton.subwindow;
		XQueryPointer( display, event.xbutton.window, &event.xbutton.root, &event.xbutton.subwindow, &event.xbutton.x_root, &event.xbutton.y_root, &event.xbutton.x, &event.xbutton.y, &event.xbutton.state );
	}
	if( XSendEvent( display, PointerWindow, True, 0xfff, &event ) == 0 ) 
		cout << "ERROR SENDING CLICK" << endl;
	XFlush( display );
	XCloseDisplay( display );
}


//When called, it simulates a mouse release event at the current cursor location
void mouseRelease()
{
	int button = Button1;
	Display *display = XOpenDisplay( NULL );
	XEvent event;
	if( display == NULL )
	{
		cout << "Error connecting to display" << endl;
		exit( EXIT_FAILURE );
	}
	memset( &event, 0x00, sizeof( event ) );
	event.xbutton.button = button;
	event.xbutton.same_screen = True;
	XQueryPointer( display, RootWindow( display, DefaultScreen( display ) ), &event.xbutton.root, &event.xbutton.window, &event.xbutton.x_root, &event.xbutton.y_root, &event.xbutton.x, &event.xbutton.y, &event.xbutton.state );
	event.xbutton.subwindow = event.xbutton.window;
	while( event.xbutton.subwindow )
	{
		event.xbutton.window = event.xbutton.subwindow;
		XQueryPointer( display, event.xbutton.window, &event.xbutton.root, &event.xbutton.subwindow, &event.xbutton.x_root, &event.xbutton.y_root, &event.xbutton.x, &event.xbutton.y, &event.xbutton.state );
	}
	event.type = ButtonRelease;
	event.xbutton.state = 0x100;
	if( XSendEvent( display, PointerWindow, True, 0xfff, &event ) == 0 ) 
		cout << "ERROR RELEASING" << endl;
	XFlush( display );
	XCloseDisplay(display);
}


//This function moves the mouse cursor to (x,y)
void mouseTo( int x, int y )
{
	 Display *display = XOpenDisplay( 0 );
	  Window root = DefaultRootWindow( display );
	  XWarpPointer( display, None, root, 0, 0, 0, 0, x, y );
	  XFlush( display );
	  XCloseDisplay( display );
}


//The main function :D
int main( int argc, char *argv[] )
{
	Mat frame;
	Mat back;
	Mat fore;
	vector < pair < Point, double > > palm_centers;
	VideoCapture cap( 0 );
	BackgroundSubtractorMOG2 bg;
	bg.set( "nmixtures", 3 );
	bg.set( "detectShadows", false );
	namedWindow( "Frame" );
	namedWindow( "Background" );
	int backgroundFrame = 500;
	for(;;)
	{
		vector < vector < Point > > contours;
		//Get the frame
		cap >> frame;

		//Update the current background model and get the foreground
		if( backgroundFrame > 0 )
		{
			bg.operator()( frame, fore );
			backgroundFrame--;
		}
		else
		{
			bg.operator()( frame, fore, 0 );
		}

		//Get background image to display it
		bg.getBackgroundImage( back );

		//Enhance edges in the foreground by applying erosion and dilation
		erode( fore, fore, Mat() );
		dilate( fore, fore, Mat() );

		//Find the contours in the foreground
		findContours( fore, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE );
		for( int i = 0; i < contours.size(); i++ )
			//Ignore all small insignificant areas
			if( contourArea( contours[i] ) >= 5000 )		    
			{
				//Draw contour
				vector < vector < Point > > tcontours;
				tcontours.push_back( contours[i] );
				drawContours( frame, tcontours, -1, cv::Scalar( 0, 0, 255 ), 2 );

				//Detect Hull in current contour
				vector < vector < Point > > hulls( 1 );
				vector < vector < int > > hullsI( 1 );
				convexHull( Mat( tcontours[0] ), hulls[0], false );
				convexHull( Mat( tcontours[0] ), hullsI[0], false );
				drawContours( frame, hulls, -1, cv::Scalar( 0, 255, 0 ), 2 );

				//Find minimum area rectangle to enclose hand
				RotatedRect rect = minAreaRect( Mat( tcontours[0] ) );

				//Find Convex Defects
				vector < Vec4i > defects;
				if( hullsI[0].size() > 0 )
				{
					Point2f rect_points[4];
					rect.points( rect_points );
					for( int j = 0; j < 4; j++ )
						line( frame, rect_points[j], rect_points[( j + 1 ) % 4], Scalar( 255, 0, 0), 1, 8 );
					Point rough_palm_center;
					convexityDefects( tcontours[0], hullsI[0], defects );
					if( defects.size() >= 3 )
					{
						vector < Point > palm_points;
						for(int j = 0; j < defects.size(); j++ )
						{
							int startidx = defects[j][0];
							Point ptStart( tcontours[0][startidx] );
							int endidx = defects[j][1];
							Point ptEnd( tcontours[0][endidx] );
							int faridx=defects[j][2];
							Point ptFar( tcontours[0][faridx] );
							
							//Sum up all the hull and defect points to compute average
							rough_palm_center += ptFar + ptStart + ptEnd;
							palm_points.push_back( ptFar );
							palm_points.push_back( ptStart );
							palm_points.push_back( ptEnd );
						}

						//Get palm center by 1st getting the average of all defect points, this is the rough palm center,
						//Then U chose the closest 3 points ang get the circle radius and center formed from them which is the palm center.
						rough_palm_center.x /= defects.size() * 3;
						rough_palm_center.y /= defects.size() * 3;
						Point closest_pt = palm_points[0];
						vector < pair < double, int> > distvec;
						for(int i = 0; i < palm_points.size(); i++ )
							distvec.push_back( make_pair( dist( rough_palm_center, palm_points[i] ), i ) );
						sort( distvec.begin(), distvec.end() );

						//Keep choosing 3 points till you find a circle with a valid radius
						//As there is a high chance that the closes points might be in a linear line or too close that it forms a very large circle
						pair< Point, double > soln_circle;
						for(int i = 0; i + 2 < distvec.size(); i++ )
						{
							Point p1 = palm_points[distvec[i + 0].second];
							Point p2 = palm_points[distvec[i + 1].second];
							Point p3 = palm_points[distvec[i + 2].second];
							soln_circle = circleFromPoints( p1, p2, p3 );   //Final palm center,radius
							if( soln_circle.second != 0 )
								break;
						}

						//Find avg palm centers for the last few frames to stabilize its centers, also find the avg radius
						palm_centers.push_back( soln_circle );
						if( palm_centers.size() > 10 )
							palm_centers.erase( palm_centers.begin() );
						Point palm_center;
						double radius = 0;
						for(int i = 0; i < palm_centers.size(); i++ )
						{
							palm_center += palm_centers[i].first;
							radius += palm_centers[i].second;
						}
						palm_center.x /= palm_centers.size();
						palm_center.y /= palm_centers.size();
						radius /= palm_centers.size();

						//Draw the palm center and the palm circle
						//The size of the palm gives the depth of the hand
						circle( frame, palm_center, 5, Scalar( 144, 144, 255 ), 3 );
						circle( frame, palm_center, radius,Scalar( 144, 144, 255 ), 2 );

						//Detect fingers by finding points that form an almost isosceles triangle with certain thesholds
						int no_of_fingers = 0;
						for(int j = 0; j < defects.size(); j++ )
						{
							int startidx = defects[j][0]; 
							Point ptStart( tcontours[0][startidx] );
							int endidx = defects[j][1]; 
							Point ptEnd( tcontours[0][endidx] );
							int faridx = defects[j][2]; 
							Point ptFar( tcontours[0][faridx] );
							//X o--------------------------o Y
							double Xdist = sqrt( dist( palm_center, ptFar ) );
							double Ydist = sqrt( dist( palm_center, ptStart ) );
							double length = sqrt( dist( ptFar, ptStart ) );
							double retLength = sqrt( dist( ptEnd, ptFar ) );
							
							//Play with these thresholds to improve performance
							if( length <= 3 * radius && Ydist >= 0.4 * radius && length >= 10 && retLength >= 10 && max( length, retLength ) / min( length, retLength ) >= 0.8 )
								if(min(Xdist,Ydist)/max(Xdist,Ydist)<=0.8)
								{
									if( ( Xdist >= 0.1 * radius && Xdist <= 1.3 * radius && Xdist < Ydist ) || ( Ydist >= 0.1 * radius && Ydist <= 1.3 * radius && Xdist > Ydist ) )
										line( frame, ptEnd, ptFar, Scalar( 0, 255, 0), 1 ), no_of_fingers++;
								}
						}
						no_of_fingers = min( 5, no_of_fingers );
						cout << "NO OF FINGERS: " << no_of_fingers << endl;
						mouseTo( palm_center.x, palm_center.y );   //Move the cursor corresponding to the palm
						if( no_of_fingers < 4 )   //If no of fingers is <4 , click , else release
							mouseClick();
						else
							mouseRelease();
					}
				}
			}
		if( backgroundFrame > 0 )
			putText( frame, "Recording Background", cvPoint( 30, 30 ), FONT_HERSHEY_COMPLEX_SMALL, 0.8, cvScalar( 200, 200, 250 ), 1, CV_AA );
		imshow( "Frame", frame );
		imshow( "Background", back );
		if(waitKey( 10 ) >= 0 ) 
			break;
	}
	return 0;
}