main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Voice Authentication and Face Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import glob\n",
    "import pickle\n",
    "import cv2\n",
    "import time\n",
    "from numpy import genfromtxt\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.models import load_model\n",
    "K.set_image_data_format('channels_first')\n",
    "np.set_printoptions(threshold=np.nan)\n",
    "\n",
    "\n",
    "import pyaudio\n",
    "from IPython.display import Audio, display, clear_output\n",
    "import wave\n",
    "from scipy.io.wavfile import read\n",
    "from sklearn.mixture import GMM \n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from sklearn import preprocessing\n",
    "import python_speech_features as mfcc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Facial Encodings\n",
    "\n",
    "The model provides output as 128 dim encoding vector for the input image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#provides 128 dim embeddings for face\n",
    "def img_to_encoding(img):\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "    \n",
    "    #converting img format to channel first\n",
    "    img = np.around(np.transpose(img, (2,0,1))/255.0, decimals=12)\n",
    "\n",
    "    x_train = np.array([img])\n",
    "\n",
    "    #facial embedding from trained model\n",
    "    embedding = model.predict_on_batch(x_train)\n",
    "    return embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Triplet Loss\n",
    "\n",
    "Two encodings are compared and if they are similar then two images are of the same person otherwise they are different. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def triplet_loss(y_true, y_pred, alpha = 0.2):\n",
    "    anchor, positive, negative = y_pred[0], y_pred[1], y_pred[2]\n",
    "    \n",
    "    # triplet loss formula \n",
    "    pos_dist = tf.reduce_sum( tf.square(tf.subtract(y_pred[0], y_pred[1])) )\n",
    "    neg_dist = tf.reduce_sum( tf.square(tf.subtract(y_pred[0], y_pred[2])) )\n",
    "    basic_loss = pos_dist - neg_dist + alpha\n",
    "    \n",
    "    loss = tf.maximum(basic_loss, 0.0)\n",
    "   \n",
    "    return loss\n",
    "\n",
    "# load the model\n",
    "model = load_model('facenet_model/model.h5', custom_objects={'triplet_loss': triplet_loss})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Audio processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Calculate and returns the delta of given feature vector matrix\n",
    "def calculate_delta(array):\n",
    "    rows,cols = array.shape\n",
    "    deltas = np.zeros((rows,20))\n",
    "    N = 2\n",
    "    for i in range(rows):\n",
    "        index = []\n",
    "        j = 1\n",
    "        while j <= N:\n",
    "            if i-j < 0:\n",
    "                first = 0\n",
    "            else:\n",
    "                first = i-j\n",
    "            if i+j > rows -1:\n",
    "                second = rows -1\n",
    "            else:\n",
    "                second = i+j\n",
    "            index.append((second,first))\n",
    "            j+=1\n",
    "        deltas[i] = ( array[index[0][0]]-array[index[0][1]] + (2 * (array[index[1][0]]-array[index[1][1]])) ) / 10\n",
    "    return deltas\n",
    "\n",
    "#convert audio to mfcc features\n",
    "def extract_features(audio,rate):    \n",
    "    mfcc_feat = mfcc.mfcc(audio,rate, 0.025, 0.01,20,appendEnergy = True, nfft=1103)\n",
    "    mfcc_feat = preprocessing.scale(mfcc_feat)\n",
    "    delta = calculate_delta(mfcc_feat)\n",
    "\n",
    "    #combining both mfcc features and delta\n",
    "    combined = np.hstack((mfcc_feat,delta)) \n",
    "    return combined"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add a New User"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "recording...\n",
      "Done\n",
      "Speak your name one more time\n",
      "recording...\n",
      "Done\n",
      "Speak your name one last time\n",
      "recording...\n",
      "Done\n",
      "mmm added successfully\n"
     ]
    }
   ],
   "source": [
    "def add_user():\n",
    "    \n",
    "    name = input(\"Enter Name:\")\n",
    "     # check for existing database\n",
    "    if os.path.exists('./face_database/embeddings.pickle'):\n",
    "        with open('./face_database/embeddings.pickle', 'rb') as database:\n",
    "            db = pickle.load(database)   \n",
    "            \n",
    "            if name in db or name == 'unknown':\n",
    "                print(\"Name Already Exists! Try Another Name...\")\n",
    "                return\n",
    "    else:\n",
    "        #if database not exists than creating new database\n",
    "        db = {}\n",
    "    \n",
    "    cap = cv2.VideoCapture(0)\n",
    "    cap.set(3, 640)\n",
    "    cap.set(4, 480)\n",
    "    \n",
    "    #detecting only frontal face using haarcascade\n",
    "    face_cascade = cv2.CascadeClassifier('./haarcascades/haarcascade_frontalface_default.xml')\n",
    "    \n",
    "    i = 3\n",
    "    face_found = False\n",
    "    \n",
    "    while True:            \n",
    "        _, frame = cap.read()\n",
    "        frame = cv2.flip(frame, 1, 0)\n",
    "            \n",
    "        #time.sleep(1.0)\n",
    "        cv2.putText(frame, 'Keep Your Face infront of Camera', (100, 200),\n",
    "                    cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2)\n",
    "        \n",
    "        cv2.putText(frame, 'Starting', (260, 270), cv2.FONT_HERSHEY_SIMPLEX, \n",
    "                    0.8, (255, 255, 255), 2)\n",
    "        \n",
    "        cv2.putText(frame, str(i), (290, 330), cv2.FONT_HERSHEY_SIMPLEX, \n",
    "                    1.3, (255, 255, 255), 3)\n",
    "\n",
    "        i-=1\n",
    "                   \n",
    "        cv2.imshow('frame', frame)\n",
    "        cv2.waitKey(1000)\n",
    "        \n",
    "        if i < 0:\n",
    "            break\n",
    "            \n",
    "    start_time = time.time()        \n",
    "    img_path = './saved_image/1.jpg'\n",
    "\n",
    "    ## Face recognition \n",
    "    while True:\n",
    "        curr_time = time.time()\n",
    "        \n",
    "        _, frame = cap.read()\n",
    "        frame = cv2.flip(frame, 1, 0)\n",
    "        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "        \n",
    "        face = face_cascade.detectMultiScale(gray, 1.3, 5)\n",
    "        \n",
    "        if len(face) == 1:\n",
    "            for(x, y, w, h) in face:\n",
    "                roi = frame[y-10:y+h+10, x-10:x+w+10]\n",
    "\n",
    "                fh, fw = roi.shape[:2]\n",
    "\n",
    "                #make sure the face roi is of required height and width\n",
    "                if fh < 20 and fw < 20:\n",
    "                    continue\n",
    "\n",
    "                face_found = True\n",
    "                #cv2.imwrite(img_path, roi)\n",
    "\n",
    "                cv2.rectangle(frame, (x-10,y-10), (x+w+10, y+h+10), (255, 200, 200), 2)\n",
    "\n",
    "         \n",
    "        if curr_time - start_time >= 3:\n",
    "            break\n",
    "            \n",
    "        cv2.imshow('frame', frame)\n",
    "        cv2.waitKey(1)\n",
    "            \n",
    "    cap.release()        \n",
    "    cv2.destroyAllWindows()\n",
    "\n",
    "    \n",
    "    if face_found:\n",
    "        img = cv2.resize(roi, (96, 96))\n",
    "\n",
    "        db[name] = img_to_encoding(img)\n",
    "\n",
    "        with open('./face_database/embeddings.pickle', \"wb\") as database:\n",
    "            pickle.dump(db, database, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "    elif len(face) > 1:\n",
    "        print(\"More than one faces found. Try again...\")\n",
    "        return\n",
    "    \n",
    "    else:\n",
    "        print('There was no face found in the frame. Try again...')\n",
    "        return\n",
    "      \n",
    "    clear_output(wait=True) \n",
    "    \n",
    "    #Voice authentication\n",
    "    FORMAT = pyaudio.paInt16\n",
    "    CHANNELS = 2\n",
    "    RATE = 44100\n",
    "    CHUNK = 1024\n",
    "    RECORD_SECONDS = 3\n",
    "    \n",
    "    source = \"./voice_database/\" + name\n",
    "    \n",
    "   \n",
    "    os.mkdir(source)\n",
    "\n",
    "    for i in range(3):\n",
    "        audio = pyaudio.PyAudio()\n",
    "\n",
    "        if i == 0:\n",
    "            j = 3\n",
    "            while j>=0:\n",
    "                time.sleep(1.0)\n",
    "                print(\"Speak your name in {} seconds\".format(j))\n",
    "                clear_output(wait=True)\n",
    "\n",
    "                j-=1\n",
    "\n",
    "        elif i ==1:\n",
    "            print(\"Speak your name one more time\")\n",
    "            time.sleep(0.5)\n",
    "\n",
    "        else:\n",
    "            print(\"Speak your name one last time\")\n",
    "            time.sleep(0.5)\n",
    "\n",
    "        # start Recording\n",
    "        stream = audio.open(format=FORMAT, channels=CHANNELS,\n",
    "                    rate=RATE, input=True,\n",
    "                    frames_per_buffer=CHUNK)\n",
    "\n",
    "        print(\"recording...\")\n",
    "        frames = []\n",
    "\n",
    "        for _ in range(0, int(RATE / CHUNK * RECORD_SECONDS)):\n",
    "            data = stream.read(CHUNK)\n",
    "            frames.append(data)\n",
    "\n",
    "        # stop Recording\n",
    "        stream.stop_stream()\n",
    "        stream.close()\n",
    "        audio.terminate()\n",
    "        \n",
    "        # saving wav file of speaker\n",
    "        waveFile = wave.open(source + '/' + str((i+1)) + '.wav', 'wb')\n",
    "        waveFile.setnchannels(CHANNELS)\n",
    "        waveFile.setsampwidth(audio.get_sample_size(FORMAT))\n",
    "        waveFile.setframerate(RATE)\n",
    "        waveFile.writeframes(b''.join(frames))\n",
    "        waveFile.close()\n",
    "        print(\"Done\")\n",
    "\n",
    "    dest =  \"./gmm_models/\"\n",
    "    count = 1\n",
    "\n",
    "    for path in os.listdir(source):\n",
    "        path = os.path.join(source, path)\n",
    "\n",
    "        features = np.array([])\n",
    "        \n",
    "        # reading audio files of speaker\n",
    "        (sr, audio) = read(path)\n",
    "        \n",
    "        # extract 40 dimensional MFCC & delta MFCC features\n",
    "        vector   = extract_features(audio,sr)\n",
    "\n",
    "        if features.size == 0:\n",
    "            features = vector\n",
    "        else:\n",
    "            features = np.vstack((features, vector))\n",
    "            \n",
    "        # when features of 3 files of speaker are concatenated, then do model training\n",
    "        if count == 3:    \n",
    "            gmm = GMM(n_components = 16, n_iter = 200, covariance_type='diag',n_init = 3)\n",
    "            gmm.fit(features)\n",
    "\n",
    "            # saving the trained gaussian model\n",
    "            pickle.dump(gmm, open(dest + name + '.gmm', 'wb'))\n",
    "            print(name + ' added successfully') \n",
    "            \n",
    "            features = np.asarray(())\n",
    "            count = 0\n",
    "        count = count + 1\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    add_user()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delete User"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# deletes a registered user from database\n",
    "def delete_user():\n",
    "    name = input(\"Enter name of the user:\")\n",
    "    \n",
    "    with open(\"./face_database/embeddings.pickle\", \"rb\") as database:\n",
    "        db = pickle.load(database)\n",
    "        user = db.pop(name, None)\n",
    "    \n",
    "        if user is not None:\n",
    "            print('User ' + name + ' deleted successfully')\n",
    "            # save the database\n",
    "            with open('face_database/embeddings.pickle', 'wb') as database:\n",
    "                    pickle.dump(db, database, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "            # remove the speaker wav files and gmm model\n",
    "            [os.remove(path) for path in glob.glob('./voice_database/' + name + '/*')]\n",
    "            os.removedirs('./voice_database/' + name)\n",
    "            os.remove('./gmm_models/' + name + '.gmm')\n",
    "        \n",
    "        else:\n",
    "            print('No such user !!')\n",
    "\n",
    "delete_user()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Voice Authentication and Face Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recognize():\n",
    "    # Voice Authentication\n",
    "    FORMAT = pyaudio.paInt16\n",
    "    CHANNELS = 2\n",
    "    RATE = 44100\n",
    "    CHUNK = 1024\n",
    "    RECORD_SECONDS = 3\n",
    "    FILENAME = \"./test.wav\"\n",
    "\n",
    "    audio = pyaudio.PyAudio()\n",
    "   \n",
    "    # start Recording\n",
    "    stream = audio.open(format=FORMAT, channels=CHANNELS,\n",
    "                    rate=RATE, input=True,\n",
    "                    frames_per_buffer=CHUNK)\n",
    "\n",
    "    print(\"recording...\")\n",
    "    frames = []\n",
    "\n",
    "    for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):\n",
    "        data = stream.read(CHUNK)\n",
    "        frames.append(data)\n",
    "    print(\"finished recording\")\n",
    "\n",
    "\n",
    "    # stop Recording\n",
    "    stream.stop_stream()\n",
    "    stream.close()\n",
    "    audio.terminate()\n",
    "\n",
    "    # saving wav file \n",
    "    waveFile = wave.open(FILENAME, 'wb')\n",
    "    waveFile.setnchannels(CHANNELS)\n",
    "    waveFile.setsampwidth(audio.get_sample_size(FORMAT))\n",
    "    waveFile.setframerate(RATE)\n",
    "    waveFile.writeframes(b''.join(frames))\n",
    "    waveFile.close()\n",
    "\n",
    "    modelpath = \"./gmm_models/\"\n",
    "\n",
    "    gmm_files = [os.path.join(modelpath,fname) for fname in \n",
    "                os.listdir(modelpath) if fname.endswith('.gmm')]\n",
    "\n",
    "    models    = [pickle.load(open(fname,'rb')) for fname in gmm_files]\n",
    "\n",
    "    speakers   = [fname.split(\"/\")[-1].split(\".gmm\")[0] for fname \n",
    "                in gmm_files]\n",
    "  \n",
    "    if len(models) == 0:\n",
    "        print(\"No Users in the Database!\")\n",
    "        return\n",
    "        \n",
    "    #read test file\n",
    "    sr,audio = read(FILENAME)\n",
    "    \n",
    "    # extract mfcc features\n",
    "    vector = extract_features(audio,sr)\n",
    "    log_likelihood = np.zeros(len(models)) \n",
    "\n",
    "    #checking with each model one by one\n",
    "    for i in range(len(models)):\n",
    "        gmm = models[i]         \n",
    "        scores = np.array(gmm.score(vector))\n",
    "        log_likelihood[i] = scores.sum()\n",
    "\n",
    "    pred = np.argmax(log_likelihood)\n",
    "    identity = speakers[pred]\n",
    "   \n",
    "    # if voice not recognized than terminate the process\n",
    "    if identity == 'unknown':\n",
    "            print(\"Not Recognized! Try again...\")\n",
    "            return\n",
    "    \n",
    "    print( \"Recognized as - \", identity)\n",
    "\n",
    "    # face recognition\n",
    "    print(\"Keep Your face infront of the camera\")\n",
    "    cap = cv2.VideoCapture(0)\n",
    "    cap.set(3, 640)\n",
    "    cap.set(4, 480)\n",
    "\n",
    "    cascade = cv2.CascadeClassifier('./haarcascades/haarcascade_frontalface_default.xml')\n",
    "    \n",
    "    #loading the database \n",
    "    database = pickle.load(open('face_database/embeddings.pickle', \"rb\"))\n",
    "    \n",
    "    time.sleep(1.0)\n",
    "    \n",
    "    start_time = time.time()\n",
    "    \n",
    "    while True:\n",
    "        curr_time = time.time()\n",
    "            \n",
    "        _, frame = cap.read()\n",
    "        frame = cv2.flip(frame, 1, 0)\n",
    "        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "        face = cascade.detectMultiScale(gray, 1.3, 5)\n",
    "         \n",
    "        name = 'unknown'\n",
    "        \n",
    "        \n",
    "        if len(face) == 1:\n",
    "\n",
    "            for (x, y, w, h) in face:\n",
    "                roi = frame[y-10:y+h+10, x-10:x+w+10]\n",
    "            \n",
    "                fh, fw = roi.shape[:2]\n",
    "                min_dist = 100\n",
    "                \n",
    "                #make sure the face is of required height and width\n",
    "                if fh < 20 and fh < 20:\n",
    "                    continue\n",
    "\n",
    "                \n",
    "                #resizing image as required by the model\n",
    "                img = cv2.resize(roi, (96, 96))\n",
    "\n",
    "                #128 d encodings from pre-trained model\n",
    "                encoding = img_to_encoding(img)\n",
    "                \n",
    "                # loop over all the recorded encodings in database \n",
    "                for knownName in database:\n",
    "                    # find the similarity between the input encoding and \n",
    "                    #recorded encodings in database using L2 norm\n",
    "                    dist = np.linalg.norm(np.subtract(database[knownName], encoding) )\n",
    "                    \n",
    "                    # check if minimum distance or not\n",
    "                    if dist < min_dist:\n",
    "                        min_dist = dist\n",
    "                        name = knownName\n",
    "\n",
    "            # if min dist is less then threshold value and \n",
    "            #both face and voice matched than unlock the door\n",
    "            if min_dist < 0.4 and name == identity:\n",
    "                print (\"Door Unlocked! Welcome \" + str(name))\n",
    "                break\n",
    "\n",
    "        #open the cam for 3 seconds\n",
    "        if curr_time - start_time >= 3:\n",
    "            break    \n",
    "\n",
    "        cv2.waitKey(1)\n",
    "        cv2.imshow('frame', frame)\n",
    "        \n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()\n",
    "   \n",
    "    if len(face) == 0:\n",
    "        print('There was no face found in the frame. Try again...')\n",
    "        \n",
    "    elif len(face) > 1:\n",
    "        print(\"More than one faces found. Try again...\")\n",
    "        \n",
    "    elif min_dist > 0.5 or name != identity:\n",
    "        print(\"Not Recognized! Try again...\")\n",
    "   \n",
    "        \n",
    "if __name__ == '__main__':\n",
    "    recognize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Another version of recognizing user will keep runnning until KeyboardInterrupt by the user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recognize():\n",
    "    # Voice Authentication\n",
    "    FORMAT = pyaudio.paInt16\n",
    "    CHANNELS = 2\n",
    "    RATE = 44100\n",
    "    CHUNK = 1024\n",
    "    RECORD_SECONDS = 3\n",
    "    FILENAME = \"./test.wav\"\n",
    "    try:\n",
    "        while True:\n",
    "            audio = pyaudio.PyAudio()\n",
    "\n",
    "            # start Recording\n",
    "            stream = audio.open(format=FORMAT, channels=CHANNELS,\n",
    "                            rate=RATE, input=True,\n",
    "                            frames_per_buffer=CHUNK)\n",
    "\n",
    "            print(\"recording...\")\n",
    "            frames = []\n",
    "\n",
    "            for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):\n",
    "                data = stream.read(CHUNK)\n",
    "                frames.append(data)\n",
    "            print(\"finished recording\")\n",
    "\n",
    "\n",
    "            # stop Recording\n",
    "            stream.stop_stream()\n",
    "            stream.close()\n",
    "            audio.terminate()\n",
    "\n",
    "            # saving wav file \n",
    "            waveFile = wave.open(FILENAME, 'wb')\n",
    "            waveFile.setnchannels(CHANNELS)\n",
    "            waveFile.setsampwidth(audio.get_sample_size(FORMAT))\n",
    "            waveFile.setframerate(RATE)\n",
    "            waveFile.writeframes(b''.join(frames))\n",
    "            waveFile.close()\n",
    "\n",
    "            modelpath = \"./gmm_models/\"\n",
    "\n",
    "            gmm_files = [os.path.join(modelpath,fname) for fname in \n",
    "                        os.listdir(modelpath) if fname.endswith('.gmm')]\n",
    "\n",
    "            models    = [pickle.load(open(fname,'rb')) for fname in gmm_files]\n",
    "\n",
    "            speakers   = [fname.split(\"/\")[-1].split(\".gmm\")[0] for fname \n",
    "                        in gmm_files]\n",
    "            \n",
    "            if len(models) == 0:\n",
    "                print(\"No Users Authorized!\")\n",
    "                break\n",
    "                \n",
    "            #read test file\n",
    "            sr,audio = read(FILENAME)\n",
    "\n",
    "            # extract mfcc features\n",
    "            vector = extract_features(audio,sr)\n",
    "            log_likelihood = np.zeros(len(models)) \n",
    "\n",
    "            #checking with each model one by one\n",
    "            for i in range(len(models)):\n",
    "                gmm = models[i]         \n",
    "                scores = np.array(gmm.score(vector))\n",
    "                log_likelihood[i] = scores.sum()\n",
    "\n",
    "            pred = np.argmax(log_likelihood)\n",
    "            identity = speakers[pred]\n",
    "\n",
    "            # if voice not recognized than terminate the process\n",
    "            if identity == 'unknown':\n",
    "                    print(\"Not Recognized! Try again...\")\n",
    "                    continue\n",
    "            \n",
    "            print( \"Recognized as - \", identity)\n",
    "\n",
    "             # face recognition\n",
    "            print(\"Keep Your face infront of the camera\")\n",
    "            cap = cv2.VideoCapture(0)\n",
    "            cap.set(3, 640)\n",
    "            cap.set(4, 480)\n",
    "            img_path = './saved_image/2.jpg'\n",
    "\n",
    "            cascade = cv2.CascadeClassifier(\n",
    "                        './haarcascades/haarcascade_frontalface_default.xml')\n",
    "\n",
    "            #loading the database \n",
    "            database = pickle.load(open('face_database/embeddings.pickle', \"rb\"))\n",
    "\n",
    "            time.sleep(1.0)\n",
    "\n",
    "            start_time = time.time()\n",
    "\n",
    "            while True:\n",
    "                curr_time = time.time()\n",
    "\n",
    "                _, frame = cap.read()\n",
    "                frame = cv2.flip(frame, 1, 0)\n",
    "                gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "                face = cascade.detectMultiScale(gray, 1.3, 5)\n",
    "\n",
    "                name = 'unknown'\n",
    "                \n",
    "                if len(face) == 1:\n",
    "\n",
    "                    for (x, y, w, h) in face:\n",
    "                        roi = frame[y-10:y+h+10, x-10:x+w+10]\n",
    "\n",
    "                        fh, fw = roi.shape[:2]\n",
    "                        min_dist = 100 \n",
    "\n",
    "                        #make sure the face is of required height and width\n",
    "                        if fh < 20 and fh < 20:\n",
    "                            continue\n",
    "\n",
    "                        #resizing image as required by the model\n",
    "                        img = cv2.resize(roi, (96, 96))\n",
    "        \n",
    "                        #128 d encodings from pre-trained model\n",
    "                        encoding = img_to_encoding(img)\n",
    "\n",
    "                        # loop over all the recorded encodings in database \n",
    "                        for knownName in database:\n",
    "                            # find the similarity between the input encodings \n",
    "                            # and recorded encodings in database using L2 norm\n",
    "                            dist = np.linalg.norm(np.subtract(database[knownName], encoding) )\n",
    "                            \n",
    "                            # check if minimum distance or not\n",
    "                            if dist < min_dist:\n",
    "                                min_dist = dist\n",
    "                                name = knownName\n",
    "\n",
    "                    # if min dist is less then threshold value \n",
    "                    # and both face and voice matched than unlock the door\n",
    "                    if min_dist < 0.4 and name == identity:\n",
    "                        print (\"Door Unlocked! Welcome \" + str(name))\n",
    "                        break\n",
    "\n",
    "                #open the cam for 3 seconds\n",
    "                if curr_time - start_time >= 3:\n",
    "                    break\n",
    "\n",
    "                cv2.waitKey(1)\n",
    "                cv2.imshow('frame', frame)\n",
    "\n",
    "            cap.release()\n",
    "            cv2.destroyAllWindows()\n",
    "\n",
    "            if len(face) == 0:\n",
    "                print('There was no face found in the frame. Try again...')\n",
    "                continue\n",
    "\n",
    "            elif len(face) > 1:\n",
    "                print(\"More than one faces found. Try again...\")\n",
    "                continue\n",
    "\n",
    "            elif min_dist > 0.4 or name != identity:\n",
    "                print(\"Not Recognized! Try again...\")\n",
    "                continue\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Stopped\")\n",
    "        pass\n",
    "    \n",
    "if __name__ == '__main__':\n",
    "    recognize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.mixture import "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.19.2'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sklearn\n",
    "sklearn.__version__"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}