data_processing/extract_slices.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import clang.cindex\n",
    "import clang.enumerations\n",
    "import csv\n",
    "import numpy as np\n",
    "import os\n",
    "import re \n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "# set the config\n",
    "try:\n",
    "    clang.cindex.Config.set_library_path(\"/usr/lib/x86_64-linux-gnu\")\n",
    "    clang.cindex.Config.set_library_file('/usr/lib/x86_64-linux-gnu/libclang-6.0.so.1')\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26733\n"
     ]
    }
   ],
   "source": [
    "split_dir = '../data/neurips_parsed/neurips_data/'\n",
    "parsed = '../data/neurips_parsed/parsed_results/'\n",
    "# split_dir = '../data/chrome_debian/raw_code/'\n",
    "# parsed = '../data/chrome_debian/parsed/'\n",
    "ggnn_json_data = json.load(open('../data/ggnn_input/devign_cfg_full_text_files.json'))\n",
    "files = [d['file_name'] for d in ggnn_json_data]\n",
    "print(len(files))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "l_funcs = ['StrNCat', 'getaddrinfo', '_ui64toa', 'fclose', 'pthread_mutex_lock', 'gets_s', 'sleep', \n",
    "           '_ui64tot', 'freopen_s', '_ui64tow', 'send', 'lstrcat', 'HMAC_Update', '__fxstat', 'StrCatBuff', \n",
    "           '_mbscat', '_mbstok_s', '_cprintf_s', 'ldap_search_init_page', 'memmove_s', 'ctime_s', 'vswprintf', \n",
    "           'vswprintf_s', '_snwprintf', '_gmtime_s', '_tccpy', '*RC6*', '_mbslwr_s', 'random', \n",
    "           '__wcstof_internal', '_wcslwr_s', '_ctime32_s', 'wcsncat*', 'MD5_Init', '_ultoa', \n",
    "           'snprintf', 'memset', 'syslog', '_vsnprintf_s', 'HeapAlloc', 'pthread_mutex_destroy', \n",
    "           'ChangeWindowMessageFilter', '_ultot', 'crypt_r', '_strupr_s_l', 'LoadLibraryExA', '_strerror_s', \n",
    "           'LoadLibraryExW', 'wvsprintf', 'MoveFileEx', '_strdate_s', 'SHA1', 'sprintfW', 'StrCatNW', \n",
    "           '_scanf_s_l', 'pthread_attr_init', '_wtmpnam_s', 'snscanf', '_sprintf_s_l', 'dlopen', \n",
    "           'sprintfA', 'timed_mutex', 'OemToCharA', 'ldap_delete_ext', 'sethostid', 'popen', 'OemToCharW', \n",
    "           '_gettws', 'vfork', '_wcsnset_s_l', 'sendmsg', '_mbsncat', 'wvnsprintfA', 'HeapFree', '_wcserror_s', \n",
    "           'realloc', '_snprintf*', 'wcstok', '_strncat*', 'StrNCpy', '_wasctime_s', 'push*', '_lfind_s', \n",
    "           'CC_SHA512', 'ldap_compare_ext_s', 'wcscat_s', 'strdup', '_chsize_s', 'sprintf_s', 'CC_MD4_Init', \n",
    "           'wcsncpy', '_wfreopen_s', '_wcsupr_s', '_searchenv_s', 'ldap_modify_ext_s', '_wsplitpath', \n",
    "           'CC_SHA384_Final', 'MD2', 'RtlCopyMemory', 'lstrcatW', 'MD4', 'MD5', '_wcstok_s_l', '_vsnwprintf_s', \n",
    "           'ldap_modify_s', 'strerror', '_lsearch_s', '_mbsnbcat_s', '_wsplitpath_s', 'MD4_Update', '_mbccpy_s', \n",
    "           '_strncpy_s_l', '_snprintf_s', 'CC_SHA512_Init', 'fwscanf_s', '_snwprintf_s', 'CC_SHA1', 'swprintf', \n",
    "           'fprintf', 'EVP_DigestInit_ex', 'strlen', 'SHA1_Init', 'strncat', '_getws_s', 'CC_MD4_Final', \n",
    "           'wnsprintfW', 'lcong48', 'lrand48', 'write', 'HMAC_Init', '_wfopen_s', 'wmemchr', '_tmakepath', \n",
    "           'wnsprintfA', 'lstrcpynW', 'scanf_s', '_mbsncpy_s_l', '_localtime64_s', 'fstream.open', '_wmakepath', \n",
    "           'Connection.open', '_tccat', 'valloc', 'setgroups', 'unlink', 'fstream.put', 'wsprintfA', '*SHA1*', \n",
    "           '_wsearchenv_s', 'ualstrcpyA', 'CC_MD5_Update', 'strerror_s', 'HeapCreate', 'ualstrcpyW', '__xstat', \n",
    "           '_wmktemp_s', 'StrCatChainW', 'ldap_search_st', '_mbstowcs_s_l', 'ldap_modify_ext', '_mbsset_s', \n",
    "           'strncpy_s', 'move', 'execle', 'StrCat', 'xrealloc', 'wcsncpy_s', '_tcsncpy*', 'execlp', \n",
    "           'RIPEMD160_Final', 'ldap_search_s', 'EnterCriticalSection', '_wctomb_s_l', 'fwrite', '_gmtime64_s', \n",
    "           'sscanf_s', 'wcscat', '_strupr_s', 'wcrtomb_s', 'VirtualLock', 'ldap_add_ext_s', '_mbscpy', \n",
    "           '_localtime32_s', 'lstrcpy', '_wcsncpy*', 'CC_SHA1_Init', '_getts', '_wfopen', '__xstat64', \n",
    "           'strcoll', '_fwscanf_s_l', '_mbslwr_s_l', 'RegOpenKey', 'makepath', 'seed48', 'CC_SHA256', \n",
    "           'sendto', 'execv', 'CalculateDigest', 'memchr', '_mbscpy_s', '_strtime_s', 'ldap_search_ext_s', \n",
    "           '_chmod', 'flock', '__fxstat64', '_vsntprintf', 'CC_SHA256_Init', '_itoa_s', '__wcserror_s', \n",
    "           '_gcvt_s', 'fstream.write', 'sprintf', 'recursive_mutex', 'strrchr', 'gethostbyaddr', '_wcsupr_s_l', \n",
    "           'strcspn', 'MD5_Final', 'asprintf', '_wcstombs_s_l', '_tcstok', 'free', 'MD2_Final', 'asctime_s', \n",
    "           '_alloca', '_wputenv_s', '_wcsset_s', '_wcslwr_s_l', 'SHA1_Update', 'filebuf.sputc', 'filebuf.sputn', \n",
    "           'SQLConnect', 'ldap_compare', 'mbstowcs_s', 'HMAC_Final', 'pthread_condattr_init', '_ultow_s', 'rand', \n",
    "           'ofstream.put', 'CC_SHA224_Final', 'lstrcpynA', 'bcopy', 'system', 'CreateFile*', 'wcscpy_s', \n",
    "           '_mbsnbcpy*', 'open', '_vsnwprintf', 'strncpy', 'getopt_long', 'CC_SHA512_Final', '_vsprintf_s_l', \n",
    "           'scanf', 'mkdir', '_localtime_s', '_snprintf', '_mbccpy_s_l', 'memcmp', 'final', '_ultoa_s', \n",
    "           'lstrcpyW', 'LoadModule', '_swprintf_s_l', 'MD5_Update', '_mbsnset_s_l', '_wstrtime_s', '_strnset_s', \n",
    "           'lstrcpyA', '_mbsnbcpy_s', 'mlock', 'IsBadHugeWritePtr', 'copy', '_mbsnbcpy_s_l', 'wnsprintf', \n",
    "           'wcscpy', 'ShellExecute', 'CC_MD4', '_ultow', '_vsnwprintf_s_l', 'lstrcpyn', 'CC_SHA1_Final', \n",
    "           'vsnprintf', '_mbsnbset_s', '_i64tow', 'SHA256_Init', 'wvnsprintf', 'RegCreateKey', 'strtok_s', \n",
    "           '_wctime32_s', '_i64toa', 'CC_MD5_Final', 'wmemcpy', 'WinExec', 'CreateDirectory*', \n",
    "           'CC_SHA256_Update', '_vsnprintf_s_l', 'jrand48', 'wsprintf', 'ldap_rename_ext_s', 'filebuf.open', \n",
    "           '_wsystem', 'SHA256_Update', '_cwscanf_s', 'wsprintfW', '_sntscanf', '_splitpath', 'fscanf_s', \n",
    "           'strpbrk', 'wcstombs_s', 'wscanf', '_mbsnbcat_s_l', 'strcpynA', 'pthread_cond_init', 'wcsrtombs_s', \n",
    "           '_wsopen_s', 'CharToOemBuffA', 'RIPEMD160_Update', '_tscanf', 'HMAC', 'StrCCpy', 'Connection.connect', \n",
    "           'lstrcatn', '_mbstok', '_mbsncpy', 'CC_SHA384_Update', 'create_directories', 'pthread_mutex_unlock', \n",
    "           'CFile.Open', 'connect', '_vswprintf_s_l', '_snscanf_s_l', 'fputc', '_wscanf_s', '_snprintf_s_l', \n",
    "           'strtok', '_strtok_s_l', 'lstrcatA', 'snwscanf', 'pthread_mutex_init', 'fputs', 'CC_SHA384_Init', \n",
    "           '_putenv_s', 'CharToOemBuffW', 'pthread_mutex_trylock', '__wcstoul_internal', '_memccpy', \n",
    "           '_snwprintf_s_l', '_strncpy*', 'wmemset', 'MD4_Init', '*RC4*', 'strcpyW', '_ecvt_s', 'memcpy_s', \n",
    "           'erand48', 'IsBadHugeReadPtr', 'strcpyA', 'HeapReAlloc', 'memcpy', 'ldap_rename_ext', 'fopen_s', \n",
    "           'srandom', '_cgetws_s', '_makepath', 'SHA256_Final', 'remove', '_mbsupr_s', 'pthread_mutexattr_init', \n",
    "           '__wcstold_internal', 'StrCpy', 'ldap_delete', 'wmemmove_s', '_mkdir', 'strcat', '_cscanf_s_l', \n",
    "           'StrCAdd', 'swprintf_s', '_strnset_s_l', 'close', 'ldap_delete_ext_s', 'ldap_modrdn', 'strchr', \n",
    "           '_gmtime32_s', '_ftcscat', 'lstrcatnA', '_tcsncat', 'OemToChar', 'mutex', 'CharToOem', 'strcpy_s', \n",
    "           'lstrcatnW', '_wscanf_s_l', '__lxstat64', 'memalign', 'MD2_Init', 'StrCatBuffW', 'StrCpyN', 'CC_MD5', \n",
    "           'StrCpyA', 'StrCatBuffA', 'StrCpyW', 'tmpnam_r', '_vsnprintf', 'strcatA', 'StrCpyNW', '_mbsnbset_s_l', \n",
    "           'EVP_DigestInit', '_stscanf', 'CC_MD2', '_tcscat', 'StrCpyNA', 'xmalloc', '_tcslen', '*MD4*', \n",
    "           'vasprintf', 'strxfrm', 'chmod', 'ldap_add_ext', 'alloca', '_snscanf_s', 'IsBadWritePtr', 'swscanf_s', \n",
    "           'wmemcpy_s', '_itoa', '_ui64toa_s', 'EVP_DigestUpdate', '__wcstol_internal', '_itow', 'StrNCatW', \n",
    "           'strncat_s', 'ualstrcpy', 'execvp', '_mbccat', 'EVP_MD_CTX_init', 'assert', 'ofstream.write', \n",
    "           'ldap_add', '_sscanf_s_l', 'drand48', 'CharToOemW', 'swscanf', '_itow_s', 'RIPEMD160_Init', \n",
    "           'CopyMemory', 'initstate', 'getpwuid', 'vsprintf', '_fcvt_s', 'CharToOemA', 'setuid', 'malloc', \n",
    "           'StrCatNA', 'strcat_s', 'srand', 'getwd', '_controlfp_s', 'olestrcpy', '__wcstod_internal', \n",
    "           '_mbsnbcat', 'lstrncat', 'des_*', 'CC_SHA224_Init', 'set*', 'vsprintf_s', 'SHA1_Final', '_umask_s', \n",
    "           'gets', 'setstate', 'wvsprintfW', 'LoadLibraryEx', 'ofstream.open', 'calloc', '_mbstrlen', \n",
    "           '_cgets_s', '_sopen_s', 'IsBadStringPtr', 'wcsncat_s', 'add*', 'nrand48', 'create_directory', \n",
    "           'ldap_search_ext', '_i64toa_s', '_ltoa_s', '_cwscanf_s_l', 'wmemcmp', '__lxstat', 'lstrlen', \n",
    "           'pthread_condattr_destroy', '_ftcscpy', 'wcstok_s', '__xmknod', 'pthread_attr_destroy', 'sethostname', \n",
    "           '_fscanf_s_l', 'StrCatN', 'RegEnumKey', '_tcsncpy', 'strcatW', 'AfxLoadLibrary', 'setenv', 'tmpnam', \n",
    "           '_mbsncat_s_l', '_wstrdate_s', '_wctime64_s', '_i64tow_s', 'CC_MD4_Update', 'ldap_add_s', '_umask', \n",
    "           'CC_SHA1_Update', '_wcsset_s_l', '_mbsupr_s_l', 'strstr', '_tsplitpath', 'memmove', '_tcscpy', \n",
    "           'vsnprintf_s', 'strcmp', 'wvnsprintfW', 'tmpfile', 'ldap_modify', '_mbsncat*', 'mrand48', 'sizeof', \n",
    "           'StrCatA', '_ltow_s', '*desencrypt*', 'StrCatW', '_mbccpy', 'CC_MD2_Init', 'RIPEMD160', 'ldap_search', \n",
    "           'CC_SHA224', 'mbsrtowcs_s', 'update', 'ldap_delete_s', 'getnameinfo', '*RC5*', '_wcsncat_s_l', \n",
    "           'DriverManager.getConnection', 'socket', '_cscanf_s', 'ldap_modrdn_s', '_wopen', 'CC_SHA256_Final', \n",
    "           '_snwprintf*', 'MD2_Update', 'strcpy', '_strncat_s_l', 'CC_MD5_Init', 'mbscpy', 'wmemmove', \n",
    "           'LoadLibraryW', '_mbslen', '*alloc', '_mbsncat_s', 'LoadLibraryA', 'fopen', 'StrLen', 'delete', \n",
    "           '_splitpath_s', 'CreateFileTransacted*', 'MD4_Final', '_open', 'CC_SHA384', 'wcslen', 'wcsncat', \n",
    "           '_mktemp_s', 'pthread_mutexattr_destroy', '_snwscanf_s', '_strset_s', '_wcsncpy_s_l', 'CC_MD2_Final', \n",
    "           '_mbstok_s_l', 'wctomb_s', 'MySQL_Driver.connect', '_snwscanf_s_l', '*_des_*', 'LoadLibrary', \n",
    "           '_swscanf_s_l', 'ldap_compare_s', 'ldap_compare_ext', '_strlwr_s', 'GetEnvironmentVariable', \n",
    "           'cuserid', '_mbscat_s', 'strspn', '_mbsncpy_s', 'ldap_modrdn2', 'LeaveCriticalSection', 'CopyFile', \n",
    "           'getpwd', 'sscanf', 'creat', 'RegSetValue', 'ldap_modrdn2_s', 'CFile.Close', '*SHA_1*', \n",
    "           'pthread_cond_destroy', 'CC_SHA512_Update', '*RC2*', 'StrNCatA', '_mbsnbcpy', '_mbsnset_s', \n",
    "           'crypt', 'excel', '_vstprintf', 'xstrdup', 'wvsprintfA', 'getopt', 'mkstemp', '_wcsnset_s', \n",
    "           '_stprintf', '_sntprintf', 'tmpfile_s', 'OpenDocumentFile', '_mbsset_s_l', '_strset_s_l', \n",
    "           '_strlwr_s_l', 'ifstream.open', 'xcalloc', 'StrNCpyA', '_wctime_s', 'CC_SHA224_Update', '_ctime64_s', \n",
    "           'MoveFile', 'chown', 'StrNCpyW', 'IsBadReadPtr', '_ui64tow_s', 'IsBadCodePtr', 'getc', \n",
    "           'OracleCommand.ExecuteOracleScalar', 'AccessDataSource.Insert', 'IDbDataAdapter.FillSchema', \n",
    "           'IDbDataAdapter.Update', 'GetWindowText*', 'SendMessage', 'SqlCommand.ExecuteNonQuery', 'streambuf.sgetc', \n",
    "           'streambuf.sgetn', 'OracleCommand.ExecuteScalar', 'SqlDataSource.Update', '_Read_s', 'IDataAdapter.Fill', \n",
    "           '_wgetenv', '_RecordsetPtr.Open*', 'AccessDataSource.Delete', 'Recordset.Open*', 'filebuf.sbumpc', 'DDX_*', \n",
    "           'RegGetValue', 'fstream.read*', 'SqlCeCommand.ExecuteResultSet', 'SqlCommand.ExecuteXmlReader', 'main', \n",
    "           'streambuf.sputbackc', 'read', 'm_lpCmdLine', 'CRichEditCtrl.Get*', 'istream.putback', \n",
    "           'SqlCeCommand.ExecuteXmlReader', 'SqlCeCommand.BeginExecuteXmlReader', 'filebuf.sgetn', \n",
    "           'OdbcDataAdapter.Update', 'filebuf.sgetc', 'SQLPutData', 'recvfrom', 'OleDbDataAdapter.FillSchema', \n",
    "           'IDataAdapter.FillSchema', 'CRichEditCtrl.GetLine', 'DbDataAdapter.Update', 'SqlCommand.ExecuteReader', \n",
    "           'istream.get', 'ReceiveFrom', '_main', 'fgetc', 'DbDataAdapter.FillSchema', 'kbhit', 'UpdateCommand.Execute*', \n",
    "           'Statement.execute', 'fgets', 'SelectCommand.Execute*', 'getch', 'OdbcCommand.ExecuteNonQuery', \n",
    "           'CDaoQueryDef.Execute', 'fstream.getline', 'ifstream.getline', 'SqlDataAdapter.FillSchema', \n",
    "           'OleDbCommand.ExecuteReader', 'Statement.execute*', 'SqlCeCommand.BeginExecuteNonQuery', \n",
    "           'OdbcCommand.ExecuteScalar', 'SqlCeDataAdapter.Update', 'sendmessage', 'mysqlpp.DBDriver', 'fstream.peek', \n",
    "           'Receive', 'CDaoRecordset.Open', 'OdbcDataAdapter.FillSchema', '_wgetenv_s', 'OleDbDataAdapter.Update', \n",
    "           'readsome', 'SqlCommand.BeginExecuteXmlReader', 'recv', 'ifstream.peek', '_Main', '_tmain', '_Readsome_s', \n",
    "           'SqlCeCommand.ExecuteReader', 'OleDbCommand.ExecuteNonQuery', 'fstream.get', 'IDbCommand.ExecuteScalar', \n",
    "           'filebuf.sputbackc', 'IDataAdapter.Update', 'streambuf.sbumpc', 'InsertCommand.Execute*', 'RegQueryValue', \n",
    "           'IDbCommand.ExecuteReader', 'SqlPipe.ExecuteAndSend', 'Connection.Execute*', 'getdlgtext', 'ReceiveFromEx', \n",
    "           'SqlDataAdapter.Update', 'RegQueryValueEx', 'SQLExecute', 'pread', 'SqlCommand.BeginExecuteReader', 'AfxWinMain', \n",
    "           'getchar', 'istream.getline', 'SqlCeDataAdapter.Fill', 'OleDbDataReader.ExecuteReader', 'SqlDataSource.Insert', \n",
    "           'istream.peek', 'SendMessageCallback', 'ifstream.read*', 'SqlDataSource.Select', 'SqlCommand.ExecuteScalar', \n",
    "           'SqlDataAdapter.Fill', 'SqlCommand.BeginExecuteNonQuery', 'getche', 'SqlCeCommand.BeginExecuteReader', 'getenv', \n",
    "           'streambuf.snextc', 'Command.Execute*', '_CommandPtr.Execute*', 'SendNotifyMessage', 'OdbcDataAdapter.Fill', \n",
    "           'AccessDataSource.Update', 'fscanf', 'QSqlQuery.execBatch', 'DbDataAdapter.Fill', 'cin', \n",
    "           'DeleteCommand.Execute*', 'QSqlQuery.exec', 'PostMessage', 'ifstream.get', 'filebuf.snextc', \n",
    "           'IDbCommand.ExecuteNonQuery', 'Winmain', 'fread', 'getpass', 'GetDlgItemTextCCheckListBox.GetCheck', \n",
    "           'DISP_PROPERTY_EX', 'pread64', 'Socket.Receive*', 'SACommand.Execute*', 'SQLExecDirect', \n",
    "           'SqlCeDataAdapter.FillSchema', 'DISP_FUNCTION', 'OracleCommand.ExecuteNonQuery', 'CEdit.GetLine', \n",
    "           'OdbcCommand.ExecuteReader', 'CEdit.Get*', 'AccessDataSource.Select', 'OracleCommand.ExecuteReader', \n",
    "           'OCIStmtExecute', 'getenv_s', 'DB2Command.Execute*', 'OracleDataAdapter.FillSchema', 'OracleDataAdapter.Fill', \n",
    "           'CComboBox.Get*', 'SqlCeCommand.ExecuteNonQuery', 'OracleCommand.ExecuteOracleNonQuery', 'mysqlpp.Query', \n",
    "           'istream.read*', 'CListBox.GetText', 'SqlCeCommand.ExecuteScalar', 'ifstream.putback', 'readlink', \n",
    "           'CHtmlEditCtrl.GetDHtmlDocument', 'PostThreadMessage', 'CListCtrl.GetItemText', 'OracleDataAdapter.Update', \n",
    "           'OleDbCommand.ExecuteScalar', 'stdin', 'SqlDataSource.Delete', 'OleDbDataAdapter.Fill', 'fstream.putback', \n",
    "           'IDbDataAdapter.Fill', '_wspawnl', 'fwprintf', 'sem_wait', '_unlink', 'ldap_search_ext_sW', 'signal', 'PQclear', \n",
    "           'PQfinish', 'PQexec', 'PQresultStatus']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys, argparse\n",
    "from graphviz import Digraph\n",
    "\n",
    "\n",
    "def read_csv(csv_file_path):\n",
    "    data = []\n",
    "    with open(csv_file_path) as fp:\n",
    "        header = fp.readline()\n",
    "        header = header.strip()\n",
    "        h_parts = [hp.strip() for hp in header.split('\\t')]\n",
    "        for line in fp:\n",
    "            line = line.strip()\n",
    "            instance = {}\n",
    "            lparts = line.split('\\t')\n",
    "            for i, hp in enumerate(h_parts):\n",
    "                if i < len(lparts):\n",
    "                    content = lparts[i].strip()\n",
    "                else:\n",
    "                    content = ''\n",
    "                instance[hp] = content\n",
    "            data.append(instance)\n",
    "        return data\n",
    "\n",
    "\n",
    "def read_code_file(file_path):\n",
    "    code_lines = {}\n",
    "    with open(file_path) as fp:\n",
    "        for ln, line in enumerate(fp):\n",
    "            assert isinstance(line, str)\n",
    "            line = line.strip()\n",
    "            if '//' in line:\n",
    "                line = line[:line.index('//')]\n",
    "            code_lines[ln + 1] = line\n",
    "        return code_lines\n",
    "\n",
    "\n",
    "def extract_nodes_with_location_info(nodes):\n",
    "    # Will return an array identifying the indices of those nodes in nodes array,\n",
    "    # another array identifying the node_id of those nodes\n",
    "    # another array indicating the line numbers\n",
    "    # all 3 return arrays should have same length indicating 1-to-1 matching.\n",
    "    node_indices = []\n",
    "    node_ids = []\n",
    "    line_numbers = []\n",
    "    node_id_to_line_number = {}\n",
    "    for node_index, node in enumerate(nodes):\n",
    "        assert isinstance(node, dict)\n",
    "        if 'location' in node.keys():\n",
    "            location = node['location']\n",
    "            if location == '':\n",
    "                continue\n",
    "            line_num = int(location.split(':')[0])\n",
    "            node_id = node['key'].strip()\n",
    "            node_indices.append(node_index)\n",
    "            node_ids.append(node_id)\n",
    "            line_numbers.append(line_num)\n",
    "            node_id_to_line_number[node_id] = line_num\n",
    "    return node_indices, node_ids, line_numbers, node_id_to_line_number\n",
    "    pass\n",
    "\n",
    "\n",
    "def create_adjacency_list(line_numbers, node_id_to_line_numbers, edges, data_dependency_only=False):\n",
    "    adjacency_list = {}\n",
    "    for ln in set(line_numbers):\n",
    "        adjacency_list[ln] = [set(), set()]\n",
    "    for edge in edges:\n",
    "        edge_type = edge['type'].strip()\n",
    "        if True :#edge_type in ['IS_AST_PARENT', 'FLOWS_TO']:\n",
    "            start_node_id = edge['start'].strip()\n",
    "            end_node_id = edge['end'].strip()\n",
    "            if start_node_id not in node_id_to_line_numbers.keys() or end_node_id not in node_id_to_line_numbers.keys():\n",
    "                continue\n",
    "            start_ln = node_id_to_line_numbers[start_node_id]\n",
    "            end_ln = node_id_to_line_numbers[end_node_id]\n",
    "            if not data_dependency_only:\n",
    "                if edge_type == 'CONTROLS': #Control Flow edges\n",
    "                    adjacency_list[start_ln][0].add(end_ln)\n",
    "            if edge_type == 'REACHES': # Data Flow edges\n",
    "                adjacency_list[start_ln][1].add(end_ln)\n",
    "    return adjacency_list\n",
    "\n",
    "\n",
    "def create_visual_graph(code, adjacency_list, file_name='test_graph', verbose=False):\n",
    "    graph = Digraph('Code Property Graph')\n",
    "    for ln in adjacency_list:\n",
    "        graph.node(str(ln), str(ln) + '\\t' + code[ln], shape='box')\n",
    "        control_dependency, data_dependency = adjacency_list[ln]\n",
    "        for anode in control_dependency:\n",
    "            graph.edge(str(ln), str(anode), color='red')\n",
    "        for anode in data_dependency:\n",
    "            graph.edge(str(ln), str(anode), color='blue')\n",
    "    graph.render(file_name, view=verbose)\n",
    "\n",
    "\n",
    "def create_forward_slice(adjacency_list, line_no):\n",
    "    sliced_lines = set()\n",
    "    sliced_lines.add(line_no)\n",
    "    stack = list()\n",
    "    stack.append(line_no)\n",
    "    while len(stack) != 0:\n",
    "        cur = stack.pop()\n",
    "        if cur not in sliced_lines:\n",
    "            sliced_lines.add(cur)\n",
    "        adjacents = adjacency_list[cur]\n",
    "        for node in adjacents:\n",
    "            if node not in sliced_lines:\n",
    "                stack.append(node)\n",
    "    sliced_lines = sorted(sliced_lines)\n",
    "    return sliced_lines\n",
    "\n",
    "\n",
    "def combine_control_and_data_adjacents(adjacency_list):\n",
    "    cgraph = {}\n",
    "    for ln in adjacency_list:\n",
    "        cgraph[ln] = set()\n",
    "        cgraph[ln] = cgraph[ln].union(adjacency_list[ln][0])\n",
    "        cgraph[ln] = cgraph[ln].union(adjacency_list[ln][1])\n",
    "    return cgraph\n",
    "\n",
    "\n",
    "def invert_graph(adjacency_list):\n",
    "    igraph = {}\n",
    "    for ln in adjacency_list.keys():\n",
    "        igraph[ln] = set()\n",
    "    for ln in adjacency_list:\n",
    "        adj = adjacency_list[ln]\n",
    "        for node in adj:\n",
    "            igraph[node].add(ln)\n",
    "    return igraph\n",
    "    pass\n",
    "\n",
    "\n",
    "def create_backward_slice(adjacency_list, line_no):\n",
    "    inverted_adjacency_list = invert_graph(adjacency_list)\n",
    "    return create_forward_slice(inverted_adjacency_list, line_no)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tokenizer:\n",
    "    # creates the object, does the inital parse\n",
    "    def __init__(self, path, tokenizer_type='original'):\n",
    "        self.index = clang.cindex.Index.create()\n",
    "        self.tu = self.index.parse(path)\n",
    "        self.path = self.extract_path(path)\n",
    "        self.symbol_table = {}\n",
    "        self.symbol_count = 1\n",
    "        self.tokenizer_type = tokenizer_type\n",
    "\n",
    "    # To output for split_functions, must have same path up to last two folders\n",
    "    def extract_path(self, path):\n",
    "        return \"\".join(path.split(\"/\")[:-2])\n",
    "\n",
    "    \n",
    "    def full_tokenize_cursor(self, cursor):\n",
    "        tokens = cursor.get_tokens()\n",
    "        result = []\n",
    "        for token in tokens:\n",
    "            if token.kind.name == \"COMMENT\":\n",
    "                continue\n",
    "            if token.kind.name == \"LITERAL\":\n",
    "                result += self.process_literal(token)\n",
    "                continue\n",
    "            if token.kind.name == \"IDENTIFIER\":\n",
    "                result += [\"ID\"]\n",
    "                continue\n",
    "            result += [token.spelling]\n",
    "        return result\n",
    "\n",
    "    def full_tokenize(self):\n",
    "        cursor = self.tu.cursor\n",
    "        return self.full_tokenize_cursor(cursor)\n",
    "\n",
    "    def process_literal(self, literal):\n",
    "        cursor_kind = clang.cindex.CursorKind\n",
    "        kind = literal.cursor.kind\n",
    "        if kind == cursor_kind.INTEGER_LITERAL:\n",
    "            return literal.spelling\n",
    "        if kind == cursor_kind.FLOATING_LITERAL:\n",
    "            return literal.spelling\n",
    "        if kind == cursor_kind.IMAGINARY_LITERAL:\n",
    "            return [\"NUM\"]       \n",
    "        if kind == cursor_kind.STRING_LITERAL:\n",
    "            return [\"STRING\"]\n",
    "        sp = literal.spelling\n",
    "        if re.match('[0-9]+', sp) is not None:\n",
    "            return sp\n",
    "        return [\"LITERAL\"]\n",
    "\n",
    "    def split_functions(self, method_only):\n",
    "        results = []\n",
    "        cursor_kind = clang.cindex.CursorKind\n",
    "        cursor = self.tu.cursor\n",
    "        for c in cursor.get_children():\n",
    "            filename = c.location.file.name if c.location.file != None else \"NONE\"\n",
    "            extracted_path = self.extract_path(filename)\n",
    "\n",
    "            if (c.kind == cursor_kind.CXX_METHOD or (method_only == False and c.kind == cursor_kind.FUNCTION_DECL)) and extracted_path == self.path:\n",
    "                name = c.spelling\n",
    "                tokens = self.full_tokenize_cursor(c)\n",
    "                filename = filename.split(\"/\")[-1]\n",
    "                results += [tokens]\n",
    "\n",
    "        return results\n",
    "    \n",
    "\n",
    "def tokenize(file_text):\n",
    "    try:\n",
    "        c_file = open('/tmp/test1.c', 'w')\n",
    "        c_file.write(file_text)\n",
    "        c_file.close()\n",
    "        tok = Tokenizer('/tmp/test1.c')\n",
    "        results = tok.split_functions(False)\n",
    "        return ' '.join(results[0])\n",
    "    except:\n",
    "        return None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26733\n",
      "0\t1\t1\t1\t1\t5\t5\n",
      "1000\t0\t0\t1\t1\t2\t2\n",
      "2000\t0\t0\t0\t0\t0\t0\n",
      "3000\t0\t0\t0\t0\t4\t4\n",
      "4000\t0\t0\t10\t10\t25\t25\n",
      "5000\t1\t1\t0\t0\t1\t1\n",
      "6000\t0\t0\t1\t1\t0\t0\n",
      "7000\t1\t1\t0\t0\t0\t0\n",
      "8000\t0\t0\t0\t0\t0\t0\n",
      "9000\t0\t0\t0\t0\t0\t0\n",
      "10000\t2\t2\t1\t1\t7\t7\n",
      "11000\t1\t1\t0\t0\t4\t4\n",
      "12000\t0\t0\t0\t0\t0\t0\n",
      "13000\t0\t0\t5\t5\t4\t4\n",
      "14000\t0\t0\t3\t3\t1\t1\n",
      "15000\t2\t2\t1\t1\t2\t2\n",
      "16000\t0\t0\t4\t4\t0\t0\n",
      "17000\t0\t0\t32\t32\t15\t15\n",
      "18000\t2\t2\t6\t6\t2\t2\n",
      "19000\t0\t0\t1\t1\t1\t1\n",
      "20000\t0\t0\t0\t0\t1\t1\n",
      "21000\t1\t1\t1\t1\t0\t0\n",
      "22000\t2\t2\t2\t2\t3\t3\n",
      "23000\t0\t0\t1\t1\t2\t2\n",
      "24000\t0\t0\t0\t0\t0\t0\n",
      "25000\t3\t3\t5\t5\t2\t2\n",
      "26000\t0\t0\t4\t4\t0\t0\n"
     ]
    }
   ],
   "source": [
    "import csv \n",
    "\n",
    "def read_file(path):\n",
    "    with open(path) as f:\n",
    "        lines = f.readlines()\n",
    "        return ' '.join(lines)\n",
    "    \n",
    "def extract_line_number(idx, nodes):\n",
    "    while idx >= 0:\n",
    "        c_node = nodes[idx]\n",
    "        if 'location' in c_node.keys():\n",
    "            location = c_node['location']\n",
    "            if location.strip() != '':\n",
    "                try:\n",
    "                    ln = int(location.split(':')[0])\n",
    "                    return ln\n",
    "                except:\n",
    "                    pass\n",
    "        idx -= 1\n",
    "    return -1\n",
    "\n",
    "all_data = []\n",
    "\n",
    "ggnn_json_data = json.load(open('../data/ggnn_input/devign_cfg_full_text_files.json'))\n",
    "files = [d['file_name'] for d in ggnn_json_data]\n",
    "print(len(files))\n",
    "    \n",
    "for i, file_name  in enumerate(files):\n",
    "    label = file_name.strip()[:-2].split('_')[-1]\n",
    "    code_text = read_file(split_dir + file_name.strip())\n",
    "    \n",
    "    nodes_file_path = parsed + file_name.strip() + '/nodes.csv'\n",
    "    edges_file_path = parsed + file_name.strip() + '/edges.csv'\n",
    "    nc = open(nodes_file_path)\n",
    "    nodes_file = csv.DictReader(nc, delimiter='\\t')\n",
    "    nodes = [node for node in nodes_file]\n",
    "    call_lines = set()\n",
    "    array_lines = set()\n",
    "    ptr_lines = set()\n",
    "    arithmatic_lines = set()\n",
    "    \n",
    "    if len(nodes) == 0:\n",
    "        continue\n",
    "    \n",
    "    for node_idx, node in enumerate(nodes):\n",
    "        ntype = node['type'].strip()\n",
    "        if ntype == 'CallExpression':\n",
    "            function_name = nodes[node_idx + 1]['code']\n",
    "            if function_name  is None or function_name.strip() == '':\n",
    "                continue\n",
    "            if function_name.strip() in l_funcs:\n",
    "                line_no = extract_line_number(node_idx, nodes)\n",
    "                if line_no > 0:\n",
    "                    call_lines.add(line_no)\n",
    "        elif ntype == 'ArrayIndexing':\n",
    "            line_no = extract_line_number(node_idx, nodes)\n",
    "            if line_no > 0:\n",
    "                array_lines.add(line_no)\n",
    "        elif ntype == 'PtrMemberAccess':\n",
    "            line_no = extract_line_number(node_idx, nodes)\n",
    "            if line_no > 0:\n",
    "                ptr_lines.add(line_no)\n",
    "        elif node['operator'].strip() in ['+', '-', '*', '/']:\n",
    "            line_no = extract_line_number(node_idx, nodes)\n",
    "            if line_no > 0:\n",
    "                arithmatic_lines.add(line_no)\n",
    "        \n",
    "    nodes = read_csv(nodes_file_path)\n",
    "    edges = read_csv(edges_file_path)\n",
    "    node_indices, node_ids, line_numbers, node_id_to_ln = extract_nodes_with_location_info(nodes)\n",
    "    adjacency_list = create_adjacency_list(line_numbers, node_id_to_ln, edges, False)\n",
    "    combined_graph = combine_control_and_data_adjacents(adjacency_list)\n",
    "    \n",
    "    array_slices = []\n",
    "    array_slices_bdir = []\n",
    "    call_slices = []\n",
    "    call_slices_bdir = []\n",
    "    arith_slices = []\n",
    "    arith_slices_bdir = []\n",
    "    ptr_slices = []\n",
    "    ptr_slices_bdir = []\n",
    "    all_slices = []\n",
    "    \n",
    "    \n",
    "    all_keys = set()\n",
    "    _keys = set()\n",
    "    for slice_ln in call_lines:\n",
    "        forward_sliced_lines = create_forward_slice(combined_graph, slice_ln)\n",
    "        backward_sliced_lines = create_backward_slice(combined_graph, slice_ln)\n",
    "        all_slice_lines = forward_sliced_lines\n",
    "        all_slice_lines.extend(backward_sliced_lines)\n",
    "        all_slice_lines = sorted(list(set(all_slice_lines)))\n",
    "        key = ' '.join([str(i) for i in all_slice_lines])\n",
    "        if key not in _keys:\n",
    "            call_slices.append(backward_sliced_lines)\n",
    "            call_slices_bdir.append(all_slice_lines)\n",
    "            _keys.add(key)\n",
    "        if key not in all_keys:\n",
    "            all_slices.append(all_slice_lines)\n",
    "            all_keys.add(key)\n",
    "            \n",
    "    _keys = set()\n",
    "    for slice_ln in array_lines:\n",
    "        forward_sliced_lines = create_forward_slice(combined_graph, slice_ln)\n",
    "        backward_sliced_lines = create_backward_slice(combined_graph, slice_ln)\n",
    "        all_slice_lines = forward_sliced_lines\n",
    "        all_slice_lines.extend(backward_sliced_lines)\n",
    "        all_slice_lines = sorted(list(set(all_slice_lines)))\n",
    "        key = ' '.join([str(i) for i in all_slice_lines])\n",
    "        if key not in _keys:\n",
    "            array_slices.append(backward_sliced_lines)\n",
    "            array_slices_bdir.append(all_slice_lines)\n",
    "            _keys.add(key)\n",
    "        if key not in all_keys:\n",
    "            all_slices.append(all_slice_lines)\n",
    "            all_keys.add(key)\n",
    "    \n",
    "    _keys = set()\n",
    "    for slice_ln in arithmatic_lines:\n",
    "        forward_sliced_lines = create_forward_slice(combined_graph, slice_ln)\n",
    "        backward_sliced_lines = create_backward_slice(combined_graph, slice_ln)\n",
    "        all_slice_lines = forward_sliced_lines\n",
    "        all_slice_lines.extend(backward_sliced_lines)\n",
    "        all_slice_lines = sorted(list(set(all_slice_lines)))\n",
    "        key = ' '.join([str(i) for i in all_slice_lines])\n",
    "        if key not in _keys:\n",
    "            arith_slices.append(backward_sliced_lines)\n",
    "            arith_slices_bdir.append(all_slice_lines)\n",
    "            _keys.add(key)\n",
    "        if key not in all_keys:\n",
    "            all_slices.append(all_slice_lines)\n",
    "            all_keys.add(key)\n",
    "    \n",
    "    _keys = set()\n",
    "    for slice_ln in ptr_lines:\n",
    "        forward_sliced_lines = create_forward_slice(combined_graph, slice_ln)\n",
    "        backward_sliced_lines = create_backward_slice(combined_graph, slice_ln)\n",
    "        all_slice_lines = forward_sliced_lines\n",
    "        all_slice_lines.extend(backward_sliced_lines)\n",
    "        all_slice_lines = sorted(list(set(all_slice_lines)))\n",
    "        key = ' '.join([str(i) for i in all_slice_lines])\n",
    "        if key not in _keys:\n",
    "            ptr_slices.append(backward_sliced_lines)\n",
    "            ptr_slices_bdir.append(all_slice_lines)\n",
    "            _keys.add(key)\n",
    "        if key not in all_keys:\n",
    "            all_slices.append(all_slice_lines)\n",
    "            all_keys.add(key)\n",
    "            \n",
    "    t_code = tokenize(code_text)\n",
    "    if t_code is None:\n",
    "        continue\n",
    "    data_instance = {\n",
    "        'file_path': split_dir + file_name.strip(),\n",
    "        'code' : code_text,\n",
    "        'tokenized': t_code,\n",
    "        'call_slices_vd': call_slices,\n",
    "        'call_slices_sy': call_slices_bdir,\n",
    "        'array_slices_vd': array_slices,\n",
    "        'array_slices_sy': array_slices_bdir,\n",
    "        'arith_slices_vd': arith_slices,\n",
    "        'arith_slices_sy': arith_slices_bdir,\n",
    "        'ptr_slices_vd': ptr_slices,\n",
    "        'ptr_slices_sy': ptr_slices_bdir,\n",
    "        'label': int(label)\n",
    "    }\n",
    "    all_data.append(data_instance)\n",
    "    \n",
    "    if i % 1000 == 0:\n",
    "        print(i, len(call_slices), len(call_slices_bdir), \n",
    "              len(array_slices), len(array_slices_bdir), \n",
    "              len(arith_slices), len(arith_slices_bdir), sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_file = open('../data/devign_full_data_with_slices', 'w')\n",
    "json.dump(all_data, output_file)\n",
    "output_file.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25492\n"
     ]
    }
   ],
   "source": [
    "print(len(all_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}