data_processing/full_data_prep_script.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import sys\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dir = '../data/full_experiment_real_data/'\n",
    "project = 'devign'\n",
    "shards = os.listdir(os.path.join(base_dir, project))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_graph_data(\n",
    "    project, portion, base_dir='../data/full_experiment_real_data/', \n",
    "    output_dir='../data/full_experiment_real_data_processed/'):\n",
    "    assert portion in ['full_graph', 'cgraph', 'dgraph', 'cdgraph']\n",
    "    shards = os.listdir(os.path.join(base_dir, project))\n",
    "    shard_count = len(shards)\n",
    "    total_functions, in_scope_function = set(), set()\n",
    "    vnt, nvnt = 0, 0\n",
    "    graphs = []\n",
    "    for sc in range(1, shard_count + 1):\n",
    "        shard_file = open(os.path.join(base_dir, project, project + '.json.shard' + str(sc)))\n",
    "        shard_data = json.load(shard_file)\n",
    "        for data in tqdm(shard_data):\n",
    "            fidx = data['id']\n",
    "            label = int(data['label'])\n",
    "            total_functions.add(fidx)\n",
    "            present = data[portion] is not None\n",
    "            code_graph = data[portion]\n",
    "            if present:\n",
    "                code_graph['id'] = fidx\n",
    "                code_graph['file_name'] = data['file_name']\n",
    "                code_graph['file_path'] = data['file_path']\n",
    "                code_graph['code'] = data['code']\n",
    "                graphs.append(code_graph)\n",
    "                in_scope_function.add(fidx)\n",
    "            else:\n",
    "                if label == 1:\n",
    "                    vnt += 1\n",
    "                else:\n",
    "                    nvnt += 1\n",
    "        shard_file.close()\n",
    "        del shard_data\n",
    "    if not os.path.exists(output_dir):\n",
    "        os.mkdir(output_dir)\n",
    "    output_file = open(os.path.join(output_dir, project + '-' + portion + '.json'), 'w')\n",
    "    json.dump(graphs, output_file)\n",
    "    output_file.close()\n",
    "    print(project, portion, len(total_functions), len(in_scope_function), vnt, nvnt, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e427e4909884465da8712b2e074fa6eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32890cb12381484e9c616eaec68c9f8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b5182d4f7dd466a86c656adfcc1423e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4b65bcb2d8244b5bcfde99240630236",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3169), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "chrome_debian\tfull_graph\t18169\t18169\t0\t0\n"
     ]
    }
   ],
   "source": [
    "extract_graph_data('chrome_debian', 'full_graph')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract_token_data('devign', 'vuld')\n",
    "# extract_token_data('devign', 'syse')\n",
    "# extract_token_data('devign', 'draper')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract_token_data('chrome_debian', 'vuld')\n",
    "# extract_token_data('chrome_debian', 'syse')\n",
    "# extract_token_data('chrome_debian', 'draper')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "keywords = [\"alignas\", \"alignof\", \"and\", \"and_eq\", \"asm\", \"atomic_cancel\", \"atomic_commit\", \n",
    "            \"atomic_noexcept\", \"auto\", \"bitand\", \"bitor\", \"bool\", \"break\", \"case\", \"catch\", \n",
    "            \"char\", \"char8_t\", \"char16_t\", \"char32_t\", \"class\", \"compl\", \"concept\", \"const\", \n",
    "            \"consteval\", \"constexpr\", \"constinit\", \"const_cast\", \"continue\", \"co_await\", \n",
    "            \"co_return\", \"co_yield\", \"decltype\", \"default\", \"delete\", \"do\", \"double\", \"dynamic_cast\", \n",
    "            \"else\", \"enum\", \"explicit\", \"export\", \"extern\", \"false\", \"float\", \"for\", \"friend\", \"goto\", \n",
    "            \"if\", \"inline\", \"int\", \"long\", \"mutable\", \"namespace\", \"new\", \"noexcept\", \"not\", \"not_eq\", \n",
    "            \"nullptr\", \"operator\", \"or\", \"or_eq\", \"private\", \"protected\", \"public\", \"reflexpr\", \n",
    "            \"register\", \"reinterpret_cast\", \"requires\", \"return\", \"short\", \"signed\", \"sizeof\", \"static\", \n",
    "            \"static_assert\", \"static_cast\", \"struct\", \"switch\", \"synchronized\", \"template\", \"this\", \n",
    "            \"thread_local\", \"throw\", \"true\", \"try\", \"typedef\", \"typeid\", \"typename\", \"union\", \"unsigned\", \n",
    "            \"using\", \"virtual\", \"void\", \"volatile\", \"wchar_t\", \"while\", \"xor\", \"xor_eq\", \"NULL\"]\n",
    "puncs = '~`!@#$%^&*()-+={[]}|\\\\;:\\'\\\"<,>.?/'\n",
    "puncs = list(puncs)\n",
    "\n",
    "l_funcs = ['StrNCat', 'getaddrinfo', '_ui64toa', 'fclose', 'pthread_mutex_lock', 'gets_s', 'sleep', \n",
    "           '_ui64tot', 'freopen_s', '_ui64tow', 'send', 'lstrcat', 'HMAC_Update', '__fxstat', 'StrCatBuff', \n",
    "           '_mbscat', '_mbstok_s', '_cprintf_s', 'ldap_search_init_page', 'memmove_s', 'ctime_s', 'vswprintf', \n",
    "           'vswprintf_s', '_snwprintf', '_gmtime_s', '_tccpy', '*RC6*', '_mbslwr_s', 'random', \n",
    "           '__wcstof_internal', '_wcslwr_s', '_ctime32_s', 'wcsncat*', 'MD5_Init', '_ultoa', \n",
    "           'snprintf', 'memset', 'syslog', '_vsnprintf_s', 'HeapAlloc', 'pthread_mutex_destroy', \n",
    "           'ChangeWindowMessageFilter', '_ultot', 'crypt_r', '_strupr_s_l', 'LoadLibraryExA', '_strerror_s', \n",
    "           'LoadLibraryExW', 'wvsprintf', 'MoveFileEx', '_strdate_s', 'SHA1', 'sprintfW', 'StrCatNW', \n",
    "           '_scanf_s_l', 'pthread_attr_init', '_wtmpnam_s', 'snscanf', '_sprintf_s_l', 'dlopen', \n",
    "           'sprintfA', 'timed_mutex', 'OemToCharA', 'ldap_delete_ext', 'sethostid', 'popen', 'OemToCharW', \n",
    "           '_gettws', 'vfork', '_wcsnset_s_l', 'sendmsg', '_mbsncat', 'wvnsprintfA', 'HeapFree', '_wcserror_s', \n",
    "           'realloc', '_snprintf*', 'wcstok', '_strncat*', 'StrNCpy', '_wasctime_s', 'push*', '_lfind_s', \n",
    "           'CC_SHA512', 'ldap_compare_ext_s', 'wcscat_s', 'strdup', '_chsize_s', 'sprintf_s', 'CC_MD4_Init', \n",
    "           'wcsncpy', '_wfreopen_s', '_wcsupr_s', '_searchenv_s', 'ldap_modify_ext_s', '_wsplitpath', \n",
    "           'CC_SHA384_Final', 'MD2', 'RtlCopyMemory', 'lstrcatW', 'MD4', 'MD5', '_wcstok_s_l', '_vsnwprintf_s', \n",
    "           'ldap_modify_s', 'strerror', '_lsearch_s', '_mbsnbcat_s', '_wsplitpath_s', 'MD4_Update', '_mbccpy_s', \n",
    "           '_strncpy_s_l', '_snprintf_s', 'CC_SHA512_Init', 'fwscanf_s', '_snwprintf_s', 'CC_SHA1', 'swprintf', \n",
    "           'fprintf', 'EVP_DigestInit_ex', 'strlen', 'SHA1_Init', 'strncat', '_getws_s', 'CC_MD4_Final', \n",
    "           'wnsprintfW', 'lcong48', 'lrand48', 'write', 'HMAC_Init', '_wfopen_s', 'wmemchr', '_tmakepath', \n",
    "           'wnsprintfA', 'lstrcpynW', 'scanf_s', '_mbsncpy_s_l', '_localtime64_s', 'fstream.open', '_wmakepath', \n",
    "           'Connection.open', '_tccat', 'valloc', 'setgroups', 'unlink', 'fstream.put', 'wsprintfA', '*SHA1*', \n",
    "           '_wsearchenv_s', 'ualstrcpyA', 'CC_MD5_Update', 'strerror_s', 'HeapCreate', 'ualstrcpyW', '__xstat', \n",
    "           '_wmktemp_s', 'StrCatChainW', 'ldap_search_st', '_mbstowcs_s_l', 'ldap_modify_ext', '_mbsset_s', \n",
    "           'strncpy_s', 'move', 'execle', 'StrCat', 'xrealloc', 'wcsncpy_s', '_tcsncpy*', 'execlp', \n",
    "           'RIPEMD160_Final', 'ldap_search_s', 'EnterCriticalSection', '_wctomb_s_l', 'fwrite', '_gmtime64_s', \n",
    "           'sscanf_s', 'wcscat', '_strupr_s', 'wcrtomb_s', 'VirtualLock', 'ldap_add_ext_s', '_mbscpy', \n",
    "           '_localtime32_s', 'lstrcpy', '_wcsncpy*', 'CC_SHA1_Init', '_getts', '_wfopen', '__xstat64', \n",
    "           'strcoll', '_fwscanf_s_l', '_mbslwr_s_l', 'RegOpenKey', 'makepath', 'seed48', 'CC_SHA256', \n",
    "           'sendto', 'execv', 'CalculateDigest', 'memchr', '_mbscpy_s', '_strtime_s', 'ldap_search_ext_s', \n",
    "           '_chmod', 'flock', '__fxstat64', '_vsntprintf', 'CC_SHA256_Init', '_itoa_s', '__wcserror_s', \n",
    "           '_gcvt_s', 'fstream.write', 'sprintf', 'recursive_mutex', 'strrchr', 'gethostbyaddr', '_wcsupr_s_l', \n",
    "           'strcspn', 'MD5_Final', 'asprintf', '_wcstombs_s_l', '_tcstok', 'free', 'MD2_Final', 'asctime_s', \n",
    "           '_alloca', '_wputenv_s', '_wcsset_s', '_wcslwr_s_l', 'SHA1_Update', 'filebuf.sputc', 'filebuf.sputn', \n",
    "           'SQLConnect', 'ldap_compare', 'mbstowcs_s', 'HMAC_Final', 'pthread_condattr_init', '_ultow_s', 'rand', \n",
    "           'ofstream.put', 'CC_SHA224_Final', 'lstrcpynA', 'bcopy', 'system', 'CreateFile*', 'wcscpy_s', \n",
    "           '_mbsnbcpy*', 'open', '_vsnwprintf', 'strncpy', 'getopt_long', 'CC_SHA512_Final', '_vsprintf_s_l', \n",
    "           'scanf', 'mkdir', '_localtime_s', '_snprintf', '_mbccpy_s_l', 'memcmp', 'final', '_ultoa_s', \n",
    "           'lstrcpyW', 'LoadModule', '_swprintf_s_l', 'MD5_Update', '_mbsnset_s_l', '_wstrtime_s', '_strnset_s', \n",
    "           'lstrcpyA', '_mbsnbcpy_s', 'mlock', 'IsBadHugeWritePtr', 'copy', '_mbsnbcpy_s_l', 'wnsprintf', \n",
    "           'wcscpy', 'ShellExecute', 'CC_MD4', '_ultow', '_vsnwprintf_s_l', 'lstrcpyn', 'CC_SHA1_Final', \n",
    "           'vsnprintf', '_mbsnbset_s', '_i64tow', 'SHA256_Init', 'wvnsprintf', 'RegCreateKey', 'strtok_s', \n",
    "           '_wctime32_s', '_i64toa', 'CC_MD5_Final', 'wmemcpy', 'WinExec', 'CreateDirectory*', \n",
    "           'CC_SHA256_Update', '_vsnprintf_s_l', 'jrand48', 'wsprintf', 'ldap_rename_ext_s', 'filebuf.open', \n",
    "           '_wsystem', 'SHA256_Update', '_cwscanf_s', 'wsprintfW', '_sntscanf', '_splitpath', 'fscanf_s', \n",
    "           'strpbrk', 'wcstombs_s', 'wscanf', '_mbsnbcat_s_l', 'strcpynA', 'pthread_cond_init', 'wcsrtombs_s', \n",
    "           '_wsopen_s', 'CharToOemBuffA', 'RIPEMD160_Update', '_tscanf', 'HMAC', 'StrCCpy', 'Connection.connect', \n",
    "           'lstrcatn', '_mbstok', '_mbsncpy', 'CC_SHA384_Update', 'create_directories', 'pthread_mutex_unlock', \n",
    "           'CFile.Open', 'connect', '_vswprintf_s_l', '_snscanf_s_l', 'fputc', '_wscanf_s', '_snprintf_s_l', \n",
    "           'strtok', '_strtok_s_l', 'lstrcatA', 'snwscanf', 'pthread_mutex_init', 'fputs', 'CC_SHA384_Init', \n",
    "           '_putenv_s', 'CharToOemBuffW', 'pthread_mutex_trylock', '__wcstoul_internal', '_memccpy', \n",
    "           '_snwprintf_s_l', '_strncpy*', 'wmemset', 'MD4_Init', '*RC4*', 'strcpyW', '_ecvt_s', 'memcpy_s', \n",
    "           'erand48', 'IsBadHugeReadPtr', 'strcpyA', 'HeapReAlloc', 'memcpy', 'ldap_rename_ext', 'fopen_s', \n",
    "           'srandom', '_cgetws_s', '_makepath', 'SHA256_Final', 'remove', '_mbsupr_s', 'pthread_mutexattr_init', \n",
    "           '__wcstold_internal', 'StrCpy', 'ldap_delete', 'wmemmove_s', '_mkdir', 'strcat', '_cscanf_s_l', \n",
    "           'StrCAdd', 'swprintf_s', '_strnset_s_l', 'close', 'ldap_delete_ext_s', 'ldap_modrdn', 'strchr', \n",
    "           '_gmtime32_s', '_ftcscat', 'lstrcatnA', '_tcsncat', 'OemToChar', 'mutex', 'CharToOem', 'strcpy_s', \n",
    "           'lstrcatnW', '_wscanf_s_l', '__lxstat64', 'memalign', 'MD2_Init', 'StrCatBuffW', 'StrCpyN', 'CC_MD5', \n",
    "           'StrCpyA', 'StrCatBuffA', 'StrCpyW', 'tmpnam_r', '_vsnprintf', 'strcatA', 'StrCpyNW', '_mbsnbset_s_l', \n",
    "           'EVP_DigestInit', '_stscanf', 'CC_MD2', '_tcscat', 'StrCpyNA', 'xmalloc', '_tcslen', '*MD4*', \n",
    "           'vasprintf', 'strxfrm', 'chmod', 'ldap_add_ext', 'alloca', '_snscanf_s', 'IsBadWritePtr', 'swscanf_s', \n",
    "           'wmemcpy_s', '_itoa', '_ui64toa_s', 'EVP_DigestUpdate', '__wcstol_internal', '_itow', 'StrNCatW', \n",
    "           'strncat_s', 'ualstrcpy', 'execvp', '_mbccat', 'EVP_MD_CTX_init', 'assert', 'ofstream.write', \n",
    "           'ldap_add', '_sscanf_s_l', 'drand48', 'CharToOemW', 'swscanf', '_itow_s', 'RIPEMD160_Init', \n",
    "           'CopyMemory', 'initstate', 'getpwuid', 'vsprintf', '_fcvt_s', 'CharToOemA', 'setuid', 'malloc', \n",
    "           'StrCatNA', 'strcat_s', 'srand', 'getwd', '_controlfp_s', 'olestrcpy', '__wcstod_internal', \n",
    "           '_mbsnbcat', 'lstrncat', 'des_*', 'CC_SHA224_Init', 'set*', 'vsprintf_s', 'SHA1_Final', '_umask_s', \n",
    "           'gets', 'setstate', 'wvsprintfW', 'LoadLibraryEx', 'ofstream.open', 'calloc', '_mbstrlen', \n",
    "           '_cgets_s', '_sopen_s', 'IsBadStringPtr', 'wcsncat_s', 'add*', 'nrand48', 'create_directory', \n",
    "           'ldap_search_ext', '_i64toa_s', '_ltoa_s', '_cwscanf_s_l', 'wmemcmp', '__lxstat', 'lstrlen', \n",
    "           'pthread_condattr_destroy', '_ftcscpy', 'wcstok_s', '__xmknod', 'pthread_attr_destroy', 'sethostname', \n",
    "           '_fscanf_s_l', 'StrCatN', 'RegEnumKey', '_tcsncpy', 'strcatW', 'AfxLoadLibrary', 'setenv', 'tmpnam', \n",
    "           '_mbsncat_s_l', '_wstrdate_s', '_wctime64_s', '_i64tow_s', 'CC_MD4_Update', 'ldap_add_s', '_umask', \n",
    "           'CC_SHA1_Update', '_wcsset_s_l', '_mbsupr_s_l', 'strstr', '_tsplitpath', 'memmove', '_tcscpy', \n",
    "           'vsnprintf_s', 'strcmp', 'wvnsprintfW', 'tmpfile', 'ldap_modify', '_mbsncat*', 'mrand48', 'sizeof', \n",
    "           'StrCatA', '_ltow_s', '*desencrypt*', 'StrCatW', '_mbccpy', 'CC_MD2_Init', 'RIPEMD160', 'ldap_search', \n",
    "           'CC_SHA224', 'mbsrtowcs_s', 'update', 'ldap_delete_s', 'getnameinfo', '*RC5*', '_wcsncat_s_l', \n",
    "           'DriverManager.getConnection', 'socket', '_cscanf_s', 'ldap_modrdn_s', '_wopen', 'CC_SHA256_Final', \n",
    "           '_snwprintf*', 'MD2_Update', 'strcpy', '_strncat_s_l', 'CC_MD5_Init', 'mbscpy', 'wmemmove', \n",
    "           'LoadLibraryW', '_mbslen', '*alloc', '_mbsncat_s', 'LoadLibraryA', 'fopen', 'StrLen', 'delete', \n",
    "           '_splitpath_s', 'CreateFileTransacted*', 'MD4_Final', '_open', 'CC_SHA384', 'wcslen', 'wcsncat', \n",
    "           '_mktemp_s', 'pthread_mutexattr_destroy', '_snwscanf_s', '_strset_s', '_wcsncpy_s_l', 'CC_MD2_Final', \n",
    "           '_mbstok_s_l', 'wctomb_s', 'MySQL_Driver.connect', '_snwscanf_s_l', '*_des_*', 'LoadLibrary', \n",
    "           '_swscanf_s_l', 'ldap_compare_s', 'ldap_compare_ext', '_strlwr_s', 'GetEnvironmentVariable', \n",
    "           'cuserid', '_mbscat_s', 'strspn', '_mbsncpy_s', 'ldap_modrdn2', 'LeaveCriticalSection', 'CopyFile', \n",
    "           'getpwd', 'sscanf', 'creat', 'RegSetValue', 'ldap_modrdn2_s', 'CFile.Close', '*SHA_1*', \n",
    "           'pthread_cond_destroy', 'CC_SHA512_Update', '*RC2*', 'StrNCatA', '_mbsnbcpy', '_mbsnset_s', \n",
    "           'crypt', 'excel', '_vstprintf', 'xstrdup', 'wvsprintfA', 'getopt', 'mkstemp', '_wcsnset_s', \n",
    "           '_stprintf', '_sntprintf', 'tmpfile_s', 'OpenDocumentFile', '_mbsset_s_l', '_strset_s_l', \n",
    "           '_strlwr_s_l', 'ifstream.open', 'xcalloc', 'StrNCpyA', '_wctime_s', 'CC_SHA224_Update', '_ctime64_s', \n",
    "           'MoveFile', 'chown', 'StrNCpyW', 'IsBadReadPtr', '_ui64tow_s', 'IsBadCodePtr', 'getc', \n",
    "           'OracleCommand.ExecuteOracleScalar', 'AccessDataSource.Insert', 'IDbDataAdapter.FillSchema', \n",
    "           'IDbDataAdapter.Update', 'GetWindowText*', 'SendMessage', 'SqlCommand.ExecuteNonQuery', 'streambuf.sgetc', \n",
    "           'streambuf.sgetn', 'OracleCommand.ExecuteScalar', 'SqlDataSource.Update', '_Read_s', 'IDataAdapter.Fill', \n",
    "           '_wgetenv', '_RecordsetPtr.Open*', 'AccessDataSource.Delete', 'Recordset.Open*', 'filebuf.sbumpc', 'DDX_*', \n",
    "           'RegGetValue', 'fstream.read*', 'SqlCeCommand.ExecuteResultSet', 'SqlCommand.ExecuteXmlReader', 'main', \n",
    "           'streambuf.sputbackc', 'read', 'm_lpCmdLine', 'CRichEditCtrl.Get*', 'istream.putback', \n",
    "           'SqlCeCommand.ExecuteXmlReader', 'SqlCeCommand.BeginExecuteXmlReader', 'filebuf.sgetn', \n",
    "           'OdbcDataAdapter.Update', 'filebuf.sgetc', 'SQLPutData', 'recvfrom', 'OleDbDataAdapter.FillSchema', \n",
    "           'IDataAdapter.FillSchema', 'CRichEditCtrl.GetLine', 'DbDataAdapter.Update', 'SqlCommand.ExecuteReader', \n",
    "           'istream.get', 'ReceiveFrom', '_main', 'fgetc', 'DbDataAdapter.FillSchema', 'kbhit', 'UpdateCommand.Execute*', \n",
    "           'Statement.execute', 'fgets', 'SelectCommand.Execute*', 'getch', 'OdbcCommand.ExecuteNonQuery', \n",
    "           'CDaoQueryDef.Execute', 'fstream.getline', 'ifstream.getline', 'SqlDataAdapter.FillSchema', \n",
    "           'OleDbCommand.ExecuteReader', 'Statement.execute*', 'SqlCeCommand.BeginExecuteNonQuery', \n",
    "           'OdbcCommand.ExecuteScalar', 'SqlCeDataAdapter.Update', 'sendmessage', 'mysqlpp.DBDriver', 'fstream.peek', \n",
    "           'Receive', 'CDaoRecordset.Open', 'OdbcDataAdapter.FillSchema', '_wgetenv_s', 'OleDbDataAdapter.Update', \n",
    "           'readsome', 'SqlCommand.BeginExecuteXmlReader', 'recv', 'ifstream.peek', '_Main', '_tmain', '_Readsome_s', \n",
    "           'SqlCeCommand.ExecuteReader', 'OleDbCommand.ExecuteNonQuery', 'fstream.get', 'IDbCommand.ExecuteScalar', \n",
    "           'filebuf.sputbackc', 'IDataAdapter.Update', 'streambuf.sbumpc', 'InsertCommand.Execute*', 'RegQueryValue', \n",
    "           'IDbCommand.ExecuteReader', 'SqlPipe.ExecuteAndSend', 'Connection.Execute*', 'getdlgtext', 'ReceiveFromEx', \n",
    "           'SqlDataAdapter.Update', 'RegQueryValueEx', 'SQLExecute', 'pread', 'SqlCommand.BeginExecuteReader', 'AfxWinMain', \n",
    "           'getchar', 'istream.getline', 'SqlCeDataAdapter.Fill', 'OleDbDataReader.ExecuteReader', 'SqlDataSource.Insert', \n",
    "           'istream.peek', 'SendMessageCallback', 'ifstream.read*', 'SqlDataSource.Select', 'SqlCommand.ExecuteScalar', \n",
    "           'SqlDataAdapter.Fill', 'SqlCommand.BeginExecuteNonQuery', 'getche', 'SqlCeCommand.BeginExecuteReader', 'getenv', \n",
    "           'streambuf.snextc', 'Command.Execute*', '_CommandPtr.Execute*', 'SendNotifyMessage', 'OdbcDataAdapter.Fill', \n",
    "           'AccessDataSource.Update', 'fscanf', 'QSqlQuery.execBatch', 'DbDataAdapter.Fill', 'cin', \n",
    "           'DeleteCommand.Execute*', 'QSqlQuery.exec', 'PostMessage', 'ifstream.get', 'filebuf.snextc', \n",
    "           'IDbCommand.ExecuteNonQuery', 'Winmain', 'fread', 'getpass', 'GetDlgItemTextCCheckListBox.GetCheck', \n",
    "           'DISP_PROPERTY_EX', 'pread64', 'Socket.Receive*', 'SACommand.Execute*', 'SQLExecDirect', \n",
    "           'SqlCeDataAdapter.FillSchema', 'DISP_FUNCTION', 'OracleCommand.ExecuteNonQuery', 'CEdit.GetLine', \n",
    "           'OdbcCommand.ExecuteReader', 'CEdit.Get*', 'AccessDataSource.Select', 'OracleCommand.ExecuteReader', \n",
    "           'OCIStmtExecute', 'getenv_s', 'DB2Command.Execute*', 'OracleDataAdapter.FillSchema', 'OracleDataAdapter.Fill', \n",
    "           'CComboBox.Get*', 'SqlCeCommand.ExecuteNonQuery', 'OracleCommand.ExecuteOracleNonQuery', 'mysqlpp.Query', \n",
    "           'istream.read*', 'CListBox.GetText', 'SqlCeCommand.ExecuteScalar', 'ifstream.putback', 'readlink', \n",
    "           'CHtmlEditCtrl.GetDHtmlDocument', 'PostThreadMessage', 'CListCtrl.GetItemText', 'OracleDataAdapter.Update', \n",
    "           'OleDbCommand.ExecuteScalar', 'stdin', 'SqlDataSource.Delete', 'OleDbDataAdapter.Fill', 'fstream.putback', \n",
    "           'IDbDataAdapter.Fill', '_wspawnl', 'fwprintf', 'sem_wait', '_unlink', 'ldap_search_ext_sW', 'signal', 'PQclear', \n",
    "           'PQfinish', 'PQexec', 'PQresultStatus']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import clang.cindex\n",
    "import clang.enumerations\n",
    "import csv\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "\n",
    "import re \n",
    "\n",
    "try:\n",
    "    # set the config\n",
    "    clang.cindex.Config.set_library_path(\"/usr/lib/x86_64-linux-gnu\")\n",
    "    clang.cindex.Config.set_library_file('/usr/lib/x86_64-linux-gnu/libclang-6.0.so.1')\n",
    "except:\n",
    "    pass            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys, argparse\n",
    "from graphviz import Digraph\n",
    "\n",
    "\n",
    "def read_csv(csv_file_path):\n",
    "    data = []\n",
    "    with open(csv_file_path) as fp:\n",
    "        header = fp.readline()\n",
    "        header = header.strip()\n",
    "        h_parts = [hp.strip() for hp in header.split('\\t')]\n",
    "        for line in fp:\n",
    "            line = line.strip()\n",
    "            instance = {}\n",
    "            lparts = line.split('\\t')\n",
    "            for i, hp in enumerate(h_parts):\n",
    "                if i < len(lparts):\n",
    "                    content = lparts[i].strip()\n",
    "                else:\n",
    "                    content = ''\n",
    "                instance[hp] = content\n",
    "            data.append(instance)\n",
    "        return data\n",
    "\n",
    "\n",
    "def read_code_file(file_path):\n",
    "    code_lines = {}\n",
    "    with open(file_path) as fp:\n",
    "        for ln, line in enumerate(fp):\n",
    "            assert isinstance(line, str)\n",
    "            line = line.strip()\n",
    "            if '//' in line:\n",
    "                line = line[:line.index('//')]\n",
    "            code_lines[ln + 1] = line\n",
    "        return code_lines\n",
    "\n",
    "\n",
    "def extract_nodes_with_location_info(nodes):\n",
    "    # Will return an array identifying the indices of those nodes in nodes array,\n",
    "    # another array identifying the node_id of those nodes\n",
    "    # another array indicating the line numbers\n",
    "    # all 3 return arrays should have same length indicating 1-to-1 matching.\n",
    "    node_indices = []\n",
    "    node_ids = []\n",
    "    line_numbers = []\n",
    "    node_id_to_line_number = {}\n",
    "    for node_index, node in enumerate(nodes):\n",
    "        assert isinstance(node, dict)\n",
    "        if 'location' in node.keys():\n",
    "            location = node['location']\n",
    "            if location == '':\n",
    "                continue\n",
    "            line_num = int(location.split(':')[0])\n",
    "            node_id = node['key'].strip()\n",
    "            node_indices.append(node_index)\n",
    "            node_ids.append(node_id)\n",
    "            line_numbers.append(line_num)\n",
    "            node_id_to_line_number[node_id] = line_num\n",
    "    return node_indices, node_ids, line_numbers, node_id_to_line_number\n",
    "    pass\n",
    "\n",
    "\n",
    "def create_adjacency_list(line_numbers, node_id_to_line_numbers, edges, data_dependency_only=False):\n",
    "    adjacency_list = {}\n",
    "    for ln in set(line_numbers):\n",
    "        adjacency_list[ln] = [set(), set()]\n",
    "    for edge in edges:\n",
    "        edge_type = edge['type'].strip()\n",
    "        if True :#edge_type in ['IS_AST_PARENT', 'FLOWS_TO']:\n",
    "            start_node_id = edge['start'].strip()\n",
    "            end_node_id = edge['end'].strip()\n",
    "            if start_node_id not in node_id_to_line_numbers.keys() or end_node_id not in node_id_to_line_numbers.keys():\n",
    "                continue\n",
    "            start_ln = node_id_to_line_numbers[start_node_id]\n",
    "            end_ln = node_id_to_line_numbers[end_node_id]\n",
    "            if not data_dependency_only:\n",
    "                if edge_type == 'CONTROLS': #Control Flow edges\n",
    "                    adjacency_list[start_ln][0].add(end_ln)\n",
    "            if edge_type == 'REACHES': # Data Flow edges\n",
    "                adjacency_list[start_ln][1].add(end_ln)\n",
    "    return adjacency_list\n",
    "\n",
    "\n",
    "def create_visual_graph(code, adjacency_list, file_name='test_graph', verbose=False):\n",
    "    graph = Digraph('Code Property Graph')\n",
    "    for ln in adjacency_list:\n",
    "        graph.node(str(ln), str(ln) + '\\t' + code[ln], shape='box')\n",
    "        control_dependency, data_dependency = adjacency_list[ln]\n",
    "        for anode in control_dependency:\n",
    "            graph.edge(str(ln), str(anode), color='red')\n",
    "        for anode in data_dependency:\n",
    "            graph.edge(str(ln), str(anode), color='blue')\n",
    "    graph.render(file_name, view=verbose)\n",
    "\n",
    "\n",
    "def create_forward_slice(adjacency_list, line_no):\n",
    "    sliced_lines = set()\n",
    "    sliced_lines.add(line_no)\n",
    "    stack = list()\n",
    "    stack.append(line_no)\n",
    "    while len(stack) != 0:\n",
    "        cur = stack.pop()\n",
    "        if cur not in sliced_lines:\n",
    "            sliced_lines.add(cur)\n",
    "        adjacents = adjacency_list[cur]\n",
    "        for node in adjacents:\n",
    "            if node not in sliced_lines:\n",
    "                stack.append(node)\n",
    "    sliced_lines = sorted(sliced_lines)\n",
    "    return sliced_lines\n",
    "\n",
    "\n",
    "def combine_control_and_data_adjacents(adjacency_list):\n",
    "    cgraph = {}\n",
    "    for ln in adjacency_list:\n",
    "        cgraph[ln] = set()\n",
    "        cgraph[ln] = cgraph[ln].union(adjacency_list[ln][0])\n",
    "        cgraph[ln] = cgraph[ln].union(adjacency_list[ln][1])\n",
    "    return cgraph\n",
    "\n",
    "\n",
    "def invert_graph(adjacency_list):\n",
    "    igraph = {}\n",
    "    for ln in adjacency_list.keys():\n",
    "        igraph[ln] = set()\n",
    "    for ln in adjacency_list:\n",
    "        adj = adjacency_list[ln]\n",
    "        for node in adj:\n",
    "            igraph[node].add(ln)\n",
    "    return igraph\n",
    "    pass\n",
    "\n",
    "\n",
    "def create_backward_slice(adjacency_list, line_no):\n",
    "    inverted_adjacency_list = invert_graph(adjacency_list)\n",
    "    return create_forward_slice(inverted_adjacency_list, line_no)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tokenizer:\n",
    "    # creates the object, does the inital parse\n",
    "    def __init__(self, path, tokenizer_type='original'):\n",
    "        self.index = clang.cindex.Index.create()\n",
    "        self.tu = self.index.parse(path)\n",
    "        self.path = self.extract_path(path)\n",
    "        self.symbol_table = {}\n",
    "        self.symbol_count = 1\n",
    "        self.tokenizer_type = tokenizer_type\n",
    "\n",
    "    # To output for split_functions, must have same path up to last two folders\n",
    "    def extract_path(self, path):\n",
    "        return \"\".join(path.split(\"/\")[:-2])\n",
    "\n",
    "    \n",
    "    def full_tokenize_cursor(self, cursor):\n",
    "        tokens = cursor.get_tokens()\n",
    "        result = []\n",
    "        for token in tokens:\n",
    "            if token.kind.name == \"COMMENT\":\n",
    "                continue\n",
    "            if token.kind.name == \"LITERAL\":\n",
    "                result += self.process_literal(token)\n",
    "                continue\n",
    "            if token.kind.name == \"IDENTIFIER\":\n",
    "                result += [\"ID\"]\n",
    "                continue\n",
    "            result += [token.spelling]\n",
    "        return result\n",
    "\n",
    "    def full_tokenize(self):\n",
    "        cursor = self.tu.cursor\n",
    "        return self.full_tokenize_cursor(cursor)\n",
    "\n",
    "    def process_literal(self, literal):\n",
    "        cursor_kind = clang.cindex.CursorKind\n",
    "        kind = literal.cursor.kind\n",
    "        if kind == cursor_kind.INTEGER_LITERAL:\n",
    "            return literal.spelling\n",
    "        if kind == cursor_kind.FLOATING_LITERAL:\n",
    "            return literal.spelling\n",
    "        if kind == cursor_kind.IMAGINARY_LITERAL:\n",
    "            return [\"NUM\"]       \n",
    "        if kind == cursor_kind.STRING_LITERAL:\n",
    "            return [\"STRING\"]\n",
    "        sp = literal.spelling\n",
    "        if re.match('[0-9]+', sp) is not None:\n",
    "            return sp\n",
    "        return [\"LITERAL\"]\n",
    "\n",
    "    def split_functions(self, method_only):\n",
    "        results = []\n",
    "        cursor_kind = clang.cindex.CursorKind\n",
    "        cursor = self.tu.cursor\n",
    "        for c in cursor.get_children():\n",
    "            filename = c.location.file.name if c.location.file != None else \"NONE\"\n",
    "            extracted_path = self.extract_path(filename)\n",
    "\n",
    "            if (c.kind == cursor_kind.CXX_METHOD or (method_only == False and c.kind == cursor_kind.FUNCTION_DECL)) and extracted_path == self.path:\n",
    "                name = c.spelling\n",
    "                tokens = self.full_tokenize_cursor(c)\n",
    "                filename = filename.split(\"/\")[-1]\n",
    "                results += [tokens]\n",
    "\n",
    "        return results\n",
    "    \n",
    "\n",
    "def tokenize(file_text):\n",
    "    try:\n",
    "        c_file = open('/tmp/test1.c', 'w')\n",
    "        c_file.write(file_text)\n",
    "        c_file.close()\n",
    "        tok = Tokenizer('/tmp/test1.c')\n",
    "        results = tok.split_functions(False)\n",
    "        return ' '.join(results[0])\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def symbolic_tokenize(code):\n",
    "    tokens = nltk.word_tokenize(code)\n",
    "    c_tokens = []\n",
    "    for t in tokens:\n",
    "        if t.strip() != '':\n",
    "            c_tokens.append(t.strip())\n",
    "    f_count = 1\n",
    "    var_count = 1\n",
    "    symbol_table = {}\n",
    "    final_tokens = []\n",
    "    for idx in range(len(c_tokens)):\n",
    "        t = c_tokens[idx]\n",
    "        if t in keywords:\n",
    "            final_tokens.append(t)\n",
    "        elif t in puncs:\n",
    "            final_tokens.append(t)\n",
    "        elif t in l_funcs:\n",
    "            final_tokens.append(t)\n",
    "        elif idx < len(c_tokens) - 1 and c_tokens[idx + 1] == '(':\n",
    "            if t in keywords:\n",
    "                final_tokens.append(t)\n",
    "            else:\n",
    "                if t not in symbol_table.keys():\n",
    "                    symbol_table[t] = \"FUNC\" + str(f_count)\n",
    "                    f_count += 1\n",
    "                final_tokens.append(symbol_table[t])\n",
    "            idx += 1\n",
    "        \n",
    "        elif t.endswith('('):\n",
    "            t = t[:-1]\n",
    "            if t in keywords:\n",
    "                final_tokens.append(t + '(')\n",
    "            else:\n",
    "                if t not in symbol_table.keys():\n",
    "                    symbol_table[t] = \"FUNC\" + str(f_count)\n",
    "                    f_count += 1\n",
    "                final_tokens.append(symbol_table[t] + '(')\n",
    "        elif t.endswith('()'):\n",
    "            t = t[:-2]\n",
    "            if t in keywords:\n",
    "                final_tokens.append(t + '()')\n",
    "            else:\n",
    "                if t not in symbol_table.keys():\n",
    "                    symbol_table[t] = \"FUNC\" + str(f_count)\n",
    "                    f_count += 1\n",
    "                final_tokens.append(symbol_table[t] + '()')\n",
    "        elif re.match(\"^\\\"*\\\"$\", t) is not None:\n",
    "            final_tokens.append(\"STRING\")\n",
    "        elif re.match(\"^[0-9]+(\\.[0-9]+)?$\", t) is not None:\n",
    "            final_tokens.append(\"NUMBER\")\n",
    "        elif re.match(\"^[0-9]*(\\.[0-9]+)$\", t) is not None:\n",
    "            final_tokens.append(\"NUMBER\")\n",
    "        else:\n",
    "            if t not in symbol_table.keys():\n",
    "                symbol_table[t] = \"VAR\" + str(var_count)\n",
    "                var_count += 1\n",
    "            final_tokens.append(symbol_table[t])\n",
    "    return ' '.join(final_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv \n",
    "\n",
    "def read_file(path):\n",
    "    with open(path) as f:\n",
    "        lines = f.readlines()\n",
    "        return ' '.join(lines)\n",
    "    \n",
    "def extract_line_number(idx, nodes):\n",
    "    while idx >= 0:\n",
    "        c_node = nodes[idx]\n",
    "        if 'location' in c_node.keys():\n",
    "            location = c_node['location']\n",
    "            if location.strip() != '':\n",
    "                try:\n",
    "                    ln = int(location.split(':')[0])\n",
    "                    return ln\n",
    "                except:\n",
    "                    pass\n",
    "        idx -= 1\n",
    "    return -1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = []\n",
    "import nltk\n",
    "from gensim.models import Word2Vec\n",
    "\n",
    "def reformat_code_line_graph(code_lines, adjacency_lists, lanel, wv_model_original, wv_model_li, label):\n",
    "    actual_lines = []\n",
    "    for ln in adjacency_lists.keys():\n",
    "        cd, dd = adjacency_lists[ln]\n",
    "        new_cd = [l for l in cd]\n",
    "        new_dd = [l for l in dd] \n",
    "        actual_lines.extend(new_cd)\n",
    "        actual_lines.extend(new_dd)\n",
    "        actual_lines.append(ln)\n",
    "    actual_lines = sorted(list(set(actual_lines)))\n",
    "    line_no_to_idx = {}\n",
    "    idx_to_line_no = {}\n",
    "    for idx, ln in enumerate(actual_lines):\n",
    "        line_no_to_idx[ln] = idx\n",
    "        idx_to_line_no[idx] = ln\n",
    "    data_point = {}\n",
    "    graph = []\n",
    "    for src in adjacency_lists.keys():\n",
    "        cd, dd = adjacency_lists[src]\n",
    "        for dest in cd:\n",
    "            graph.append([line_no_to_idx[src], 0, line_no_to_idx[dest]])\n",
    "            graph.append([line_no_to_idx[dest], 1, line_no_to_idx[src]])\n",
    "        for dest in dd:\n",
    "            graph.append([line_no_to_idx[src], 2, line_no_to_idx[dest]])\n",
    "            graph.append([line_no_to_idx[dest], 3, line_no_to_idx[src]])\n",
    "    original_tokens = []\n",
    "    symbolic_tokens = []\n",
    "    line_features_wv = []\n",
    "    sym_line_features_wv = []\n",
    "    \n",
    "    for lidx in range(len(idx_to_line_no.keys())):\n",
    "        actual_code_line = code_lines[idx_to_line_no[lidx]]\n",
    "        actual_line_tokens = nltk.wordpunct_tokenize(actual_code_line)\n",
    "        symbolic_line_tokens = symbolic_tokenize(actual_code_line).split()\n",
    "        original_tokens.append(actual_line_tokens)\n",
    "        symbolic_tokens.append(symbolic_line_tokens)\n",
    "        \n",
    "        nrp = np.zeros(100)\n",
    "        for token in actual_line_tokens:\n",
    "            try:\n",
    "                embedding = wv_model_original.wv[token]\n",
    "            except:\n",
    "                embedding = np.zeros(100)\n",
    "            nrp = np.add(nrp, embedding)\n",
    "        if len(actual_line_tokens) > 0:\n",
    "            fNrp = np.divide(nrp, len(actual_line_tokens))\n",
    "        else:\n",
    "            fNrp = nrp\n",
    "        line_features_wv.append(fNrp.tolist())\n",
    "        \n",
    "        nrp = np.zeros(64)\n",
    "        for token in symbolic_line_tokens:\n",
    "            try:\n",
    "                embedding = wv_model_li.wv[token]\n",
    "            except:\n",
    "                embedding = np.zeros(64)\n",
    "            nrp = np.add(nrp, embedding)\n",
    "        if len(actual_line_tokens) > 0:\n",
    "            fNrp = np.divide(nrp, len(symbolic_line_tokens))\n",
    "        else:\n",
    "            fNrp = nrp\n",
    "        sym_line_features_wv.append(fNrp.tolist())\n",
    "    data_point = {\n",
    "        'node_features': line_features_wv,\n",
    "        'node_features_sym': sym_line_features_wv,\n",
    "        'graph': graph,\n",
    "        'original_tokens': original_tokens,\n",
    "        'symbolic_tokens': symbolic_tokens,\n",
    "        'targets': [[label]]\n",
    "    }\n",
    "    return data_point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import json\n",
    "\n",
    "wv_model_li = Word2Vec.load('../data/Word2Vec/li_et_al_wv')\n",
    "                                    \n",
    "ggnn_data = []\n",
    "\n",
    "def extract_line_graph_data(\n",
    "    project, base_dir='../data/full_experiment_real_data/', \n",
    "    output_dir='../data/full_experiment_real_data_processed/'):\n",
    "    if project == 'devign':\n",
    "        split_dir = '../data/neurips_parsed/neurips_data/'\n",
    "        parsed = '../data/neurips_parsed/parsed_results/'\n",
    "        wv_path = '../data/neurips_parsed/raw_code_neurips.100'\n",
    "        wv_model_original = Word2Vec.load(wv_path)\n",
    "    else:\n",
    "        split_dir = '../data/chrome_debian/raw_code/'\n",
    "        parsed = '../data/chrome_debian/parsed/'\n",
    "        wv_path = '../data/chrome_debian/raw_code_deb_chro.100'\n",
    "        wv_model_original = Word2Vec.load(wv_path)\n",
    "    shards = os.listdir(os.path.join(base_dir, project))\n",
    "    shard_count = len(shards)\n",
    "    total_functions, in_scope_function = set(), set()\n",
    "    vnt, nvnt = 0, 0\n",
    "    graphs = []\n",
    "    for sc in range(1, shard_count + 1):\n",
    "        shard_file = open(os.path.join(base_dir, project, project + '.json.shard' + str(sc)))\n",
    "        shard_data = json.load(shard_file)\n",
    "        try:\n",
    "            for data in tqdm(shard_data):\n",
    "                file_name = data['file_name']    \n",
    "                label = int(file_name.strip()[:-2].split('_')[-1])\n",
    "                code_text = read_code_file(split_dir + file_name.strip())\n",
    "                nodes_file_path = parsed + file_name.strip() + '/nodes.csv'\n",
    "                edges_file_path = parsed + file_name.strip() + '/edges.csv'\n",
    "                nc = open(nodes_file_path)\n",
    "                nodes_file = csv.DictReader(nc, delimiter='\\t')\n",
    "                nodes = [node for node in nodes_file]\n",
    "                if len(nodes) == 0:\n",
    "                    continue\n",
    "                nodes = read_csv(nodes_file_path)\n",
    "                edges = read_csv(edges_file_path)\n",
    "                node_indices, node_ids, line_numbers, node_id_to_ln = extract_nodes_with_location_info(nodes)\n",
    "                adjacency_list = create_adjacency_list(line_numbers, node_id_to_ln, edges, False)\n",
    "                combined_graph = combine_control_and_data_adjacents(adjacency_list)\n",
    "                data_point = reformat_code_line_graph(code_text, adjacency_list, label, wv_model_original, wv_model_li, label)\n",
    "                graphs.append(data_point)\n",
    "        finally:\n",
    "            pass\n",
    "        del shard_data\n",
    "    output_file = open(os.path.join(output_dir, project + '-line-ggnn.json'), 'w')\n",
    "    json.dump(graphs, output_file)\n",
    "    output_file.close()\n",
    "    print(len(graphs))\n",
    "#     return graphs\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [01:33<00:00, 53.26it/s] \n",
      "100%|██████████| 5000/5000 [01:18<00:00, 63.85it/s] \n",
      "100%|██████████| 5000/5000 [01:12<00:00, 69.13it/s] \n",
      "100%|██████████| 5000/5000 [01:13<00:00, 68.23it/s] \n",
      "100%|██████████| 5000/5000 [01:16<00:00, 65.06it/s] \n",
      "100%|██████████| 492/492 [00:07<00:00, 68.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25492\n"
     ]
    }
   ],
   "source": [
    "graph_data = extract_line_graph_data(\"devign\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [01:24<00:00, 59.31it/s] \n",
      "100%|██████████| 5000/5000 [01:10<00:00, 71.43it/s] \n",
      "100%|██████████| 5000/5000 [01:12<00:00, 68.78it/s] \n",
      "100%|██████████| 3169/3169 [00:41<00:00, 76.54it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18169\n"
     ]
    }
   ],
   "source": [
    "graph_data = extract_line_graph_data(\"chrome_debian\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}