diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 77790511..7a115172 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: runs-on: ${{ matrix.os }}-latest strategy: matrix: - os: [Ubuntu, MacOS] + os: [Ubuntu, MacOS, Windows] python-version: [3.7, 3.8] steps: - uses: actions/checkout@v2 diff --git a/.travis.yml b/.travis.yml index f3b56b6b..0e45f1c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,5 +39,6 @@ script: - pipenv run flake8 src - pipenv run flake8 tests - mdl README.md + - mdl docs after_success: - pipenv run codecov diff --git a/Pipfile b/Pipfile index b6031dff..4b732bbe 100644 --- a/Pipfile +++ b/Pipfile @@ -24,6 +24,7 @@ textblob = "*" scipy = "*" pylint = "*" importlib-metadata = "*" +atomicwrites = "*" [pipenv] allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock index f123ae81..9e16af1e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "789c09afdf33e00d8e16fe6d623c8f52136034d7234f35dd57aa58f92a2e4694" + "sha256": "6b3bc1c36fb65fe6bdb73ed8dec80bcae9330690e3c1033ffeeb05209f5ce6dc" }, "pipfile-spec": 6, "requires": {}, @@ -22,14 +22,6 @@ "index": "pypi", "version": "==4.1.0" }, - "appnope": { - "hashes": [ - "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442", - "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a" - ], - "markers": "sys_platform == 'darwin' and platform_system == 'Darwin'", - "version": "==0.1.2" - }, "argon2-cffi": { "hashes": [ "sha256:05a8ac07c7026542377e38389638a8a1e9b78f1cd8439cd7493b39f08dd75fbf", @@ -63,11 +55,11 @@ }, "astroid": { "hashes": [ - "sha256:6b0ed1af831570e500e2437625979eaa3b36011f66ddfc4ce930128610258ca9", - "sha256:cd80bf957c49765dce6d92c43163ff9d2abc43132ce64d4b1b47717c6d2522df" + "sha256:ad63b8552c70939568966811a088ef0bc880f99a24a00834abd0e3681b514f91", + "sha256:bea3f32799fbb8581f58431c12591bc20ce11cbc90ad82e2ea5717d94f2080d5" ], "markers": "python_version >= '3.6'", - "version": "==2.5.2" + "version": "==2.5.3" }, "async-generator": { "hashes": [ @@ -77,6 +69,14 @@ "markers": "python_version >= '3.5'", "version": "==1.10" }, + "atomicwrites": { + "hashes": [ + "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197", + "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a" + ], + "index": "pypi", + "version": "==1.4.0" + }, "attrs": { "hashes": [ "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", @@ -222,6 +222,14 @@ "index": "pypi", "version": "==2.1.11" }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "sys_platform == 'win32' and sys_platform == 'win32' and sys_platform == 'win32'", + "version": "==0.4.4" + }, "commonmark": { "hashes": [ "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60", @@ -285,7 +293,7 @@ "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d", "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==5.5" }, "cycler": { @@ -315,10 +323,11 @@ }, "decorator": { "hashes": [ - "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", - "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7" + "sha256:d9f2d2863183a3c0df05f4b786f2e6b8752c093b3547a558f287bf3022fd2bf4", + "sha256:f2e71efb39412bfd23d878e896a51b07744f2e2250b2e87d158e76828c5ae202" ], - "version": "==4.4.2" + "markers": "python_version >= '3.5'", + "version": "==5.0.6" }, "defusedxml": { "hashes": [ @@ -411,11 +420,11 @@ }, "ipykernel": { "hashes": [ - "sha256:98321abefdf0505fb3dc7601f60fc4087364d394bd8fad53107eb1adee9ff475", - "sha256:efd07253b54d84d26e0878d268c8c3a41582a18750da633c2febfd2ece0d467d" + "sha256:21abd584543759e49010975a4621603b3cf871b1039cb3879a14094717692614", + "sha256:a682e4f7affd86d9ce9b699d21bcab6d5ec9fbb2bfcb194f2706973b252bc509" ], "markers": "python_version >= '3.4'", - "version": "==5.5.0" + "version": "==5.5.3" }, "ipython": { "hashes": [ @@ -444,7 +453,7 @@ "sha256:0a943902919f65c5684ac4e0154b1ad4fac6dcaa5d9f3426b732f1c8b5419be6", "sha256:2bb1680aad211e3c9944dbce1d4ba09a989f04e238296c87fe2139faa26d655d" ], - "markers": "python_version >= '3.6' and python_version < '4.0'", + "markers": "python_version >= '3.6' and python_version < '4'", "version": "==5.8.0" }, "jedi": { @@ -635,28 +644,28 @@ }, "matplotlib": { "hashes": [ - "sha256:133469eb5d54a0a963d08a07356d03f6de41b6789f2a9d15aed0e6f3aabbf502", - "sha256:2b54bbce73115f6f95912f57348b2e0fc03cc86a09c73191f1216d79c16cda3c", - "sha256:36e2f68b7d2f22eacc4d9d4dfdf3a975c9db31fa6e36e16a2d60bf4c1ff5a35c", - "sha256:3847ce29a01e1d1f0aa691263e510ce8504a20d158a811ed3a45c4c1d9e708e7", - "sha256:3bcee5685afb76d2a31f8a6b190caf2e52be250d214155b91830110416767de3", - "sha256:3c617fc6f7d6ac0c6142e34f3066520273195c75fb5371eabfddb3da04a75174", - "sha256:4002b0928bcd695eb7f23bf18c52e3cf23501a9c06a1de596079d74932aa4068", - "sha256:424ddb3422c65b284a38a97eb48f5cb64b66a44a773e0c71281a347f1738f146", - "sha256:7571d470cd5df2d6c27ee0e6dabbebd130eae91da19c3749c6946918f8bcad49", - "sha256:7e76956b8dd4bd8a6be71e401e4a24d975e57d5849a80c24a35551040ff82df0", - "sha256:85388289b356f5f19464b84eaa7532cd10d3f701505ae8d988b3dfb1332d785f", - "sha256:85d7b8da66bb71054f3d73210b4fe24cca8afc6f92f00569b2ad24ca32438034", - "sha256:8b046715f710b58dc89241eb3c8e907a30b269e915df6a134bb419f05a6507bf", - "sha256:966a1bb53ce56dcd542b4edb667166aac14c8eaf2a3acd68f403eccedf0d21a7", - "sha256:a2828982c5329cceea67bae0af49e2bfae3471499d6d6aa4359efa0ef491b0f7", - "sha256:a960ee5a139011e1d203df75eafc4e98013927f36ab69510313bc7d9d4641e38", - "sha256:e879b5915f0c8e765d0e1ae4ecf93fafbde4a1dee88409427ac0ca55d31a41a9", - "sha256:fa0f7f3b2a0067ee5806264478f39fa52d7ddf8ee9b3012e5257b2ff39a15e0b", - "sha256:fe4676d7f25d2915562aa551996843e97e7d25aefc54617a3b0fe74a967bc2c8" + "sha256:1f83a32e4b6045191f9d34e4dc68c0a17c870b57ef9cca518e516da591246e79", + "sha256:2eee37340ca1b353e0a43a33da79d0cd4bcb087064a0c3c3d1329cdea8fbc6f3", + "sha256:53ceb12ef44f8982b45adc7a0889a7e2df1d758e8b360f460e435abe8a8cd658", + "sha256:574306171b84cd6854c83dc87bc353cacc0f60184149fb00c9ea871eca8c1ecb", + "sha256:7561fd541477d41f3aa09457c434dd1f7604f3bd26d7858d52018f5dfe1c06d1", + "sha256:7a54efd6fcad9cb3cd5ef2064b5a3eeb0b63c99f26c346bdcf66e7c98294d7cc", + "sha256:7f16660edf9a8bcc0f766f51c9e1b9d2dc6ceff6bf636d2dbd8eb925d5832dfd", + "sha256:81e6fe8b18ef5be67f40a1d4f07d5a4ed21d3878530193898449ddef7793952f", + "sha256:84a10e462120aa7d9eb6186b50917ed5a6286ee61157bfc17c5b47987d1a9068", + "sha256:84d4c4f650f356678a5d658a43ca21a41fca13f9b8b00169c0b76e6a6a948908", + "sha256:86dc94e44403fa0f2b1dd76c9794d66a34e821361962fe7c4e078746362e3b14", + "sha256:90dbc007f6389bcfd9ef4fe5d4c78c8d2efe4e0ebefd48b4f221cdfed5672be2", + "sha256:9f374961a3996c2d1b41ba3145462c3708a89759e604112073ed6c8bdf9f622f", + "sha256:a18cc1ab4a35b845cf33b7880c979f5c609fd26c2d6e74ddfacb73dcc60dd956", + "sha256:a97781453ac79409ddf455fccf344860719d95142f9c334f2a8f3fff049ffec3", + "sha256:a989022f89cda417f82dbf65e0a830832afd8af743d05d1414fb49549287ff04", + "sha256:ac2a30a09984c2719f112a574b6543ccb82d020fd1b23b4d55bf4759ba8dd8f5", + "sha256:be4430b33b25e127fc4ea239cc386389de420be4d63e71d5359c20b562951ce1", + "sha256:c45e7bf89ea33a2adaef34774df4e692c7436a18a48bcb0e47a53e698a39fa39" ], "index": "pypi", - "version": "==3.4.0" + "version": "==3.4.1" }, "mccabe": { "hashes": [ @@ -712,11 +721,11 @@ }, "nbformat": { "hashes": [ - "sha256:1d223e64a18bfa7cdf2db2e9ba8a818312fc2a0701d2e910b58df66809385a56", - "sha256:3949fdc8f5fa0b1afca16fb307546e78494fa7a7bceff880df8168eafda0e7ac" + "sha256:b516788ad70771c6250977c1374fcca6edebe6126fd2adb5a69aa5c2356fd1c8", + "sha256:eb8447edd7127d043361bc17f2f5a807626bc8e878c7709a1c647abda28a9171" ], "markers": "python_version >= '3.5'", - "version": "==5.1.2" + "version": "==5.1.3" }, "nest-asyncio": { "hashes": [ @@ -728,9 +737,11 @@ }, "nltk": { "hashes": [ - "sha256:845365449cd8c5f9731f7cb9f8bd6fd0767553b9d53af9eb1b3abf7700936b35" + "sha256:1235660f52ab10fda34d5277096724747f767b2903e1c0c4e14bde013552c9ba", + "sha256:cbc2ed576998fcf7cd181eeb3ca029e5f0025b264074b4beb57ce780673f8b86" ], - "version": "==3.5" + "markers": "python_version >= '3.5'", + "version": "==3.6.1" }, "notebook": { "hashes": [ @@ -808,19 +819,11 @@ }, "parso": { "hashes": [ - "sha256:15b00182f472319383252c18d5913b69269590616c947747bc50bf4ac768f410", - "sha256:8519430ad07087d4c997fda3a7918f7cfa27cb58972a8c89c2a0295a1c940e9e" + "sha256:12b83492c6239ce32ff5eed6d3639d6a536170723c6f3f1506869f1ace413398", + "sha256:a8c4922db71e4fdb90e0d0bc6e50f9b273d3397925e5e60a717e719201778d22" ], "markers": "python_version >= '3.6'", - "version": "==0.8.1" - }, - "pexpect": { - "hashes": [ - "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937", - "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c" - ], - "markers": "sys_platform != 'win32'", - "version": "==4.8.0" + "version": "==0.8.2" }, "pickleshare": { "hashes": [ @@ -831,42 +834,42 @@ }, "pillow": { "hashes": [ - "sha256:15306d71a1e96d7e271fd2a0737038b5a92ca2978d2e38b6ced7966583e3d5af", - "sha256:1940fc4d361f9cc7e558d6f56ff38d7351b53052fd7911f4b60cd7bc091ea3b1", - "sha256:1f93f2fe211f1ef75e6f589327f4d4f8545d5c8e826231b042b483d8383e8a7c", - "sha256:30d33a1a6400132e6f521640dd3f64578ac9bfb79a619416d7e8802b4ce1dd55", - "sha256:328240f7dddf77783e72d5ed79899a6b48bc6681f8d1f6001f55933cb4905060", - "sha256:46c2bcf8e1e75d154e78417b3e3c64e96def738c2a25435e74909e127a8cba5e", - "sha256:5762ebb4436f46b566fc6351d67a9b5386b5e5de4e58fdaa18a1c83e0e20f1a8", - "sha256:5a2d957eb4aba9d48170b8fe6538ec1fbc2119ffe6373782c03d8acad3323f2e", - "sha256:5cf03b9534aca63b192856aa601c68d0764810857786ea5da652581f3a44c2b0", - "sha256:5daba2b40782c1c5157a788ec4454067c6616f5a0c1b70e26ac326a880c2d328", - "sha256:63cd413ac52ee3f67057223d363f4f82ce966e64906aea046daf46695e3c8238", - "sha256:6efac40344d8f668b6c4533ae02a48d52fd852ef0654cc6f19f6ac146399c733", - "sha256:71b01ee69e7df527439d7752a2ce8fb89e19a32df484a308eca3e81f673d3a03", - "sha256:71f31ee4df3d5e0b366dd362007740106d3210fb6a56ec4b581a5324ba254f06", - "sha256:72027ebf682abc9bafd93b43edc44279f641e8996fb2945104471419113cfc71", - "sha256:74cd9aa648ed6dd25e572453eb09b08817a1e3d9f8d1bd4d8403d99e42ea790b", - "sha256:81b3716cc9744ffdf76b39afb6247eae754186838cedad0b0ac63b2571253fe6", - "sha256:8565355a29655b28fdc2c666fd9a3890fe5edc6639d128814fafecfae2d70910", - "sha256:87f42c976f91ca2fc21a3293e25bd3cd895918597db1b95b93cbd949f7d019ce", - "sha256:89e4c757a91b8c55d97c91fa09c69b3677c227b942fa749e9a66eef602f59c28", - "sha256:8c4e32218c764bc27fe49b7328195579581aa419920edcc321c4cb877c65258d", - "sha256:903293320efe2466c1ab3509a33d6b866dc850cfd0c5d9cc92632014cec185fb", - "sha256:90882c6f084ef68b71bba190209a734bf90abb82ab5e8f64444c71d5974008c6", - "sha256:98afcac3205d31ab6a10c5006b0cf040d0026a68ec051edd3517b776c1d78b09", - "sha256:a01da2c266d9868c4f91a9c6faf47a251f23b9a862dce81d2ff583135206f5be", - "sha256:aeab4cd016e11e7aa5cfc49dcff8e51561fa64818a0be86efa82c7038e9369d0", - "sha256:b07c660e014852d98a00a91adfbe25033898a9d90a8f39beb2437d22a203fc44", - "sha256:bead24c0ae3f1f6afcb915a057943ccf65fc755d11a1410a909c1fefb6c06ad1", - "sha256:d1d6bca39bb6dd94fba23cdb3eeaea5e30c7717c5343004d900e2a63b132c341", - "sha256:e2cd8ac157c1e5ae88b6dd790648ee5d2777e76f1e5c7d184eaddb2938594f34", - "sha256:e5739ae63636a52b706a0facec77b2b58e485637e1638202556156e424a02dc2", - "sha256:f36c3ff63d6fc509ce599a2f5b0d0732189eed653420e7294c039d342c6e204a", - "sha256:f91b50ad88048d795c0ad004abbe1390aa1882073b1dca10bfd55d0b8cf18ec5" + "sha256:01425106e4e8cee195a411f729cff2a7d61813b0b11737c12bd5991f5f14bcd5", + "sha256:031a6c88c77d08aab84fecc05c3cde8414cd6f8406f4d2b16fed1e97634cc8a4", + "sha256:083781abd261bdabf090ad07bb69f8f5599943ddb539d64497ed021b2a67e5a9", + "sha256:0d19d70ee7c2ba97631bae1e7d4725cdb2ecf238178096e8c82ee481e189168a", + "sha256:0e04d61f0064b545b989126197930807c86bcbd4534d39168f4aa5fda39bb8f9", + "sha256:12e5e7471f9b637762453da74e390e56cc43e486a88289995c1f4c1dc0bfe727", + "sha256:22fd0f42ad15dfdde6c581347eaa4adb9a6fc4b865f90b23378aa7914895e120", + "sha256:238c197fc275b475e87c1453b05b467d2d02c2915fdfdd4af126145ff2e4610c", + "sha256:3b570f84a6161cf8865c4e08adf629441f56e32f180f7aa4ccbd2e0a5a02cba2", + "sha256:463822e2f0d81459e113372a168f2ff59723e78528f91f0bd25680ac185cf797", + "sha256:4d98abdd6b1e3bf1a1cbb14c3895226816e666749ac040c4e2554231068c639b", + "sha256:5afe6b237a0b81bd54b53f835a153770802f164c5570bab5e005aad693dab87f", + "sha256:5b70110acb39f3aff6b74cf09bb4169b167e2660dabc304c1e25b6555fa781ef", + "sha256:5cbf3e3b1014dddc45496e8cf38b9f099c95a326275885199f427825c6522232", + "sha256:624b977355cde8b065f6d51b98497d6cd5fbdd4f36405f7a8790e3376125e2bb", + "sha256:63728564c1410d99e6d1ae8e3b810fe012bc440952168af0a2877e8ff5ab96b9", + "sha256:66cc56579fd91f517290ab02c51e3a80f581aba45fd924fcdee01fa06e635812", + "sha256:6c32cc3145928c4305d142ebec682419a6c0a8ce9e33db900027ddca1ec39178", + "sha256:8bb1e155a74e1bfbacd84555ea62fa21c58e0b4e7e6b20e4447b8d07990ac78b", + "sha256:95d5ef984eff897850f3a83883363da64aae1000e79cb3c321915468e8c6add5", + "sha256:a013cbe25d20c2e0c4e85a9daf438f85121a4d0344ddc76e33fd7e3965d9af4b", + "sha256:a787ab10d7bb5494e5f76536ac460741788f1fbce851068d73a87ca7c35fc3e1", + "sha256:a7d5e9fad90eff8f6f6106d3b98b553a88b6f976e51fce287192a5d2d5363713", + "sha256:aac00e4bc94d1b7813fe882c28990c1bc2f9d0e1aa765a5f2b516e8a6a16a9e4", + "sha256:b91c36492a4bbb1ee855b7d16fe51379e5f96b85692dc8210831fbb24c43e484", + "sha256:c03c07ed32c5324939b19e36ae5f75c660c81461e312a41aea30acdd46f93a7c", + "sha256:c5236606e8570542ed424849f7852a0ff0bce2c4c8d0ba05cc202a5a9c97dee9", + "sha256:c6b39294464b03457f9064e98c124e09008b35a62e3189d3513e5148611c9388", + "sha256:cb7a09e173903541fa888ba010c345893cd9fc1b5891aaf060f6ca77b6a3722d", + "sha256:d68cb92c408261f806b15923834203f024110a2e2872ecb0bd2a110f89d3c602", + "sha256:dc38f57d8f20f06dd7c3161c59ca2c86893632623f33a42d592f097b00f720a9", + "sha256:e98eca29a05913e82177b3ba3d198b1728e164869c613d76d0de4bde6768a50e", + "sha256:f217c3954ce5fd88303fc0c317af55d5e0204106d86dea17eb8205700d47dec2" ], "markers": "python_version >= '3.6'", - "version": "==8.1.2" + "version": "==8.2.0" }, "plac": { "hashes": [ @@ -907,10 +910,11 @@ }, "prometheus-client": { "hashes": [ - "sha256:9da7b32f02439d8c04f7777021c304ed51d9ec180604700c1ba72a4d44dceb03", - "sha256:b08c34c328e1bf5961f0b4352668e6c8f145b4a087e09b7296ef62cbe4693d35" + "sha256:030e4f9df5f53db2292eec37c6255957eb76168c6f974e4176c711cf91ed34aa", + "sha256:b6c5a9643e3545bcbfd9451766cbaa5d9c67e7303c7bc32c750b6fa70ecb107d" ], - "version": "==0.9.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.1" }, "prompt-toolkit": { "hashes": [ @@ -922,36 +926,28 @@ }, "protobuf": { "hashes": [ - "sha256:0f2da2fcc4102b6c3b57f03c9d8d5e37c63f8bc74deaa6cb54e0cc4524a77247", - "sha256:1655fc0ba7402560d749de13edbfca1ac45d1753d8f4e5292989f18f5a00c215", - "sha256:1771ef20e88759c4d81db213e89b7a1fc53937968e12af6603c658ee4bcbfa38", - "sha256:1a66261a402d05c8ad8c1fde8631837307bf8d7e7740a4f3941fc3277c2e1528", - "sha256:24f4697f57b8520c897a401b7f9a5ae45c369e22c572e305dfaf8053ecb49687", - "sha256:256c0b2e338c1f3228d3280707606fe5531fde85ab9d704cde6fdeb55112531f", - "sha256:2b974519a2ae83aa1e31cff9018c70bbe0e303a46a598f982943c49ae1d4fcd3", - "sha256:30fe4249a364576f9594180589c3f9c4771952014b5f77f0372923fc7bafbbe2", - "sha256:45a91fc6f9aa86d3effdeda6751882b02de628519ba06d7160daffde0c889ff8", - "sha256:70054ae1ce5dea7dec7357db931fcf487f40ea45b02cb719ee6af07eb1e906fb", - "sha256:74ac159989e2b02d761188a2b6f4601ff5e494d9b9d863f5ad6e98e5e0c54328", - "sha256:822ac7f87fc2fb9b24edd2db390538b60ef50256e421ca30d65250fad5a3d477", - "sha256:83c7c7534f050cb25383bb817159416601d1cc46c40bc5e851ec8bbddfc34a2f", - "sha256:88d8f21d1ac205eedb6dea943f8204ed08201b081dba2a966ab5612788b9bb1e", - "sha256:9ec20a6ded7d0888e767ad029dbb126e604e18db744ac0a428cf746e040ccecd", - "sha256:9ec220d90eda8bb7a7a1434a8aed4fe26d7e648c1a051c2885f3f5725b6aa71a", - "sha256:b9069e45b6e78412fba4a314ea38b4a478686060acf470d2b131b3a2c50484ec", - "sha256:d9ed0955b794f1e5f367e27f8a8ff25501eabe34573f003f06639c366ca75f73", - "sha256:eaada29bbf087dea7d8bce4d1d604fc768749e8809e9c295922accd7c8fce4d5", - "sha256:eac23a3e56175b710f3da9a9e8e2aa571891fbec60e0c5a06db1c7b1613b5cfd" - ], - "version": "==3.15.6" - }, - "ptyprocess": { - "hashes": [ - "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", - "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220" - ], - "markers": "os_name != 'nt'", - "version": "==0.7.0" + "sha256:0277f62b1e42210cafe79a71628c1d553348da81cbd553402a7f7549c50b11d0", + "sha256:07eec4e2ccbc74e95bb9b3afe7da67957947ee95bdac2b2e91b038b832dd71f0", + "sha256:1c0e9e56202b9dccbc094353285a252e2b7940b74fdf75f1b4e1b137833fabd7", + "sha256:1f0b5d156c3df08cc54bc2c8b8b875648ea4cd7ebb2a9a130669f7547ec3488c", + "sha256:2dc0e8a9e4962207bdc46a365b63a3f1aca6f9681a5082a326c5837ef8f4b745", + "sha256:3053f13207e7f13dc7be5e9071b59b02020172f09f648e85dc77e3fcb50d1044", + "sha256:4a054b0b5900b7ea7014099e783fb8c4618e4209fffcd6050857517b3f156e18", + "sha256:510e66491f1a5ac5953c908aa8300ec47f793130097e4557482803b187a8ee05", + "sha256:5ff9fa0e67fcab442af9bc8d4ec3f82cb2ff3be0af62dba047ed4187f0088b7d", + "sha256:90270fe5732c1f1ff664a3bd7123a16456d69b4e66a09a139a00443a32f210b8", + "sha256:a0a08c6b2e6d6c74a6eb5bf6184968eefb1569279e78714e239d33126e753403", + "sha256:c5566f956a26cda3abdfacc0ca2e21db6c9f3d18f47d8d4751f2209d6c1a5297", + "sha256:dab75b56a12b1ceb3e40808b5bd9dfdaef3a1330251956e6744e5b6ed8f8830b", + "sha256:efa4c4d4fc9ba734e5e85eaced70e1b63fb3c8d08482d839eb838566346f1737", + "sha256:f17b352d7ce33c81773cf81d536ca70849de6f73c96413f17309f4b43ae7040b", + "sha256:f42c2f5fb67da5905bfc03733a311f72fa309252bcd77c32d1462a1ad519521e", + "sha256:f6077db37bfa16494dca58a4a02bfdacd87662247ad6bc1f7f8d13ff3f0013e1", + "sha256:f80afc0a0ba13339bbab25ca0409e9e2836b12bb012364c06e97c2df250c3343", + "sha256:f9cadaaa4065d5dd4d15245c3b68b967b3652a3108e77f292b58b8c35114b56c", + "sha256:fad4f971ec38d8df7f4b632c819bf9bbf4f57cfd7312cf526c69ce17ef32436a" + ], + "version": "==3.15.8" }, "py": { "hashes": [ @@ -961,33 +957,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.10.0" }, - "pyarrow": { - "hashes": [ - "sha256:03e2435da817bc2b5d0fad6f2e53305eb36c24004ddfcb2b30e4217a1a80cf22", - "sha256:2be3a9eab4bfd00024dc3c83fa03de1c1d04a0f47ebaf3dc483cd100546eacbf", - "sha256:2c3353d38d137f1158595b3b18dcef711f3d8fdb57cf7ae2d861d07235064bc1", - "sha256:2d5c95eb04a3d2e786e097b53534893eade6c8b3faf10f53a06143384b4446b1", - "sha256:31e6fc0868963aba4e6b8a3e218c9a5ff347bca870d622da0b3d58269d0c5398", - "sha256:3b46487c45faaea8d1a5aa65002e2832ae2e1c9e68ecb461cda4fa59891cf490", - "sha256:3ea6574d1ae2d9bff7e6e1715f64c31bdc01b42387a5c78311a8ce9c09cfe135", - "sha256:4bf8cc43e1db1e0517466209ee8e8f459d9b5e1b4074863317f2a965cf59889e", - "sha256:5faa2dc73444bdcf042f121383965a47362be1f946303d46e8fd80f8d26cd90c", - "sha256:72206cde1857d5420601feae75f53921cffab4326b42262a858c7b8be67982b7", - "sha256:960a9b0fd599601ddac42f16d5acf049637ec08957359c6741d6eb2bf0dbae97", - "sha256:978bbe8ec9090d1133a25f00f32ed92600f9d315fbfa29a17952bee01f0d7fe5", - "sha256:a07e286e81ceb20f8f0c45f69760d2ebc434fe83794d5f9b44f89fc2dc6dc24d", - "sha256:a76031ef19d11db2fef79a97cc69997c97bea35aa07efbe042a177c7e3b1a390", - "sha256:b08c119cc2b9fcd1567797fedb245a2f4352a3084a22b7298272afe7cf7a4730", - "sha256:b1cf92df9f336f31706249e543dc0ffce3c67a78204ce540f1173c6c07dfafec", - "sha256:b7a8903f2b8a80498725ef5d4a35cd7dd5a98b74e080d42692545e61a6cbfbe4", - "sha256:bf6684fe9e38f8ddb696e38901461eab783ec1d565974ebd5862270320b3e27f", - "sha256:cfea99a01d844c3db5e25374a6cdcf3b5ba1698bfe95d41272c295a4581e884c", - "sha256:d5666a7fa2668f3ff95df028c2072d59e8b17e73d682068e8505dafa2688f3cc", - "sha256:dec007a0f7adba86bd170252140ede01646b45c3a470d5862ce00d8e40cd29bd" - ], - "markers": "python_version < '3.9'", - "version": "==3.0.0" - }, "pycparser": { "hashes": [ "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", @@ -1013,11 +982,11 @@ }, "pylint": { "hashes": [ - "sha256:466e067526b609c1f22b97895e2a64e5fc3d897016b91ef0f4f80480e89a1ef9", - "sha256:d29e1140d96362eb0b68fa0fb1a6192532f586886d6c52db79872a906a50be58" + "sha256:209d712ec870a0182df034ae19f347e725c1e615b2269519ab58a35b3fcbbe7a", + "sha256:bd38914c7731cdc518634a8d3c5585951302b6e2b6de60fbb3f7a0220e21eeee" ], "index": "pypi", - "version": "==2.7.3" + "version": "==2.7.4" }, "pyparsing": { "hashes": [ @@ -1036,11 +1005,11 @@ }, "pytest": { "hashes": [ - "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9", - "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839" + "sha256:671238a46e4df0f3498d1c3270e5deb9b32d25134c99b7d75370a68cfbe9b634", + "sha256:6ad9c7bdf517a808242b998ac20063c41532a570d088d77eec1ee12b0b5574bc" ], "index": "pypi", - "version": "==6.2.2" + "version": "==6.2.3" }, "python-dateutil": { "hashes": [ @@ -1057,6 +1026,38 @@ ], "version": "==2021.1" }, + "pywin32": { + "hashes": [ + "sha256:1c204a81daed2089e55d11eefa4826c05e604d27fe2be40b6bf8db7b6a39da63", + "sha256:27a30b887afbf05a9cbb05e3ffd43104a9b71ce292f64a635389dbad0ed1cd85", + "sha256:350c5644775736351b77ba68da09a39c760d75d2467ecec37bd3c36a94fbed64", + "sha256:60a8fa361091b2eea27f15718f8eb7f9297e8d51b54dbc4f55f3d238093d5190", + "sha256:638b68eea5cfc8def537e43e9554747f8dee786b090e47ead94bfdafdb0f2f50", + "sha256:8151e4d7a19262d6694162d6da85d99a16f8b908949797fd99c83a0bfaf5807d", + "sha256:a3b4c48c852d4107e8a8ec980b76c94ce596ea66d60f7a697582ea9dce7e0db7", + "sha256:b1609ce9bd5c411b81f941b246d683d6508992093203d4eb7f278f4ed1085c3f", + "sha256:d7e8c7efc221f10d6400c19c32a031add1c4a58733298c09216f57b4fde110dc", + "sha256:fbb3b1b0fbd0b4fc2a3d1d81fe0783e30062c1abed1d17c32b7879d55858cfae" + ], + "markers": "sys_platform == 'win32'", + "version": "==300" + }, + "pywinpty": { + "hashes": [ + "sha256:1e525a4de05e72016a7af27836d512db67d06a015aeaf2fa0180f8e6a039b3c2", + "sha256:2740eeeb59297593a0d3f762269b01d0285c1b829d6827445fcd348fb47f7e70", + "sha256:2d7e9c881638a72ffdca3f5417dd1563b60f603e1b43e5895674c2a1b01f95a0", + "sha256:33df97f79843b2b8b8bc5c7aaf54adec08cc1bae94ee99dfb1a93c7a67704d95", + "sha256:5fb2c6c6819491b216f78acc2c521b9df21e0f53b9a399d58a5c151a3c4e2a2d", + "sha256:8fc5019ff3efb4f13708bd3b5ad327589c1a554cb516d792527361525a7cb78c", + "sha256:b358cb552c0f6baf790de375fab96524a0498c9df83489b8c23f7f08795e966b", + "sha256:dbd838de92de1d4ebf0dce9d4d5e4fc38d0b7b1de837947a18b57a882f219139", + "sha256:dd22c8efacf600730abe4a46c1388355ce0d4ab75dc79b15d23a7bd87bf05b48", + "sha256:e854211df55d107f0edfda8a80b39dfc87015bef52a8fe6594eb379240d81df2" + ], + "markers": "os_name == 'nt'", + "version": "==0.5.7" + }, "pyzmq": { "hashes": [ "sha256:13465c1ff969cab328bc92f7015ce3843f6e35f8871ad79d236e4fbc85dbe4cb", @@ -1097,49 +1098,49 @@ }, "regex": { "hashes": [ - "sha256:07ef35301b4484bce843831e7039a84e19d8d33b3f8b2f9aab86c376813d0139", - "sha256:13f50969028e81765ed2a1c5fcfdc246c245cf8d47986d5172e82ab1a0c42ee5", - "sha256:14de88eda0976020528efc92d0a1f8830e2fb0de2ae6005a6fc4e062553031fa", - "sha256:159fac1a4731409c830d32913f13f68346d6b8e39650ed5d704a9ce2f9ef9cb3", - "sha256:18e25e0afe1cf0f62781a150c1454b2113785401ba285c745acf10c8ca8917df", - "sha256:201e2619a77b21a7780580ab7b5ce43835e242d3e20fef50f66a8df0542e437f", - "sha256:360a01b5fa2ad35b3113ae0c07fb544ad180603fa3b1f074f52d98c1096fa15e", - "sha256:39c44532d0e4f1639a89e52355b949573e1e2c5116106a395642cbbae0ff9bcd", - "sha256:3d9356add82cff75413bec360c1eca3e58db4a9f5dafa1f19650958a81e3249d", - "sha256:3d9a7e215e02bd7646a91fb8bcba30bc55fd42a719d6b35cf80e5bae31d9134e", - "sha256:4651f839dbde0816798e698626af6a2469eee6d9964824bb5386091255a1694f", - "sha256:486a5f8e11e1f5bbfcad87f7c7745eb14796642323e7e1829a331f87a713daaa", - "sha256:4b8a1fb724904139149a43e172850f35aa6ea97fb0545244dc0b805e0154ed68", - "sha256:4c0788010a93ace8a174d73e7c6c9d3e6e3b7ad99a453c8ee8c975ddd9965643", - "sha256:4c2e364491406b7888c2ad4428245fc56c327e34a5dfe58fd40df272b3c3dab3", - "sha256:575a832e09d237ae5fedb825a7a5bc6a116090dd57d6417d4f3b75121c73e3be", - "sha256:5770a51180d85ea468234bc7987f5597803a4c3d7463e7323322fe4a1b181578", - "sha256:633497504e2a485a70a3268d4fc403fe3063a50a50eed1039083e9471ad0101c", - "sha256:63f3ca8451e5ff7133ffbec9eda641aeab2001be1a01878990f6c87e3c44b9d5", - "sha256:709f65bb2fa9825f09892617d01246002097f8f9b6dde8d1bb4083cf554701ba", - "sha256:808404898e9a765e4058bf3d7607d0629000e0a14a6782ccbb089296b76fa8fe", - "sha256:882f53afe31ef0425b405a3f601c0009b44206ea7f55ee1c606aad3cc213a52c", - "sha256:8bd4f91f3fb1c9b1380d6894bd5b4a519409135bec14c0c80151e58394a4e88a", - "sha256:8e65e3e4c6feadf6770e2ad89ad3deb524bcb03d8dc679f381d0568c024e0deb", - "sha256:976a54d44fd043d958a69b18705a910a8376196c6b6ee5f2596ffc11bff4420d", - "sha256:a0d04128e005142260de3733591ddf476e4902c0c23c1af237d9acf3c96e1b38", - "sha256:a0df9a0ad2aad49ea3c7f65edd2ffb3d5c59589b85992a6006354f6fb109bb18", - "sha256:a2ee026f4156789df8644d23ef423e6194fad0bc53575534101bb1de5d67e8ce", - "sha256:a59a2ee329b3de764b21495d78c92ab00b4ea79acef0f7ae8c1067f773570afa", - "sha256:b97ec5d299c10d96617cc851b2e0f81ba5d9d6248413cd374ef7f3a8871ee4a6", - "sha256:b98bc9db003f1079caf07b610377ed1ac2e2c11acc2bea4892e28cc5b509d8d5", - "sha256:b9d8d286c53fe0cbc6d20bf3d583cabcd1499d89034524e3b94c93a5ab85ca90", - "sha256:bcd945175c29a672f13fce13a11893556cd440e37c1b643d6eeab1988c8b209c", - "sha256:c66221e947d7207457f8b6f42b12f613b09efa9669f65a587a2a71f6a0e4d106", - "sha256:c782da0e45aff131f0bed6e66fbcfa589ff2862fc719b83a88640daa01a5aff7", - "sha256:cb4ee827857a5ad9b8ae34d3c8cc51151cb4a3fe082c12ec20ec73e63cc7c6f0", - "sha256:d47d359545b0ccad29d572ecd52c9da945de7cd6cf9c0cfcb0269f76d3555689", - "sha256:dc9963aacb7da5177e40874585d7407c0f93fb9d7518ec58b86e562f633f36cd", - "sha256:ea2f41445852c660ba7c3ebf7d70b3779b20d9ca8ba54485a17740db49f46932", - "sha256:f5d0c921c99297354cecc5a416ee4280bd3f20fd81b9fb671ca6be71499c3fdf", - "sha256:f85d6f41e34f6a2d1607e312820971872944f1661a73d33e1e82d35ea3305e14" - ], - "version": "==2021.3.17" + "sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5", + "sha256:11d773d75fa650cd36f68d7ca936e3c7afaae41b863b8c387a22aaa78d3c5c79", + "sha256:18c071c3eb09c30a264879f0d310d37fe5d3a3111662438889ae2eb6fc570c31", + "sha256:1e1c20e29358165242928c2de1482fb2cf4ea54a6a6dea2bd7a0e0d8ee321500", + "sha256:281d2fd05555079448537fe108d79eb031b403dac622621c78944c235f3fcf11", + "sha256:314d66636c494ed9c148a42731b3834496cc9a2c4251b1661e40936814542b14", + "sha256:32e65442138b7b76dd8173ffa2cf67356b7bc1768851dded39a7a13bf9223da3", + "sha256:339456e7d8c06dd36a22e451d58ef72cef293112b559010db3d054d5560ef439", + "sha256:3916d08be28a1149fb97f7728fca1f7c15d309a9f9682d89d79db75d5e52091c", + "sha256:3a9cd17e6e5c7eb328517969e0cb0c3d31fd329298dd0c04af99ebf42e904f82", + "sha256:47bf5bf60cf04d72bf6055ae5927a0bd9016096bf3d742fa50d9bf9f45aa0711", + "sha256:4c46e22a0933dd783467cf32b3516299fb98cfebd895817d685130cc50cd1093", + "sha256:4c557a7b470908b1712fe27fb1ef20772b78079808c87d20a90d051660b1d69a", + "sha256:52ba3d3f9b942c49d7e4bc105bb28551c44065f139a65062ab7912bef10c9afb", + "sha256:563085e55b0d4fb8f746f6a335893bda5c2cef43b2f0258fe1020ab1dd874df8", + "sha256:598585c9f0af8374c28edd609eb291b5726d7cbce16be6a8b95aa074d252ee17", + "sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000", + "sha256:67bdb9702427ceddc6ef3dc382455e90f785af4c13d495f9626861763ee13f9d", + "sha256:6d1b01031dedf2503631d0903cb563743f397ccaf6607a5e3b19a3d76fc10480", + "sha256:741a9647fcf2e45f3a1cf0e24f5e17febf3efe8d4ba1281dcc3aa0459ef424dc", + "sha256:7c2a1af393fcc09e898beba5dd59196edaa3116191cc7257f9224beaed3e1aa0", + "sha256:7d9884d86dd4dd489e981d94a65cd30d6f07203d90e98f6f657f05170f6324c9", + "sha256:90f11ff637fe8798933fb29f5ae1148c978cccb0452005bf4c69e13db951e765", + "sha256:919859aa909429fb5aa9cf8807f6045592c85ef56fdd30a9a3747e513db2536e", + "sha256:96fcd1888ab4d03adfc9303a7b3c0bd78c5412b2bfbe76db5b56d9eae004907a", + "sha256:97f29f57d5b84e73fbaf99ab3e26134e6687348e95ef6b48cfd2c06807005a07", + "sha256:980d7be47c84979d9136328d882f67ec5e50008681d94ecc8afa8a65ed1f4a6f", + "sha256:a91aa8619b23b79bcbeb37abe286f2f408d2f2d6f29a17237afda55bb54e7aac", + "sha256:ade17eb5d643b7fead300a1641e9f45401c98eee23763e9ed66a43f92f20b4a7", + "sha256:b9c3db21af35e3b3c05764461b262d6f05bbca08a71a7849fd79d47ba7bc33ed", + "sha256:bd28bc2e3a772acbb07787c6308e00d9626ff89e3bfcdebe87fa5afbfdedf968", + "sha256:bf5824bfac591ddb2c1f0a5f4ab72da28994548c708d2191e3b87dd207eb3ad7", + "sha256:c0502c0fadef0d23b128605d69b58edb2c681c25d44574fc673b0e52dce71ee2", + "sha256:c38c71df845e2aabb7fb0b920d11a1b5ac8526005e533a8920aea97efb8ec6a4", + "sha256:ce15b6d103daff8e9fee13cf7f0add05245a05d866e73926c358e871221eae87", + "sha256:d3029c340cfbb3ac0a71798100ccc13b97dddf373a4ae56b6a72cf70dfd53bc8", + "sha256:e512d8ef5ad7b898cdb2d8ee1cb09a8339e4f8be706d27eaa180c2f177248a10", + "sha256:e8e5b509d5c2ff12f8418006d5a90e9436766133b564db0abaec92fd27fcee29", + "sha256:ee54ff27bf0afaf4c3b3a62bcd016c12c3fdb4ec4f413391a90bd38bc3624605", + "sha256:fa4537fb4a98fe8fde99626e4681cc644bdcf2a795038533f9f711513a862ae6", + "sha256:fd45ff9293d9274c5008a2054ecef86a9bfe819a67c7be1afb65e69b405b3042" + ], + "version": "==2021.4.4" }, "requests": { "hashes": [ @@ -1237,10 +1238,11 @@ }, "smart-open": { "hashes": [ - "sha256:d9f5a0f173ccb9bbae528db5a3804f57145815774f77ef755b9b0f3b4b2a9dcb" + "sha256:02e5e02207d955a1f9fd924c37900ac272536bd24393ae731b6945d4f766015f", + "sha256:ed310ac51a797051b42f9437a566ac8149a3abf8c80ab994a8bd92d96fb7cf44" ], "markers": "python_version >= '3.6'", - "version": "==4.2.0" + "version": "==5.0.0" }, "smmap": { "hashes": [ @@ -1289,11 +1291,11 @@ }, "streamlit": { "hashes": [ - "sha256:35c2c901b5fee15174a0cc03e465e9b13a10f8f411d925c2ca9dd012bdc8ab20", - "sha256:8532aad5a05af08dd48e955fa64bc9fe9e3fb99a4c95514215a597ba56e05121" + "sha256:bfe5532d28596e6801de7c5a9ccbbafc55121dca30cd37bd08d3348fda084123", + "sha256:f74b8cfda27a2201a704b54bae39f25cc6c751c442aa5d4700440b1f1e42c71d" ], "index": "pypi", - "version": "==0.79.0" + "version": "==0.78.0" }, "terminado": { "hashes": [ @@ -1409,11 +1411,11 @@ }, "tqdm": { "hashes": [ - "sha256:9fdf349068d047d4cfbe24862c425883af1db29bcddf4b0eeb2524f6fbdb23c7", - "sha256:d666ae29164da3e517fcf125e41d4fe96e5bb375cd87ff9763f6b38b5592fe33" + "sha256:daec693491c52e9498632dfbe9ccfc4882a557f5fa08982db1b4d3adbe0887c3", + "sha256:ebdebdb95e3477ceea267decfc0784859aa3df3e27e22d23b83e9b272bf157ae" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==4.59.0" + "version": "==4.60.0" }, "traitlets": { "hashes": [ @@ -1435,7 +1437,7 @@ "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.4" }, "validators": { @@ -1453,6 +1455,29 @@ ], "version": "==0.8.2" }, + "watchdog": { + "hashes": [ + "sha256:035f4816daf3c62e03503c267620f3aa8fc7472df85ff3ef1e0c100ea1ed2744", + "sha256:0f7e9de9ba84af15e9e9fc29c3b13c972daa4d2b11de29aa86b26a26bc877c06", + "sha256:13c9ff58508dce55ba416eb0ef7af5aa5858558f2ec51112f099fd03503b670b", + "sha256:19675b8d1f00dabe74a0e66d87980623250d9360a21612e8c27b70a4b214ceeb", + "sha256:1cd715c4fb803581ded8943f39a51f21c17375d009ca9e3398d6b20638863a70", + "sha256:1f518a6940cde8720b8826a705c164e6b9bd6cf8c00f14269ffac51e017e06ec", + "sha256:3e933f3567c4521dd1a5d59fd54a522cae90bebcbeb8b74b84a2f33c90f08388", + "sha256:41b1a773f364f232b5bc184688e8d60451745d9e0971ac60c648bd47be8f4733", + "sha256:532fedd993e75554671faa36cd04c580ced3fae084254a779afbbd8aaf00566b", + "sha256:74528772516228f6a015a647027057939ff0b695a0b864cb3037e8e1aabc7ca0", + "sha256:89102465764e453609463cf620e744da1b0aa1f9f321b05961e2e7e15b3c9d8b", + "sha256:a412b1914e27f67b0a10e1ee19b5d035a9f7c115a062bbbd640653d9820ba4c8", + "sha256:ac6adbdf32e1d180574f9d0819e80259ae48e68727e80c3d950ed5a023714c3e", + "sha256:adda34bfe6db05485c1dfcd98232bdec385f991fe16358750c2163473eefb985", + "sha256:d2fcbc15772a82cd139c803a513c45b0fbc72a10a8a34dc2a8b429110b6f1236", + "sha256:d54e187b76053982180532cb7fd31152201c438b348c456f699929f8a89e786d", + "sha256:e0114e48ee981b38e328eaa0d5a625c7b4fc144b8dc7f7637749d6b5f7fefb0e" + ], + "markers": "platform_system != 'Darwin'", + "version": "==2.0.2" + }, "wcwidth": { "hashes": [ "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", @@ -1497,6 +1522,14 @@ ], "version": "==1.4.4" }, + "atomicwrites": { + "hashes": [ + "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197", + "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a" + ], + "index": "pypi", + "version": "==1.4.0" + }, "attrs": { "hashes": [ "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", @@ -1520,6 +1553,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==7.1.2" }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "sys_platform == 'win32' and sys_platform == 'win32' and sys_platform == 'win32'", + "version": "==0.4.4" + }, "coverage": { "hashes": [ "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c", @@ -1575,7 +1616,7 @@ "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d", "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==5.5" }, "flake8": { @@ -1664,11 +1705,11 @@ }, "pytest": { "hashes": [ - "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9", - "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839" + "sha256:671238a46e4df0f3498d1c3270e5deb9b32d25134c99b7d75370a68cfbe9b634", + "sha256:6ad9c7bdf517a808242b998ac20063c41532a570d088d77eec1ee12b0b5574bc" ], "index": "pypi", - "version": "==6.2.2" + "version": "==6.2.3" }, "pytest-cov": { "hashes": [ @@ -1680,49 +1721,49 @@ }, "regex": { "hashes": [ - "sha256:07ef35301b4484bce843831e7039a84e19d8d33b3f8b2f9aab86c376813d0139", - "sha256:13f50969028e81765ed2a1c5fcfdc246c245cf8d47986d5172e82ab1a0c42ee5", - "sha256:14de88eda0976020528efc92d0a1f8830e2fb0de2ae6005a6fc4e062553031fa", - "sha256:159fac1a4731409c830d32913f13f68346d6b8e39650ed5d704a9ce2f9ef9cb3", - "sha256:18e25e0afe1cf0f62781a150c1454b2113785401ba285c745acf10c8ca8917df", - "sha256:201e2619a77b21a7780580ab7b5ce43835e242d3e20fef50f66a8df0542e437f", - "sha256:360a01b5fa2ad35b3113ae0c07fb544ad180603fa3b1f074f52d98c1096fa15e", - "sha256:39c44532d0e4f1639a89e52355b949573e1e2c5116106a395642cbbae0ff9bcd", - "sha256:3d9356add82cff75413bec360c1eca3e58db4a9f5dafa1f19650958a81e3249d", - "sha256:3d9a7e215e02bd7646a91fb8bcba30bc55fd42a719d6b35cf80e5bae31d9134e", - "sha256:4651f839dbde0816798e698626af6a2469eee6d9964824bb5386091255a1694f", - "sha256:486a5f8e11e1f5bbfcad87f7c7745eb14796642323e7e1829a331f87a713daaa", - "sha256:4b8a1fb724904139149a43e172850f35aa6ea97fb0545244dc0b805e0154ed68", - "sha256:4c0788010a93ace8a174d73e7c6c9d3e6e3b7ad99a453c8ee8c975ddd9965643", - "sha256:4c2e364491406b7888c2ad4428245fc56c327e34a5dfe58fd40df272b3c3dab3", - "sha256:575a832e09d237ae5fedb825a7a5bc6a116090dd57d6417d4f3b75121c73e3be", - "sha256:5770a51180d85ea468234bc7987f5597803a4c3d7463e7323322fe4a1b181578", - "sha256:633497504e2a485a70a3268d4fc403fe3063a50a50eed1039083e9471ad0101c", - "sha256:63f3ca8451e5ff7133ffbec9eda641aeab2001be1a01878990f6c87e3c44b9d5", - "sha256:709f65bb2fa9825f09892617d01246002097f8f9b6dde8d1bb4083cf554701ba", - "sha256:808404898e9a765e4058bf3d7607d0629000e0a14a6782ccbb089296b76fa8fe", - "sha256:882f53afe31ef0425b405a3f601c0009b44206ea7f55ee1c606aad3cc213a52c", - "sha256:8bd4f91f3fb1c9b1380d6894bd5b4a519409135bec14c0c80151e58394a4e88a", - "sha256:8e65e3e4c6feadf6770e2ad89ad3deb524bcb03d8dc679f381d0568c024e0deb", - "sha256:976a54d44fd043d958a69b18705a910a8376196c6b6ee5f2596ffc11bff4420d", - "sha256:a0d04128e005142260de3733591ddf476e4902c0c23c1af237d9acf3c96e1b38", - "sha256:a0df9a0ad2aad49ea3c7f65edd2ffb3d5c59589b85992a6006354f6fb109bb18", - "sha256:a2ee026f4156789df8644d23ef423e6194fad0bc53575534101bb1de5d67e8ce", - "sha256:a59a2ee329b3de764b21495d78c92ab00b4ea79acef0f7ae8c1067f773570afa", - "sha256:b97ec5d299c10d96617cc851b2e0f81ba5d9d6248413cd374ef7f3a8871ee4a6", - "sha256:b98bc9db003f1079caf07b610377ed1ac2e2c11acc2bea4892e28cc5b509d8d5", - "sha256:b9d8d286c53fe0cbc6d20bf3d583cabcd1499d89034524e3b94c93a5ab85ca90", - "sha256:bcd945175c29a672f13fce13a11893556cd440e37c1b643d6eeab1988c8b209c", - "sha256:c66221e947d7207457f8b6f42b12f613b09efa9669f65a587a2a71f6a0e4d106", - "sha256:c782da0e45aff131f0bed6e66fbcfa589ff2862fc719b83a88640daa01a5aff7", - "sha256:cb4ee827857a5ad9b8ae34d3c8cc51151cb4a3fe082c12ec20ec73e63cc7c6f0", - "sha256:d47d359545b0ccad29d572ecd52c9da945de7cd6cf9c0cfcb0269f76d3555689", - "sha256:dc9963aacb7da5177e40874585d7407c0f93fb9d7518ec58b86e562f633f36cd", - "sha256:ea2f41445852c660ba7c3ebf7d70b3779b20d9ca8ba54485a17740db49f46932", - "sha256:f5d0c921c99297354cecc5a416ee4280bd3f20fd81b9fb671ca6be71499c3fdf", - "sha256:f85d6f41e34f6a2d1607e312820971872944f1661a73d33e1e82d35ea3305e14" - ], - "version": "==2021.3.17" + "sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5", + "sha256:11d773d75fa650cd36f68d7ca936e3c7afaae41b863b8c387a22aaa78d3c5c79", + "sha256:18c071c3eb09c30a264879f0d310d37fe5d3a3111662438889ae2eb6fc570c31", + "sha256:1e1c20e29358165242928c2de1482fb2cf4ea54a6a6dea2bd7a0e0d8ee321500", + "sha256:281d2fd05555079448537fe108d79eb031b403dac622621c78944c235f3fcf11", + "sha256:314d66636c494ed9c148a42731b3834496cc9a2c4251b1661e40936814542b14", + "sha256:32e65442138b7b76dd8173ffa2cf67356b7bc1768851dded39a7a13bf9223da3", + "sha256:339456e7d8c06dd36a22e451d58ef72cef293112b559010db3d054d5560ef439", + "sha256:3916d08be28a1149fb97f7728fca1f7c15d309a9f9682d89d79db75d5e52091c", + "sha256:3a9cd17e6e5c7eb328517969e0cb0c3d31fd329298dd0c04af99ebf42e904f82", + "sha256:47bf5bf60cf04d72bf6055ae5927a0bd9016096bf3d742fa50d9bf9f45aa0711", + "sha256:4c46e22a0933dd783467cf32b3516299fb98cfebd895817d685130cc50cd1093", + "sha256:4c557a7b470908b1712fe27fb1ef20772b78079808c87d20a90d051660b1d69a", + "sha256:52ba3d3f9b942c49d7e4bc105bb28551c44065f139a65062ab7912bef10c9afb", + "sha256:563085e55b0d4fb8f746f6a335893bda5c2cef43b2f0258fe1020ab1dd874df8", + "sha256:598585c9f0af8374c28edd609eb291b5726d7cbce16be6a8b95aa074d252ee17", + "sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000", + "sha256:67bdb9702427ceddc6ef3dc382455e90f785af4c13d495f9626861763ee13f9d", + "sha256:6d1b01031dedf2503631d0903cb563743f397ccaf6607a5e3b19a3d76fc10480", + "sha256:741a9647fcf2e45f3a1cf0e24f5e17febf3efe8d4ba1281dcc3aa0459ef424dc", + "sha256:7c2a1af393fcc09e898beba5dd59196edaa3116191cc7257f9224beaed3e1aa0", + "sha256:7d9884d86dd4dd489e981d94a65cd30d6f07203d90e98f6f657f05170f6324c9", + "sha256:90f11ff637fe8798933fb29f5ae1148c978cccb0452005bf4c69e13db951e765", + "sha256:919859aa909429fb5aa9cf8807f6045592c85ef56fdd30a9a3747e513db2536e", + "sha256:96fcd1888ab4d03adfc9303a7b3c0bd78c5412b2bfbe76db5b56d9eae004907a", + "sha256:97f29f57d5b84e73fbaf99ab3e26134e6687348e95ef6b48cfd2c06807005a07", + "sha256:980d7be47c84979d9136328d882f67ec5e50008681d94ecc8afa8a65ed1f4a6f", + "sha256:a91aa8619b23b79bcbeb37abe286f2f408d2f2d6f29a17237afda55bb54e7aac", + "sha256:ade17eb5d643b7fead300a1641e9f45401c98eee23763e9ed66a43f92f20b4a7", + "sha256:b9c3db21af35e3b3c05764461b262d6f05bbca08a71a7849fd79d47ba7bc33ed", + "sha256:bd28bc2e3a772acbb07787c6308e00d9626ff89e3bfcdebe87fa5afbfdedf968", + "sha256:bf5824bfac591ddb2c1f0a5f4ab72da28994548c708d2191e3b87dd207eb3ad7", + "sha256:c0502c0fadef0d23b128605d69b58edb2c681c25d44574fc673b0e52dce71ee2", + "sha256:c38c71df845e2aabb7fb0b920d11a1b5ac8526005e533a8920aea97efb8ec6a4", + "sha256:ce15b6d103daff8e9fee13cf7f0add05245a05d866e73926c358e871221eae87", + "sha256:d3029c340cfbb3ac0a71798100ccc13b97dddf373a4ae56b6a72cf70dfd53bc8", + "sha256:e512d8ef5ad7b898cdb2d8ee1cb09a8339e4f8be706d27eaa180c2f177248a10", + "sha256:e8e5b509d5c2ff12f8418006d5a90e9436766133b564db0abaec92fd27fcee29", + "sha256:ee54ff27bf0afaf4c3b3a62bcd016c12c3fdb4ec4f413391a90bd38bc3624605", + "sha256:fa4537fb4a98fe8fde99626e4681cc644bdcf2a795038533f9f711513a862ae6", + "sha256:fd45ff9293d9274c5008a2054ecef86a9bfe819a67c7be1afb65e69b405b3042" + ], + "version": "==2021.4.4" }, "toml": { "hashes": [ @@ -1734,38 +1775,38 @@ }, "typed-ast": { "hashes": [ - "sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1", - "sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d", - "sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6", - "sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd", - "sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37", - "sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151", - "sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07", - "sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440", - "sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70", - "sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496", - "sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea", - "sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400", - "sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc", - "sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606", - "sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc", - "sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581", - "sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412", - "sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a", - "sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2", - "sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787", - "sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f", - "sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937", - "sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64", - "sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487", - "sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b", - "sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41", - "sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a", - "sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3", - "sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166", - "sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10" - ], - "version": "==1.4.2" + "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace", + "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff", + "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266", + "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528", + "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6", + "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808", + "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4", + "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363", + "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341", + "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04", + "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41", + "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e", + "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3", + "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899", + "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805", + "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c", + "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c", + "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39", + "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a", + "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3", + "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7", + "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f", + "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075", + "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0", + "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40", + "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428", + "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927", + "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3", + "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f", + "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65" + ], + "version": "==1.4.3" }, "typing-extensions": { "hashes": [ diff --git a/docs/LANDING_PAGE.md b/docs/LANDING_PAGE.md new file mode 100644 index 00000000..1944a0fb --- /dev/null +++ b/docs/LANDING_PAGE.md @@ -0,0 +1,133 @@ +# Welcome to GatorMiner! + +GatorMiner is an automated text-mining tool written in Python to measure the technical +responsibility of students in computer science courses. It is being used to analyze +students' markdown reflection documents and five questions survey based on +Natural Language Processing in the Department of Computer Science at Allegheny +College. + +## Data Retrieving + +There are currently two ways to import text data for analysis: through local file system or AWS DynamoDB. + +### Local File System + +What is a local file system? + +- A controlled place where data can be stored and received. In this case, this + is where GatorMiner keeps data isolated so it can be easily identified. + +In GatorMiner, you can type in the path(s) to the directories(s) that hold +reflection markdown documents. You are welcome to try the tool with the sample documents we provided +in the 'resources', for example: + +``` +resources/sample_md_reflections/lab1, resources/sample_md_reflections/lab2, resources/sample_md_reflections/lab3 +``` + +### AWS + +Retrieving reflection documents from AWS is a feature integrated with the use +of [GatorGrader](https://github.com/GatorEducator/gatorgrader) where students' +markdown reflection documents are being collected and stored inside the a +pre-configured DynamoDB database. In order to use this feature, you will need +to have some credential tokens (listed below) stored as environment variables: + +``` +export GATOR_ENDPOINT= +export GATOR_API_KEY= +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +``` + +It is likely that you already have these prepared when using GatorMiner in +conjunction with GatorGrader, since these would already be exported when +setting up the AWS services. You can read more about setting up an AWS service +with GatorGrader [here](https://github.com/enpuyou/script-api-lambda-dynamodb). + +Once the documents are successfully imported, you can then navigate through +the select box in the sidebar to view the text analysis: + +browser + +## Analysis + +### Frequency Analysis + +Frequency analysis is the quantification and analysis of word usage in text (how often a word appears within a certain text). Overall, frequency analysis can provide amazing insight into the many aspects of assignments that instructors may not always be able to observe. There is a lot of value in making this information available in a user-friendly and intuitive fashion. This can be achieved using GatorMiner frequency analysis. + +Within the GatorMiner tool, you have the ability to choose `Frequency Analysis` as an analysis option after the path to the desired reflection documents is submitted. + +When the tool runs a frequency analysis, on any number of assignments, it provides 3 different options to choose from: + +- Overall +- Student +- Question + +When `Overall` is selected, the application will display a vertical bar chart containing a list of the words used with the highest frequency for each given assignment. + +When `Student` is selected, a dropdown menu is provided allowing you to pick which student the tool should display frequency data for. As with `Overall`, this data is also displayed as a vertical bar chart and you can display multiple students' data on the same page in order to compare and contrast the types of words that are being used by student. + +Finally, when `Question` is selected, the option to pick one or more specific questions appears. The tool then produces and displays a vertical bar chart which contains frequency information for each of the selected questions in the assignment. This is helpful for comparing the ways in which different terms are utilized within different questions in an assignment. + +browser + +### Sentiment Analysis + +Sentiment analysis (or opinion mining) is the use of natural language processing, text analysis, computational linguistics, and biometrics to systematically identify, extract, quantify, and study affective states and subjective information. Overall, +this is a technique to determine whether data is positive, negative, or neutral. + +Within the GatorMiner tool, you have the ability to choose `Sentiment Analysis` as an analysis option after the path to the desired reflection documents is submitted. + +When the tool runs a Sentiment analysis, on any number of assignments, it provides 3 different options to choose from: + +- Overall +- Student +- Question + +When `Overall` is selected, a scatter plot and a bar chart appear on the screen +displaying the overall sentiment polarity in, for example, assignment-01 given by the users. + +When `Student` is selected, it allows the user to choose a specific student to +observe. When chosen it shows the sentiment shown by the chosen user with a mini bar graph and a bigger version of that using a histogram. Inside this feature, you can also change the number of plots per row. + +Finally, when `Question` is selected, it allows the user to choose a certain question in the drop down menu. When chosen, it shows the user the sentiment the question was given. + +browser + +### Document Similarity + +Document similarity analyzes documents and compares text to determine frequency of words between documents. + +Within the GatorMiner tool, you have the ability to choose `Document Similarity` as an analysis option after the path to the desired reflection documents is submitted. + +In the `Document Similarity` section, you are able to select the type of similarity analysis `TF-IDF` and `Spacy`. + +When `TF-IDF` is selected, the application will display a frequency matrix showing the correlation between documents. It does this buy dividing the frequency of the word by the total number of terms in a document. + +When `Spacy` is selected, the application will display a drop down named 'Model name' with two options: + +- `en_core_web_sm` which is used to produce a correlation matrix for **SMALLER** files. (<13mb) +- `en_core_web_md` which is used to produce a correlation matrix for **LARGER** files. (>13mb) + +**Warning exceeding these file limits could cause the program to crash.** + +**See [Spacy.io](https://spacy.io/models/en) for more details of file limits.** + +browser + +### Topic Modeling + +Topic modeling analyzes documents to find keywords in order to determine the documents' dominant topics. + +Within the GatorMiner tool, you have the ability to choose `Topic Modeling` as an analysis option after the path to the desired reflection documents is submitted. + +In the `Topic Modeling` section, you are able to select the type of topic modeling analysis `Histogram` and `Scatter`. + +When `Histogram` is selected, the application will display a histogram in which the dominant topic is on the x-axis and the count of records is on the y-axis. A legend in the top right corner will display the names of the reflection files new to the color that corresponds with them. + +When `Scatter` is selected, the application will display a scatter plot. The legend on the right side will display the colors that correspond to topic numbers and the shapes that correspond with topics. + +Sliders are also provided that can adjust the amount of topics or adjust the amount of words per topic. + +browser diff --git a/src/analyzer.py b/src/analyzer.py index b686b255..f9f0424b 100644 --- a/src/analyzer.py +++ b/src/analyzer.py @@ -1,5 +1,7 @@ """Text Proprocessing""" from collections import Counter +from textblob import TextBlob +import pandas as pd import re import string from typing import List, Tuple @@ -138,3 +140,29 @@ def noun_phrase(input_text): for chunk in doc.noun_chunks: n_phrase_lst.append(str(chunk)) return n_phrase_lst + + +def top_polarized_word(tokens_column): + """Create columns for positive and negative words""" + # Start off with empty lists + pos_series = [] + neg_series = [] + # For each item in the already existing tokens column in the data frame, + for token_element in tokens_column: + words = sorted_sentiment_word_list(token_element) + # Add the words at the end of the list to the display series since + # they have the most positive sentiment value + pos_series.append(", ".join(words[::-1][0:3])) + neg_series.append(", ".join(words[0:3])) + # Return an entire series based on the display_series list + return pd.Series(pos_series), pd.Series(neg_series) + + +def sorted_sentiment_word_list(token_element): + """Creates and sorts a word list from a list of tokens""" + # Convert the token list into a set so that it only has the unique words + words = set(token_element) + # Convert back into list to iterate through + unique_words = list(words) + # Returning the sorted list + return sorted(unique_words, key=lambda x: TextBlob(x).sentiment.polarity) diff --git a/src/constants.py b/src/constants.py index 0756e431..f3028ffc 100644 --- a/src/constants.py +++ b/src/constants.py @@ -18,3 +18,7 @@ NORMAL = "normalized" ASSIGNMENT = "assignment" SENTI = "sentiment" + +# Columns +POSITIVE = "Positive words" +NEGATIVE = "Negative words" diff --git a/src/markdown.py b/src/markdown.py index 92c2ab41..488df6cd 100644 --- a/src/markdown.py +++ b/src/markdown.py @@ -1,6 +1,7 @@ """Markdown parser""" import os import logging +from io import StringIO from typing import Dict, List import commonmark import pandas as pd @@ -36,19 +37,25 @@ def get_file_names(directory_name: str) -> List[str]: return file_list -def merge_dict(dict_1, dict_2: Dict[str, str]) -> Dict[str, List[str]]: +def merge_dict(dict_1, dict_2: Dict[str, str], preserve: bool) -> \ + Dict[str, List[str]]: """Merge two dictionaries and store values of common keys in list""" if dict_1 is None: dict_1 = {k: [] for k in dict_2.keys()} elif isinstance(list(dict_1.values())[0], list) is False: dict_1 = {k: [v] for k, v in dict_1.items()} + if(preserve): + for key in dict_2.keys(): + if key not in dict_1: + dict_1[key] = [] + for f in range(0, len(list(dict_1.values())[0])): + dict_1[key].append("") for key in dict_1.keys(): try: dict_1[key].append(dict_2[key]) except KeyError as err: dict_1[key].append("") logging.warning(f"Key does not exist: {err}") - return dict_1 @@ -58,7 +65,7 @@ def collect_md(directory: str, is_clean=True) -> Dict[str, List[str]]: main_md_dict = None for file in file_names: individual_dict = md_parser(read_file(file), is_clean) - main_md_dict = merge_dict(main_md_dict, individual_dict) + main_md_dict = merge_dict(main_md_dict, individual_dict, False) return main_md_dict @@ -95,10 +102,19 @@ def md_parser(input_md: str, is_clean=True) -> Dict[str, str]: md_dict[cur_heading] += subnode.literal + " " else: continue - print(md_dict) return md_dict +def import_uploaded_files(paths: List) -> Dict[str, List[str]]: + """Importing the individual files""" + main_md_dict = None + for path in paths: + stringio = StringIO(path.getvalue().decode("utf-8")) + individual_dict = md_parser(stringio.read(), True) + main_md_dict = merge_dict(main_md_dict, individual_dict, True) + return main_md_dict + + def build_pd(md_dict): """build dictionary into dataframe""" md_df = pd.DataFrame(md_dict) diff --git a/src/visualization.py b/src/visualization.py index 333d9d7e..0100f097 100644 --- a/src/visualization.py +++ b/src/visualization.py @@ -141,6 +141,8 @@ def senti_circleplot(senti_df, student_id): tooltip=[ alt.Tooltip(cts.SENTI, title="polarity"), alt.Tooltip(student_id, title="author"), + alt.Tooltip(cts.POSITIVE, title="Positive words"), + alt.Tooltip(cts.NEGATIVE, title="Negative words"), ], ) ).interactive() diff --git a/streamlit_web.py b/streamlit_web.py index 41720487..2866720a 100644 --- a/streamlit_web.py +++ b/streamlit_web.py @@ -1,7 +1,6 @@ """Web interface""" import re - import base64 import numpy as np import os @@ -33,6 +32,7 @@ stu_id = None success_msg = None debug_mode = False +main_md_dict = None def main(): @@ -42,8 +42,9 @@ def main(): data_retreive_method = st.sidebar.selectbox( "Choose the data retrieving method", [ - "Local file system", + "Path input", "AWS", + "Upload local files", ], ) if retreive_data(data_retreive_method): @@ -62,7 +63,7 @@ def main(): if debug_mode: st.write(main_df) if analysis_mode == "Home": - readme() + landing_src() else: if analysis_mode == "Frequency Analysis": st.title(analysis_mode) @@ -84,26 +85,26 @@ def main(): interactive() success_msg.empty() -def readme(): +def landing_src(): """function to load and configurate readme source""" - with open("README.md") as readme_file: - readme_src = readme_file.read() + with open("docs/LANDING_PAGE.md") as landing_file: + landing_src = landing_file.read() for file in os.listdir("resources/images"): if file.endswith(".png"): img_path = f"resources/images/{file}" with open(img_path, "rb") as f: img_bin = base64.b64encode(f.read()).decode() - readme_src = readme_src.replace(img_path, f"data:image/png;base64,{img_bin}") + landing_src = landing_src.replace(img_path, f"data:image/png;base64,{img_bin}") - st.markdown(readme_src, unsafe_allow_html=True) + st.markdown(landing_src, unsafe_allow_html=True) def landing_pg(): """landing page""" landing = st.sidebar.selectbox("Welcome", ["Home", "Interactive"]) if landing == "Home": - readme() + landing_src() else: interactive() @@ -112,11 +113,11 @@ def retreive_data(data_retreive): """pipeline to retrieve data from user input to output""" global preprocessed_df global main_df - if data_retreive == "Local file system": + if data_retreive == "Path input": input_assignments = st.sidebar.text_input( "Enter path(s) to markdown documents (seperate by comma)" ) - else: + elif data_retreive == "AWS": input_assignments = st.sidebar.text_input( "Enter assignment names of the markdown \ documents(seperate by comma)" @@ -124,17 +125,22 @@ def retreive_data(data_retreive): st.sidebar.info( "You will need to store keys and endpoints in the \ environment variables") + else: + input_assignments = st.sidebar.file_uploader("Choose a Markdown file", + type=['md'], + accept_multiple_files=True) if not input_assignments: landing_pg() else: - input_assignments = re.split(r"[;,\s]\s*", input_assignments) + if data_retreive == "AWS" or data_retreive == "Path input": + input_assignments = re.split(r"[;,\s]\s*", input_assignments) try: main_df, preprocessed_df = import_data( data_retreive, input_assignments) except TypeError: st.sidebar.warning( "No data imported. Please check the reflection document input") - readme() + landing_src() else: global success_msg success_msg = None @@ -164,14 +170,16 @@ def load_model(name): def import_data(data_retreive_method, paths): """pipeline to import data from local or aws""" json_lst = [] - if data_retreive_method == "Local file system": + global main_md_dict + if data_retreive_method == "Path input": try: for path in paths: json_lst.append(md.collect_md(path)) except FileNotFoundError as err: st.sidebar.text(err) - readme() - else: + with open("README.md") as readme_file: + st.markdown(readme_file.read()) + elif data_retreive_method == "AWS": passbuild = st.sidebar.checkbox( "Only retreive build success records", value=True) try: @@ -181,7 +189,18 @@ def import_data(data_retreive_method, paths): json_lst.append(ju.clean_report(response)) except (EnvironmentError, Exception) as err: st.sidebar.error(err) - readme() + with open("README.md") as readme_file: + st.markdown(readme_file.read()) + else: + try: + if len(paths) < 2: + st.sidebar.warning("Please select more than one file!") + else: + json_lst.append(md.import_uploaded_files(paths)) + except FileNotFoundError as err: + st.sidebar.text(err) + with open("README.md") as readme_file: + st.markdown(readme_file.read()) # when data is retreived if json_lst: raw_df = pd.DataFrame() @@ -292,7 +311,8 @@ def student_freq(freq_range): .to_string(), freq_range, ) - ind_df = pd.DataFrame(individual_freq, columns=["word", "freq"]) + ind_df = pd.DataFrame(individual_freq, + columns=["word", "freq"]) ind_df["assignments"] = item ind_df["student"] = student freq_df = freq_df.append(ind_df) @@ -357,6 +377,9 @@ def question_freq(freq_range): def sentiment(): """main function for sentiment analysis""" senti_df = main_df.copy(deep=True) + # Initializing the new columns with a numpy array, so the entire series is returned + senti_df[cts.POSITIVE], senti_df[cts.NEGATIVE] = az.top_polarized_word(senti_df[cts.TOKEN].values) + # calculate overall sentiment from the combined text senti_df[cts.SENTI] = senti_df["combined"].apply( lambda x: TextBlob(az.lemmatized_text(x)).sentiment.polarity @@ -372,7 +395,8 @@ def sentiment(): st.header(f"Overall sentiment polarity in **{assign_text}**") overall_senti(senti_df) elif senti_type == "Student": - st.header(f"View sentiment by individual students in **{assign_text}**") + st.header(f"View sentiment by individual students in " + f"**{assign_text}**") student_senti(senti_df) elif senti_type == "Question": st.header( @@ -388,7 +412,6 @@ def overall_senti(senti_df): st.altair_chart(vis.stu_senti_lineplot(senti_df, stu_id)) st.altair_chart((vis.senti_combinedplot(senti_df, stu_id))) - def student_senti(input_df): """page for display individual student's sentiment""" students = st.multiselect( @@ -639,7 +662,8 @@ def interactive(): # Newlines seem to mess with the rendering html = html.replace("\n", " ") HTML_WRAPPER = """
\ + #e6e9ef; border-radius: 0.25rem; padding: 1rem; + margin-bottom: 2.5rem">\ {}
""" st.write(HTML_WRAPPER.format(html), unsafe_allow_html=True) else: @@ -651,6 +675,5 @@ def interactive(): summaries = sz.summarize_text(input_text) st.write(summaries) - if __name__ == "__main__": main() diff --git a/tests/test_analyzer.py b/tests/test_analyzer.py index 2a085b1a..66a89cf9 100644 --- a/tests/test_analyzer.py +++ b/tests/test_analyzer.py @@ -1,6 +1,8 @@ """Test module for analyzer.py""" import pytest import src.analyzer as az +import pandas as pd +import src.constants as cts def test_tokenize(): @@ -157,3 +159,20 @@ def test_tfidf(): term_frequency, vector = az.compute_tfidf(input_tokens) assert term_frequency is not None assert vector is not None + + +def test_top_polarized_word(): + """Tests if the positive/negative words columns are created""" + df = pd.DataFrame(columns=[cts.TOKEN, cts.POSITIVE, cts.NEGATIVE]) + input_tokens = [ + ["incredible", "horrible", "terrific", "terrible"], + ["amazing", "devastating", "boring", "cool"], + ["alarming", "awesome", "beautiful", "ugly"], + ] + df[cts.TOKEN] = pd.Series(input_tokens) + df[cts.POSITIVE], df[cts.NEGATIVE] = \ + az.top_polarized_word(df[cts.TOKEN].values) + assert df[cts.POSITIVE] is not None + assert df[cts.NEGATIVE] is not None + assert df[cts.POSITIVE].size is df[cts.TOKEN].size + assert df[cts.NEGATIVE].size is df[cts.TOKEN].size diff --git a/tests/test_markdown.py b/tests/test_markdown.py index 547203c1..6f02a264 100644 --- a/tests/test_markdown.py +++ b/tests/test_markdown.py @@ -1,15 +1,31 @@ """Test module for markdown.py""" import pytest import src.markdown as md +import io +import os def test_merge_dict(): """Test that merge_dict returns a dict of key and value lists""" test_dict_1 = {"key1": "value1"} test_dict_2 = {"key1": "value2"} - output = md.merge_dict(test_dict_1, test_dict_2) + output_preserved = md.merge_dict(test_dict_1, test_dict_2, True) + output_not_preserved = md.merge_dict(test_dict_1, test_dict_2, False) expected = {"key1": ["value1", "value2"]} - assert expected == output + assert expected == output_preserved + assert expected == output_not_preserved + + +def test_merge_dict_keys(): + """Test that merge_dict adds empty string for unique keys.""" + test_dict_1 = {"key1": "value1"} + test_dict_2 = {"key1": "value2", "key2": "value3"} + output_preserved = md.merge_dict(test_dict_1, test_dict_2, True) + output_not_preserved = md.merge_dict(test_dict_1, test_dict_2, False) + expected_preserved = {"key1": ["value1", "value2"], "key2": ["", "value3"]} + expected_not_preserved = {"key1": ["value1", "value2"]} + assert expected_preserved == output_preserved + assert expected_not_preserved == output_not_preserved def test_get_file_names(tmp_path): @@ -21,7 +37,8 @@ def test_get_file_names(tmp_path): para_1.write_text("text") para_2.write_text("text") output = md.get_file_names(directory) - expected = [f"{directory}/hello.md", f"{directory}/world.md"] + expected = [f"{directory}{os.path.sep}hello.md", + f"{directory}{os.path.sep}world.md"] assert expected[0] in output assert expected[1] in output @@ -117,3 +134,17 @@ def test_md_parser_clean(input_text, expected): """Test if md parser cleans the markdown documents""" output = md.md_parser(input_text) assert output == expected + + +def test_import_uploaded_files(): + """Test if the uploaded files are actually imported""" + uploaded_files = [] + uploaded_files.append(io.BytesIO( + open("resources/sample_md_reflections/lab1/reflection1.md", "rb") + .read())) + uploaded_files.append(io.BytesIO( + open("resources/sample_md_reflections/lab1/reflection2.md", "rb") + .read())) + json_lst = [] + json_lst.append(md.import_uploaded_files(uploaded_files)) + assert json_lst is not []