From e8be37736c08060a54bd3d41bae647fdbfdbd9f9 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 12:23:20 -0400 Subject: [PATCH 01/17] First push as MetaStalk --- .deepsource.toml | 4 +- .gitlab-ci.yml | 86 +++++-- CHANGELOG.md | 8 +- .../22-canon_tags.jpg | Bin .../32-lens_data.jpeg | Bin .../45-gps_ifd.jpg | Bin .../Canon_40D_photoshop_import.jpg | Bin .../DSCN0010.jpg | Bin .../DSCN0012.jpg | Bin .../DSCN0021.jpg | Bin .../DSCN0025.jpg | Bin .../DSCN0027.jpg | Bin .../DSCN0029.jpg | Bin .../DSCN0038.jpg | Bin .../DSCN0040.jpg | Bin .../DSCN0042.jpg | Bin .../Fujifilm_FinePix_E500.jpg | Bin .../Nikon_D70.jpg | Bin .../Olympus_C8080WZ.jpg | Bin .../Panasonic_DMC-FZ30.jpg | Bin .../dotnet-256x256-alpha-palette.png | Bin .../invalid_exif.png | Bin .../ExamplePhotos => ExamplePhotos}/jolla.jpg | Bin .../sony-powershota5.jpg | Bin MetaStalk/__init__.py | 3 + PyStalk.py => MetaStalk/main.py | 8 +- {modules => MetaStalk/modules}/DateTime.py | 1 + {modules => MetaStalk/modules}/GPSCheck.py | 2 +- {modules => MetaStalk/modules}/PieChart.py | 2 +- {modules => MetaStalk/modules}/Stats.py | 0 {modules => MetaStalk/modules}/__init__.py | 0 MetaStalk/tests/test_main.py | 9 + {utils => MetaStalk/utils}/__init__.py | 0 {utils => MetaStalk/utils}/assets/favicon.ico | Bin {utils => MetaStalk/utils}/logger.py | 0 {utils => MetaStalk/utils}/web.py | 4 +- Pipfile | 1 - Pipfile.lock | 240 +++++++++--------- Readme.md | 17 +- setup.py | 45 ++++ 40 files changed, 263 insertions(+), 167 deletions(-) rename {utils/ExamplePhotos => ExamplePhotos}/22-canon_tags.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/32-lens_data.jpeg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/45-gps_ifd.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/Canon_40D_photoshop_import.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/DSCN0010.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/DSCN0012.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/DSCN0021.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/DSCN0025.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/DSCN0027.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/DSCN0029.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/DSCN0038.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/DSCN0040.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/DSCN0042.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/Fujifilm_FinePix_E500.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/Nikon_D70.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/Olympus_C8080WZ.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/Panasonic_DMC-FZ30.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/dotnet-256x256-alpha-palette.png (100%) rename {utils/ExamplePhotos => ExamplePhotos}/invalid_exif.png (100%) rename {utils/ExamplePhotos => ExamplePhotos}/jolla.jpg (100%) rename {utils/ExamplePhotos => ExamplePhotos}/sony-powershota5.jpg (100%) create mode 100644 MetaStalk/__init__.py rename PyStalk.py => MetaStalk/main.py (95%) rename {modules => MetaStalk/modules}/DateTime.py (93%) rename {modules => MetaStalk/modules}/GPSCheck.py (92%) rename {modules => MetaStalk/modules}/PieChart.py (90%) rename {modules => MetaStalk/modules}/Stats.py (100%) rename {modules => MetaStalk/modules}/__init__.py (100%) create mode 100644 MetaStalk/tests/test_main.py rename {utils => MetaStalk/utils}/__init__.py (100%) rename {utils => MetaStalk/utils}/assets/favicon.ico (100%) rename {utils => MetaStalk/utils}/logger.py (100%) rename {utils => MetaStalk/utils}/web.py (90%) create mode 100644 setup.py diff --git a/.deepsource.toml b/.deepsource.toml index 63f8746..af150da 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -1,9 +1,7 @@ version = 1 test_patterns = [ - "Pystalk.py", - "modules/**", - "utils/**" + "PyStalk/**", ] [[analyzers]] diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3984f52..6555291 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ stages: - Code_Checking - test +- Test_Build include: - template: SAST.gitlab-ci.yml @@ -14,20 +15,18 @@ Coverage: coverage: '/TOTAL\s*\d*\s*\d*\s*(\d*)%/' artifacts: paths: - - PyStalk.log + - MetaStalk.log - htmlcov/* - bandit.json before_script: - pip install -U pip pipenv - curl https://deepsource.io/cli | sh - - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter - - chmod +x ./cc-test-reporter - - ./cc-test-reporter before-build + - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; chmod +x ./cc-test-reporter; ./cc-test-reporter before-build script: - pipenv install --dev - pipenv install coverage - - pipenv run coverage run -a --source . ./PyStalk.py ./utils/ExamplePhotos/ -t -d - - pipenv run coverage run -a --source . ./PyStalk.py ./utils/ExamplePhotos/Panasonic_DMC-FZ30.jpg ./utils/ExamplePhotos/DSCN0010.jpg -t -d + - pipenv run coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/ -t -d + - pipenv run coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d after_script: - pipenv run coverage report - pipenv run coverage html @@ -42,18 +41,9 @@ Coverage: - pip install -U pip pipenv script: - pipenv install --dev --skip-lock - - pipenv run pylint --ignored-classes=_socketobject PyStalk.py ./utils/ ./modules/ - - pipenv run flake8 PyStalk.py ./utils/ ./modules/ --statistics --show-source - - pipenv run bandit -r -f json PyStalk.py ./utils/ ./modules/ > bandit.json - - time pipenv run python3 ./PyStalk.py ./utils/ExamplePhotos/ -d -t - artifacts: - paths: - - bandit.json - - -python-3.5: - extends: ".check" - image: python:3.5 + - pipenv run pylint --ignored-classes=_socketobject ./MetaStalk + - pipenv run flake8 ./MetaStalk --statistics --show-source + - time pipenv run python3 ./MetaStalk/main.py ./ExamplePhotos/ -d -t python-3.6: extends: ".check" @@ -65,4 +55,62 @@ python-3.7: python-3.8: extends: ".check" - image: python:3.8 \ No newline at end of file + image: python:3.8 + + +.build: + stage: test + before_script: + - pip install -U setuptools twine --quiet + script: + - python setup.py sdist bdist_wheel + - python -m twine check dist/* + except: + - tags + +python-3.6-build: + extends: ".build" + image: python:3.6 + artifacts: + paths: + - dist/ + +python-3.7-build: + extends: ".build" + image: python:3.7 + artifacts: + paths: + - dist/ + +python-3.8-build: + extends: ".build" + image: python:3.8 + artifacts: + paths: + - dist/ + +.test: + stage: Test_Build + before_script: + - pip install -U pip + script: + - pip install dist/*.whl + - pip show -f metastalk + - command -v metastalk + - metastalk ./ExamplePhotos/ -t -d + - metastalk ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d + except: + - tags + +python-3.6-test: + extends: ".test" + image: python:3.6 + +python-3.7-test: + extends: ".test" + image: python:3.7 + +python-3.8-test: + extends: ".test" + image: python:3.8 + \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 780779d..dc789ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,18 +4,22 @@ ## [N/A] - UNRELEASED -All backend changes. No changes to PyStalk itself. +Rename to MetaStalk to create PyPi package and a lot backend changes. ## Added - License scanning - [Codecov](https://codecov.io/gl/Cyb3r-Jak3/pystalk) -- Pipfile and Pipfile.lock for pipenv +- Pipfile and Pipfile.lock for pipenv. - Added .gitlab folder for service desk. +- setup.py for pypi. ### Changed - Reverted dependency scanner to default template. +- Name from PyStalk to MetaStalk as PyStalk is taken. +- Moved all files to MetaStalk directory. +- Moved ExamplePhotos to own directory. ## [v1.4] - 2020-02-02 diff --git a/utils/ExamplePhotos/22-canon_tags.jpg b/ExamplePhotos/22-canon_tags.jpg similarity index 100% rename from utils/ExamplePhotos/22-canon_tags.jpg rename to ExamplePhotos/22-canon_tags.jpg diff --git a/utils/ExamplePhotos/32-lens_data.jpeg b/ExamplePhotos/32-lens_data.jpeg similarity index 100% rename from utils/ExamplePhotos/32-lens_data.jpeg rename to ExamplePhotos/32-lens_data.jpeg diff --git a/utils/ExamplePhotos/45-gps_ifd.jpg b/ExamplePhotos/45-gps_ifd.jpg similarity index 100% rename from utils/ExamplePhotos/45-gps_ifd.jpg rename to ExamplePhotos/45-gps_ifd.jpg diff --git a/utils/ExamplePhotos/Canon_40D_photoshop_import.jpg b/ExamplePhotos/Canon_40D_photoshop_import.jpg similarity index 100% rename from utils/ExamplePhotos/Canon_40D_photoshop_import.jpg rename to ExamplePhotos/Canon_40D_photoshop_import.jpg diff --git a/utils/ExamplePhotos/DSCN0010.jpg b/ExamplePhotos/DSCN0010.jpg similarity index 100% rename from utils/ExamplePhotos/DSCN0010.jpg rename to ExamplePhotos/DSCN0010.jpg diff --git a/utils/ExamplePhotos/DSCN0012.jpg b/ExamplePhotos/DSCN0012.jpg similarity index 100% rename from utils/ExamplePhotos/DSCN0012.jpg rename to ExamplePhotos/DSCN0012.jpg diff --git a/utils/ExamplePhotos/DSCN0021.jpg b/ExamplePhotos/DSCN0021.jpg similarity index 100% rename from utils/ExamplePhotos/DSCN0021.jpg rename to ExamplePhotos/DSCN0021.jpg diff --git a/utils/ExamplePhotos/DSCN0025.jpg b/ExamplePhotos/DSCN0025.jpg similarity index 100% rename from utils/ExamplePhotos/DSCN0025.jpg rename to ExamplePhotos/DSCN0025.jpg diff --git a/utils/ExamplePhotos/DSCN0027.jpg b/ExamplePhotos/DSCN0027.jpg similarity index 100% rename from utils/ExamplePhotos/DSCN0027.jpg rename to ExamplePhotos/DSCN0027.jpg diff --git a/utils/ExamplePhotos/DSCN0029.jpg b/ExamplePhotos/DSCN0029.jpg similarity index 100% rename from utils/ExamplePhotos/DSCN0029.jpg rename to ExamplePhotos/DSCN0029.jpg diff --git a/utils/ExamplePhotos/DSCN0038.jpg b/ExamplePhotos/DSCN0038.jpg similarity index 100% rename from utils/ExamplePhotos/DSCN0038.jpg rename to ExamplePhotos/DSCN0038.jpg diff --git a/utils/ExamplePhotos/DSCN0040.jpg b/ExamplePhotos/DSCN0040.jpg similarity index 100% rename from utils/ExamplePhotos/DSCN0040.jpg rename to ExamplePhotos/DSCN0040.jpg diff --git a/utils/ExamplePhotos/DSCN0042.jpg b/ExamplePhotos/DSCN0042.jpg similarity index 100% rename from utils/ExamplePhotos/DSCN0042.jpg rename to ExamplePhotos/DSCN0042.jpg diff --git a/utils/ExamplePhotos/Fujifilm_FinePix_E500.jpg b/ExamplePhotos/Fujifilm_FinePix_E500.jpg similarity index 100% rename from utils/ExamplePhotos/Fujifilm_FinePix_E500.jpg rename to ExamplePhotos/Fujifilm_FinePix_E500.jpg diff --git a/utils/ExamplePhotos/Nikon_D70.jpg b/ExamplePhotos/Nikon_D70.jpg similarity index 100% rename from utils/ExamplePhotos/Nikon_D70.jpg rename to ExamplePhotos/Nikon_D70.jpg diff --git a/utils/ExamplePhotos/Olympus_C8080WZ.jpg b/ExamplePhotos/Olympus_C8080WZ.jpg similarity index 100% rename from utils/ExamplePhotos/Olympus_C8080WZ.jpg rename to ExamplePhotos/Olympus_C8080WZ.jpg diff --git a/utils/ExamplePhotos/Panasonic_DMC-FZ30.jpg b/ExamplePhotos/Panasonic_DMC-FZ30.jpg similarity index 100% rename from utils/ExamplePhotos/Panasonic_DMC-FZ30.jpg rename to ExamplePhotos/Panasonic_DMC-FZ30.jpg diff --git a/utils/ExamplePhotos/dotnet-256x256-alpha-palette.png b/ExamplePhotos/dotnet-256x256-alpha-palette.png similarity index 100% rename from utils/ExamplePhotos/dotnet-256x256-alpha-palette.png rename to ExamplePhotos/dotnet-256x256-alpha-palette.png diff --git a/utils/ExamplePhotos/invalid_exif.png b/ExamplePhotos/invalid_exif.png similarity index 100% rename from utils/ExamplePhotos/invalid_exif.png rename to ExamplePhotos/invalid_exif.png diff --git a/utils/ExamplePhotos/jolla.jpg b/ExamplePhotos/jolla.jpg similarity index 100% rename from utils/ExamplePhotos/jolla.jpg rename to ExamplePhotos/jolla.jpg diff --git a/utils/ExamplePhotos/sony-powershota5.jpg b/ExamplePhotos/sony-powershota5.jpg similarity index 100% rename from utils/ExamplePhotos/sony-powershota5.jpg rename to ExamplePhotos/sony-powershota5.jpg diff --git a/MetaStalk/__init__.py b/MetaStalk/__init__.py new file mode 100644 index 0000000..99ca0f2 --- /dev/null +++ b/MetaStalk/__init__.py @@ -0,0 +1,3 @@ +"""PyStalk +""" +__version__ = "v1.4.0" diff --git a/PyStalk.py b/MetaStalk/main.py similarity index 95% rename from PyStalk.py rename to MetaStalk/main.py index b32b180..36af8ca 100644 --- a/PyStalk.py +++ b/MetaStalk/main.py @@ -4,10 +4,10 @@ import argparse import os import logging -import sys import timeit from hachoir.parser import createParser from hachoir.metadata import extractMetadata + import utils import modules @@ -17,7 +17,7 @@ def start(): """ Sets up PyStalk and parses arguments""" - parser = argparse.ArgumentParser(prog="PyStalk", + parser = argparse.ArgumentParser(prog="MetaStalk", description="Tool to graph " "image metadata.") parser.add_argument('files', nargs='*', default=None, @@ -33,11 +33,11 @@ def start(): const=logging.INFO) args = parser.parse_args() - log = utils.make_logger("PyStalk", args.loglevel) + log = utils.make_logger("MetaStalk", args.loglevel) log.info("Starting up") if not args.files: log.error("ERROR: No path was inputted.") - sys.exit(1) + raise IOError("No path was inputted.") run(args, log) diff --git a/modules/DateTime.py b/MetaStalk/modules/DateTime.py similarity index 93% rename from modules/DateTime.py rename to MetaStalk/modules/DateTime.py index b239f24..06b1914 100644 --- a/modules/DateTime.py +++ b/MetaStalk/modules/DateTime.py @@ -4,6 +4,7 @@ def date_time(photos, log): """Makes a table with gps timestamp of photos""" + log.info("Starting DateTime Charts") datetime = [] datetime_original = [] datetime_digitized = [] diff --git a/modules/GPSCheck.py b/MetaStalk/modules/GPSCheck.py similarity index 92% rename from modules/GPSCheck.py rename to MetaStalk/modules/GPSCheck.py index ae7a588..a33c042 100644 --- a/modules/GPSCheck.py +++ b/MetaStalk/modules/GPSCheck.py @@ -4,7 +4,7 @@ def GPS_Check(photos: list, log): """Takes a list of photos and creates a geo plot of them""" - log.info("Starting GPS Check") + log.info("Starting GPS Chart") lats = [] longs = [] gps_photos = [] diff --git a/modules/PieChart.py b/MetaStalk/modules/PieChart.py similarity index 90% rename from modules/PieChart.py rename to MetaStalk/modules/PieChart.py index ca5b78d..01f364c 100644 --- a/modules/PieChart.py +++ b/MetaStalk/modules/PieChart.py @@ -34,6 +34,6 @@ def PieChart(photos: list, pietype: str, log): try: table.append(each[pietype]) except KeyError: - log.info("%s has no %s data", each["item"], pietype) + log.debug("%s has no %s data", each["item"], pietype) return create_chart(table, pietype) diff --git a/modules/Stats.py b/MetaStalk/modules/Stats.py similarity index 100% rename from modules/Stats.py rename to MetaStalk/modules/Stats.py diff --git a/modules/__init__.py b/MetaStalk/modules/__init__.py similarity index 100% rename from modules/__init__.py rename to MetaStalk/modules/__init__.py diff --git a/MetaStalk/tests/test_main.py b/MetaStalk/tests/test_main.py new file mode 100644 index 0000000..7abaf09 --- /dev/null +++ b/MetaStalk/tests/test_main.py @@ -0,0 +1,9 @@ +"""Test suite for PyStalk""" +import pytest +import PyStalk + + +def empty_directory_test(): + """Shows result for empty directory""" + with pytest.raises(FileNotFoundError): + PyStalk.run() diff --git a/utils/__init__.py b/MetaStalk/utils/__init__.py similarity index 100% rename from utils/__init__.py rename to MetaStalk/utils/__init__.py diff --git a/utils/assets/favicon.ico b/MetaStalk/utils/assets/favicon.ico similarity index 100% rename from utils/assets/favicon.ico rename to MetaStalk/utils/assets/favicon.ico diff --git a/utils/logger.py b/MetaStalk/utils/logger.py similarity index 100% rename from utils/logger.py rename to MetaStalk/utils/logger.py diff --git a/utils/web.py b/MetaStalk/utils/web.py similarity index 90% rename from utils/web.py rename to MetaStalk/utils/web.py index 2962225..66e9783 100644 --- a/utils/web.py +++ b/MetaStalk/utils/web.py @@ -10,13 +10,13 @@ def graph(plots: dict, log, t_start: float, test=False): """Creates the graphs""" external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css'] app = dash.Dash(__name__, external_stylesheets=external_stylesheets) - app.title = "PyStalk" + app.title = "MetaStalk" graphs = [] for name, chart in plots.items(): graphs.append(dcc.Graph(id="graph-{}".format(name), figure=chart)) t_stop = timeit.default_timer() app.layout = html.Div([ - html.H1("PyStalk", style={"textAlign": "center"}), + html.H1("MetaStalk", style={"textAlign": "center"}), html.H6(html.A('Cyber Jake', href="https://twitter.com/Cyb3r_Jak3"), style={"textAlign": "center"}), html.Div(children=graphs), diff --git a/Pipfile b/Pipfile index 3221d63..298f40c 100644 --- a/Pipfile +++ b/Pipfile @@ -6,7 +6,6 @@ verify_ssl = true [dev-packages] flake8 = "*" pylint = "*" -bandit = "*" pytest = "*" [packages] diff --git a/Pipfile.lock b/Pipfile.lock index 13b6151..72384c4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "aa3bfd77ff1cbb5719c1f874a8407c8f525743af50342bb5e772aee4a5355294" + "sha256": "6ff3c13fe55702c68765fa233a4656891777dc297256dab69458d02165489a7f" }, "pipfile-spec": 6, "requires": { @@ -16,56 +16,94 @@ ] }, "default": { + "brotli": { + "hashes": [ + "sha256:0538dc1744fd17c314d2adc409ea7d1b779783b89fd95bcfb0c2acc93a6ea5a7", + "sha256:0970a47f471782912d7705160b2b0a9306e68e6fadf9cffcaeb42d8f0951e26c", + "sha256:113f51658e6fe548dce4b3749f6ef6c24de4184ba9c10a909cbee4261c2a5da0", + "sha256:1e1aa9c4d1558889f42749c8baf846007953bfd32c8209230cf1cd1f5ef33495", + "sha256:2f2f4f78f29ac4a45d15b3d9fc3fd9705e0ad313a44b129f6e1d0c6916bad0e2", + "sha256:3269f6de1dd150fd0cce1c158b61ff5ac06d627fd3ae9c6ea03aed26fbbff7ea", + "sha256:3f4a1f6240916c7984c7f2542786710f622992508dafee0b1714e6d340fb9ffd", + "sha256:50dd9ad2a2bb12da4e9002a438672d182f98e546e99952de80280a1e1729664f", + "sha256:5519a4b01b1a4f965083cbfa2ef2b9774c5a5f352341c47b50776ad109423d72", + "sha256:5eb27722d320370315971c427eb8aa7cc0791f2a458840d357ac653bd0ad3a14", + "sha256:5f06b4d5b6f58e5b5c220c2f23cad034dc5efa51b01fde2351ced1605bd980e2", + "sha256:71ceee286ea7ec613f1c36f1c6181864a6ca24ebb55e371276f33d6af8742834", + "sha256:72848d25a5f9e736db4af4512e0c3feecc094d57d241f8f1ae959115a2c39756", + "sha256:743001bca75f4a6b4454be3510feca46f9d61a0c782a9bc2bc684bdb245e279e", + "sha256:7ac98c71a15648fd11bc1f32608b6110e396121280790082e32b9a3109048bc6", + "sha256:9d1c2dd27a1083fefd05b1b2f8df4a6bc2aaa6c21dd82cd41c8ae5e7c23a87f8", + "sha256:a13ce9b419fe9f277c63f700efb0e444331509d1881b5610d2ba7e9080606967", + "sha256:a19ef0952b9d2803df88dff07f45a6c92d5676afb9b8d69cf32232d684036d11", + "sha256:ad766ca8b8c1419b71a22756b45264f45725c86133dc80a7cbe30b6b78c75620", + "sha256:ad7963f261988ee0883816b6b9f206f11461c9b3cb5cfbca0c9ab5adc406d395", + "sha256:af0451e23016631a2f52925a10d738ac4a0f794ac315c30380b22efc0c90cbc6", + "sha256:c16201060c5a3f8742e3deae759014251ac92f382f82bc2a41dc079ff18c3f24", + "sha256:c43b202f65891861a9a336984a103de25de235f756de69e32db893156f767013", + "sha256:c675c6cce4295cb1a692f3de7416aacace7314e064b94bc86e93aceefce7fd3e", + "sha256:d17cec0b992b1434f5f9df9986563605a4d1b1acd5574c87fc2ac014bcbd3316", + "sha256:dc91f6129953861a73d9a65c52a8dd682b561a9ebaf65283541645cab6489917", + "sha256:e2f4cbd1760d2bf2f30e396c2301999aab0191aec031a6a8a04950b2f575a536", + "sha256:f192e6d3556714105c10486bbd6d045e38a0c04d9da3cef21e0a8dfd8e162df4", + "sha256:f775b07026af2b1b0b5a8b05e41571cdcf3a315a67df265d60af301656a5425b", + "sha256:f969ec7f56ba9636679e69ca07fba548312ccaca37412ee823c7f413541ad7e0", + "sha256:f9dc52cd70907aafb99a773b66b156f2f995c7a0d284397c487c8b71ddbef2f9", + "sha256:f9ee88bb52352588ceb811d045b5c9bb1dc38927bc150fd156244f60ff3f59f1", + "sha256:fc7212e36ebeb81aebf7949c92897b622490d7c0e333a479c0395591e7994600" + ], + "version": "==1.0.7" + }, "click": { "hashes": [ - "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", - "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], - "version": "==7.1.1" + "version": "==7.1.2" }, "dash": { "hashes": [ - "sha256:6a488d380f02ba7c8e624a7c4d89f9e39368b8d9ed4a6f463364894a237f1fd8" + "sha256:57e32410a4f0f9fef0cdcd61f801bd01738f343ad308fa51592859ef9bc5c5ad" ], "index": "pypi", - "version": "==1.9.1" + "version": "==1.11.0" }, "dash-core-components": { "hashes": [ - "sha256:9a8901b67d3a1038fb79153c17234d5713c2d7cca0382718c6376c786b280e63" + "sha256:ea72f514d841caa1b42a1c6092d00c05cc63768916d33563e994114c1a022656" ], - "version": "==1.8.1" + "version": "==1.9.1" }, "dash-html-components": { "hashes": [ - "sha256:8992097a044f5add21f86ccbe0caef8c499394a04d5f2b6cc4458a42f37cca98" + "sha256:dafb54ae8ab601fffe50c74d72b32783dec2beea65fd1c7e7dd6a66e20e545ba" ], - "version": "==1.0.2" + "version": "==1.0.3" }, "dash-renderer": { "hashes": [ - "sha256:74aab93b6332290d9bff5f81060a5c4cbf0c393f8ccff85b2edb1ae976bdd5f0" + "sha256:c56e57f0b55ff0d7bb0f4df71060879460b2d22387fe01394fa84f85c71f66ed" ], - "version": "==1.2.4" + "version": "==1.4.0" }, "dash-table": { "hashes": [ - "sha256:64905af3214b509a95769ba9dfc884677696c270a572ed06b52c57fdcf9d9c77" + "sha256:5ceabd434bc63a9d1aa5bf4f00b55d2e5342072e274f0ee6fc53bc58bef4c1c2" ], - "version": "==4.6.1" + "version": "==4.6.2" }, "flask": { "hashes": [ - "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", - "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" + "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060", + "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557" ], - "version": "==1.1.1" + "version": "==1.1.2" }, "flask-compress": { "hashes": [ - "sha256:468693f4ddd11ac6a41bca4eb5f94b071b763256d54136f77957cfee635badb3" + "sha256:f367b2b46003dd62be34f7fb1379938032656dca56377a9bc90e7188e4289a7c" ], - "version": "==1.4.0" + "version": "==1.5.0" }, "future": { "hashes": [ @@ -90,10 +128,10 @@ }, "jinja2": { "hashes": [ - "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250", - "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49" + "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", + "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" ], - "version": "==2.11.1" + "version": "==2.11.2" }, "markupsafe": { "hashes": [ @@ -135,29 +173,29 @@ }, "numpy": { "hashes": [ - "sha256:1598a6de323508cfeed6b7cd6c4efb43324f4692e20d1f76e1feec7f59013448", - "sha256:1b0ece94018ae21163d1f651b527156e1f03943b986188dd81bc7e066eae9d1c", - "sha256:2e40be731ad618cb4974d5ba60d373cdf4f1b8dcbf1dcf4d9dff5e212baf69c5", - "sha256:4ba59db1fcc27ea31368af524dcf874d9277f21fd2e1f7f1e2e0c75ee61419ed", - "sha256:59ca9c6592da581a03d42cc4e270732552243dc45e87248aa8d636d53812f6a5", - "sha256:5e0feb76849ca3e83dd396254e47c7dba65b3fa9ed3df67c2556293ae3e16de3", - "sha256:6d205249a0293e62bbb3898c4c2e1ff8a22f98375a34775a259a0523111a8f6c", - "sha256:6fcc5a3990e269f86d388f165a089259893851437b904f422d301cdce4ff25c8", - "sha256:82847f2765835c8e5308f136bc34018d09b49037ec23ecc42b246424c767056b", - "sha256:87902e5c03355335fc5992a74ba0247a70d937f326d852fc613b7f53516c0963", - "sha256:9ab21d1cb156a620d3999dd92f7d1c86824c622873841d6b080ca5495fa10fef", - "sha256:a1baa1dc8ecd88fb2d2a651671a84b9938461e8a8eed13e2f0a812a94084d1fa", - "sha256:a244f7af80dacf21054386539699ce29bcc64796ed9850c99a34b41305630286", - "sha256:a35af656a7ba1d3decdd4fae5322b87277de8ac98b7d9da657d9e212ece76a61", - "sha256:b1fe1a6f3a6f355f6c29789b5927f8bd4f134a4bd9a781099a7c4f66af8850f5", - "sha256:b5ad0adb51b2dee7d0ee75a69e9871e2ddfb061c73ea8bc439376298141f77f5", - "sha256:ba3c7a2814ec8a176bb71f91478293d633c08582119e713a0c5351c0f77698da", - "sha256:cd77d58fb2acf57c1d1ee2835567cd70e6f1835e32090538f17f8a3a99e5e34b", - "sha256:cdb3a70285e8220875e4d2bc394e49b4988bdb1298ffa4e0bd81b2f613be397c", - "sha256:deb529c40c3f1e38d53d5ae6cd077c21f1d49e13afc7936f7f868455e16b64a0", - "sha256:e7894793e6e8540dbeac77c87b489e331947813511108ae097f1715c018b8f3d" - ], - "version": "==1.18.2" + "sha256:00d7b54c025601e28f468953d065b9b121ddca7fff30bed7be082d3656dd798d", + "sha256:02ec9582808c4e48be4e93cd629c855e644882faf704bc2bd6bbf58c08a2a897", + "sha256:0e6f72f7bb08f2f350ed4408bb7acdc0daba637e73bce9f5ea2b207039f3af88", + "sha256:1be2e96314a66f5f1ce7764274327fd4fb9da58584eaff00b5a5221edefee7d6", + "sha256:2466fbcf23711ebc5daa61d28ced319a6159b260a18839993d871096d66b93f7", + "sha256:2b573fcf6f9863ce746e4ad00ac18a948978bb3781cffa4305134d31801f3e26", + "sha256:3f0dae97e1126f529ebb66f3c63514a0f72a177b90d56e4bce8a0b5def34627a", + "sha256:50fb72bcbc2cf11e066579cb53c4ca8ac0227abb512b6cbc1faa02d1595a2a5d", + "sha256:57aea170fb23b1fd54fa537359d90d383d9bf5937ee54ae8045a723caa5e0961", + "sha256:709c2999b6bd36cdaf85cf888d8512da7433529f14a3689d6e37ab5242e7add5", + "sha256:7d59f21e43bbfd9a10953a7e26b35b6849d888fc5a331fa84a2d9c37bd9fe2a2", + "sha256:904b513ab8fbcbdb062bed1ce2f794ab20208a1b01ce9bd90776c6c7e7257032", + "sha256:96dd36f5cdde152fd6977d1bbc0f0561bccffecfde63cd397c8e6033eb66baba", + "sha256:9933b81fecbe935e6a7dc89cbd2b99fea1bf362f2790daf9422a7bb1dc3c3085", + "sha256:bbcc85aaf4cd84ba057decaead058f43191cc0e30d6bc5d44fe336dc3d3f4509", + "sha256:dccd380d8e025c867ddcb2f84b439722cf1f23f3a319381eac45fd077dee7170", + "sha256:e22cd0f72fc931d6abc69dc7764484ee20c6a60b0d0fee9ce0426029b1c1bdae", + "sha256:ed722aefb0ebffd10b32e67f48e8ac4c5c4cf5d3a785024fdf0e9eb17529cd9d", + "sha256:efb7ac5572c9a57159cf92c508aad9f856f1cb8e8302d7fdb99061dbe52d712c", + "sha256:efdba339fffb0e80fcc19524e4fdbda2e2b5772ea46720c44eaac28096d60720", + "sha256:f22273dd6a403ed870207b853a856ff6327d5cbce7a835dfa0645b3fc00273ec" + ], + "version": "==1.18.4" }, "pandas": { "hashes": [ @@ -183,11 +221,11 @@ }, "plotly": { "hashes": [ - "sha256:b6b1e13e7f04dc9a9f714ab8ecc63d00a4207a736c9835912c96b6884002be1e", - "sha256:e6eab2b6010921f5fb998860125b90748e581de66ebc6107b686829417d98fc4" + "sha256:61f34955f04201a1ebcd59feaafa7eae7c16ef9b3f439870be01fb85a949292f", + "sha256:ac0ca0854350bfcd833f3a8eb08aa50e184660502bb46fe907701f896ff349bd" ], "index": "pypi", - "version": "==4.5.4" + "version": "==4.6.0" }, "python-dateutil": { "hashes": [ @@ -198,10 +236,10 @@ }, "pytz": { "hashes": [ - "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", - "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" + "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", + "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" ], - "version": "==2019.3" + "version": "==2020.1" }, "retrying": { "hashes": [ @@ -218,27 +256,27 @@ }, "werkzeug": { "hashes": [ - "sha256:169ba8a33788476292d04186ab33b01d6add475033dfc07215e6d219cc077096", - "sha256:6dc65cf9091cf750012f56f2cad759fa9e879f511b5ff8685e456b4e3bf90d16" + "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", + "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c" ], - "version": "==1.0.0" + "version": "==1.0.1" } }, "develop": { "astroid": { "hashes": [ - "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", - "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" + "sha256:29fa5d46a2404d01c834fcb802a3943685f1fc538eb2a02a161349f5505ac196", + "sha256:2fecea42b20abb1922ed65c7b5be27edfba97211b04b2b6abc6a43549a024ea6" ], - "version": "==2.3.3" + "version": "==2.4.0" }, "atomicwrites": { "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197", + "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a" ], "markers": "sys_platform == 'win32'", - "version": "==1.3.0" + "version": "==1.4.0" }, "attrs": { "hashes": [ @@ -247,14 +285,6 @@ ], "version": "==19.3.0" }, - "bandit": { - "hashes": [ - "sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952", - "sha256:41e75315853507aa145d62a78a2a6c5e3240fe14ee7c601459d0df9418196065" - ], - "index": "pypi", - "version": "==1.6.2" - }, "colorama": { "hashes": [ "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", @@ -278,27 +308,13 @@ "index": "pypi", "version": "==3.7.9" }, - "gitdb": { - "hashes": [ - "sha256:284a6a4554f954d6e737cddcff946404393e030b76a282c6640df8efd6b3da5e", - "sha256:598e0096bb3175a0aab3a0b5aedaa18a9a25c6707e0eca0695ba1a0baf1b2150" - ], - "version": "==4.0.2" - }, - "gitpython": { - "hashes": [ - "sha256:43da89427bdf18bf07f1164c6d415750693b4d50e28fc9b68de706245147b9dd", - "sha256:e426c3b587bd58c482f0b7fe6145ff4ac7ae6c82673fc656f489719abca6f4cb" - ], - "version": "==3.1.0" - }, "importlib-metadata": { "hashes": [ - "sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302", - "sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b" + "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f", + "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e" ], "markers": "python_version < '3.8'", - "version": "==1.5.0" + "version": "==1.6.0" }, "isort": { "hashes": [ @@ -354,13 +370,6 @@ ], "version": "==20.3" }, - "pbr": { - "hashes": [ - "sha256:139d2625547dbfa5fb0b81daebb39601c478c21956dc57e2e07b74450a8c506b", - "sha256:61aa52a0f18b71c5cc58232d2cf8f8d09cd67fcad60b742a60124cb8d6951488" - ], - "version": "==5.4.4" - }, "pluggy": { "hashes": [ "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", @@ -391,18 +400,18 @@ }, "pylint": { "hashes": [ - "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", - "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" + "sha256:588e114e3f9a1630428c35b7dd1c82c1c93e1b0e78ee312ae4724c5e1a1e0245", + "sha256:bd556ba95a4cf55a1fc0004c00cf4560b1e70598a54a74c6904d933c8f3bd5a8" ], "index": "pypi", - "version": "==2.4.4" + "version": "==2.5.0" }, "pyparsing": { "hashes": [ - "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", - "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "version": "==2.4.6" + "version": "==2.4.7" }, "pytest": { "hashes": [ @@ -412,22 +421,6 @@ "index": "pypi", "version": "==5.4.1" }, - "pyyaml": { - "hashes": [ - "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", - "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", - "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", - "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", - "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", - "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", - "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", - "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", - "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", - "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", - "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" - ], - "version": "==5.3.1" - }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -435,19 +428,12 @@ ], "version": "==1.14.0" }, - "smmap": { - "hashes": [ - "sha256:171484fe62793e3626c8b05dd752eb2ca01854b0c55a1efc0dc4210fccb65446", - "sha256:5fead614cf2de17ee0707a8c6a5f2aa5a2fc6c698c70993ba42f515485ffda78" - ], - "version": "==3.0.1" - }, - "stevedore": { + "toml": { "hashes": [ - "sha256:18afaf1d623af5950cc0f7e75e70f917784c73b652a34a12d90b309451b5500b", - "sha256:a4e7dc759fb0f2e3e2f7d8ffe2358c19d45b9b8297f393ef1256858d82f69c9b" + "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" ], - "version": "==1.32.0" + "version": "==0.10.0" }, "typed-ast": { "hashes": [ @@ -485,9 +471,9 @@ }, "wrapt": { "hashes": [ - "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" ], - "version": "==1.11.2" + "version": "==1.12.1" }, "zipp": { "hashes": [ diff --git a/Readme.md b/Readme.md index 91adad4..36ac955 100644 --- a/Readme.md +++ b/Readme.md @@ -1,9 +1,10 @@ -# PyStalk +# MetaStalk -[![GitHub](https://img.shields.io/github/license/Cyb3r-Jak3/pystalk?style=flat)](https://github.com/Cyb3r-Jak3/PyStalk/blob/master/LICENSE) ![Gitlab pipeline status (branch)](https://img.shields.io/gitlab/pipeline/Cyb3r-Jak3/pystalk/master?label=Build&style=flat) +[![GitHub](https://img.shields.io/github/license/Cyb3r-Jak3/pystalk?style=flat)](https://github.com/Cyb3r-Jak3/PyStalk/blob/master/LICENSE) ![Gitlab pipeline status (branch)](https://img.shields.io/gitlab/pipeline/Cyb3r-Jak3/metastalk/master?label=Build&style=flat) [![Test Coverage](https://api.codeclimate.com/v1/badges/896b338971314c13a56e/test_coverage)](https://codeclimate.com/github/Cyb3r-Jak3/PyStalk/test_coverage) [![Maintainability](https://api.codeclimate.com/v1/badges/896b338971314c13a56e/maintainability)](https://codeclimate.com/github/Cyb3r-Jak3/PyStalk/maintainability) -[![CodeFactor](https://www.codefactor.io/repository/github/cyb3r-jak3/pystalk/badge)](https://www.codefactor.io/repository/github/cyb3r-jak3/pystalk) [![codecov](https://codecov.io/gl/Cyb3r-Jak3/pystalk/branch/master/graph/badge.svg)](https://codecov.io/gl/Cyb3r-Jak3/pystalk) +[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/Cyb3r-Jak3/PyStalk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/Cyb3r-Jak3/PyStalk/?branch=master) +[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/Cyb3r-Jak3/PyStalk.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Cyb3r-Jak3/PyStalk/context:python) ## About @@ -18,14 +19,16 @@ Examples photos from [ianare/exif-samples](https://github.com/ianare/exif-sample All development is done on GitLab and pushed to GitHub. Please read [contributing.md](CONTRIBUTING.md) for development. +Python 3.6 and up. + ## How to use ```bash -git clone https://github.com/Cyb3r-Jak3/pystalk -cd pystalk +git clone https://gitlab.com/Cyb3r-Jak3/metastalk +cd metastalk pip install -r requirements.txt -python PyStalk.py -#i.e. python PyStalk.py tests/ExamplePhotos/ +python ./MetaStalk/main.py +#i.e. python ./MetaStalk/main.py ./ExamplePhotos/ ``` This project also use Pipfile and Pipfile.lock if you would rather pipenv over requirements.txt diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..90783c6 --- /dev/null +++ b/setup.py @@ -0,0 +1,45 @@ +"""Sets up PyStalk to be installed""" +import os +from setuptools import setup, find_packages +from MetaStalk import __version__ + + +def read(fname): + """Reads README.md as long description""" + return open(os.path.join(os.path.dirname(__file__), fname)).read() + + +setup( + name="MetaStalk", + version=__version__, + author="Cyb3r Jak3", + author_email="jake@jwhite.network", + install_requires=[ + "hachoir >= 3.1.1", + "plotly >= 4.6.0", + "pandas >= 1.0.3", + "dash >= 1.11.0"], + description="Metadata analyzer", + license="MPL 2.0", + python_requires=">=3.6", + classifiers=[ + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + "Operating System :: OS Independent", + "Natural Language :: English", + "Programming Language :: Python :: 3 :: Only", + "Development Status :: 1 - Planning" + ], + packages=find_packages(), + long_description=read('README.md'), + long_description_content_type='text/markdown', + entry_points={ + "console_scripts": ["metastalk=MetaStalk.main:run"] + }, + url="https://gitlab.com/Cyb3r-Jak3/MetaStalk", + project_urls={ + "Issues": "https://gitlab.com/Cyb3r-Jak3/MetaStalk/issues", + "Source": "https://gitlab.com/Cyb3r-Jak3/MetaStalk/-/tree/master", + "CI": "https://gitlab.com/Cyb3r-Jak3/MetaStalk/pipelines" + }, + +) From 5730a183a51a0cecae4494f4a6c5312afc54d526 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 12:27:51 -0400 Subject: [PATCH 02/17] Fixed issue with utils and modules import during build --- .gitlab-ci.yml | 2 +- MetaStalk/main.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6555291..3bbf6d2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,7 @@ Coverage: - curl https://deepsource.io/cli | sh - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; chmod +x ./cc-test-reporter; ./cc-test-reporter before-build script: - - pipenv install --dev + - pipenv install --dev --skip-lock - pipenv install coverage - pipenv run coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/ -t -d - pipenv run coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d diff --git a/MetaStalk/main.py b/MetaStalk/main.py index 36af8ca..66b2456 100644 --- a/MetaStalk/main.py +++ b/MetaStalk/main.py @@ -8,8 +8,8 @@ from hachoir.parser import createParser from hachoir.metadata import extractMetadata -import utils -import modules +import utils # pylint: disable=import-error +import modules # pylint: disable=import-error t_start = timeit.default_timer() From 3d89eedc7cc7259dec1623814d6246cd065ce177 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 12:48:08 -0400 Subject: [PATCH 03/17] Dropped support for pipenv during pipeline --- .gitlab-ci.yml | 29 ++++++++++++++++------------- .pylintrc | 15 --------------- MetaStalk/main.py | 9 ++++++--- requirements-dev.txt | 1 - tox.ini | 11 +++++++++++ 5 files changed, 33 insertions(+), 32 deletions(-) delete mode 100644 .pylintrc create mode 100644 tox.ini diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3bbf6d2..28af558 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,12 @@ include: - template: License-Scanning.gitlab-ci.yml +license_scanning: + only: + refs: + - tags + - master + Coverage: image: python:3.7 stage: test @@ -19,18 +25,16 @@ Coverage: - htmlcov/* - bandit.json before_script: - - pip install -U pip pipenv + - pip install -U pip coverage -r requirements.txt - curl https://deepsource.io/cli | sh - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; chmod +x ./cc-test-reporter; ./cc-test-reporter before-build script: - - pipenv install --dev --skip-lock - - pipenv install coverage - - pipenv run coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/ -t -d - - pipenv run coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d + - coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/ -t -d + - coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d after_script: - - pipenv run coverage report - - pipenv run coverage html - - pipenv run coverage xml + - coverage report + - coverage html + - coverage xml - ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml - ./cc-test-reporter after-build - bash <(curl -s https://codecov.io/bash) @@ -38,12 +42,11 @@ Coverage: .check: stage: Code_Checking before_script: - - pip install -U pip pipenv + - pip install -U pip -r requirements.txt -r requirements-dev.txt script: - - pipenv install --dev --skip-lock - - pipenv run pylint --ignored-classes=_socketobject ./MetaStalk - - pipenv run flake8 ./MetaStalk --statistics --show-source - - time pipenv run python3 ./MetaStalk/main.py ./ExamplePhotos/ -d -t + - pylint --rcfile=tox.ini ./MetaStalk + - flake8 ./MetaStalk --statistics --show-source + - time python3 ./MetaStalk/main.py ./ExamplePhotos/ -d -t python-3.6: extends: ".check" diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 562f4f2..0000000 --- a/.pylintrc +++ /dev/null @@ -1,15 +0,0 @@ -[LOGGING] - -# Format style used to check logging format string. `old` means using % -# formatting, while `new` is for `{}` formatting. -logging-format-style=new - -[MESSAGES CONTROL] -disable=invalid-name, logging-too-many-args - -#[MASTER] - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -# Ignore all .py files under the 3rdparty subdirectory. -#ignore-patterns=**/utils/**/blank.py \ No newline at end of file diff --git a/MetaStalk/main.py b/MetaStalk/main.py index 66b2456..3e8cd24 100644 --- a/MetaStalk/main.py +++ b/MetaStalk/main.py @@ -5,11 +5,14 @@ import os import logging import timeit +import sys from hachoir.parser import createParser from hachoir.metadata import extractMetadata -import utils # pylint: disable=import-error -import modules # pylint: disable=import-error + +sys.path.append(".") +import MetaStalk.utils as utils +import MetaStalk.modules as modules t_start = timeit.default_timer() @@ -61,7 +64,7 @@ def run(args, log: logging.Logger): "Manufacturer": modules.PieChart(photos, "Camera manufacturer", log), "Focal": modules.PieChart(photos, "Camera focal", log), "Producer": modules.PieChart(photos, "Producer", log) - } + } utils.graph(plots, log, t_start, args.test) diff --git a/requirements-dev.txt b/requirements-dev.txt index 406cafd..65e5732 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,3 @@ pytest -bandit pylint flake8 \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..013f79e --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, while `new` is for `{}` formatting. +logging-format-style=new + +[MESSAGES CONTROL] +disable=invalid-name, logging-too-many-args, wrong-import-position + +[flake8] +ignore=E402 \ No newline at end of file From 1af32c9b015e5c5cdaa8264609445abebd721877 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 12:51:50 -0400 Subject: [PATCH 04/17] Fixed bracket error --- .gitlab-ci.yml | 6 +++--- MetaStalk/modules/DateTime.py | 4 ++-- MetaStalk/utils/web.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28af558..bbdc892 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,7 @@ Coverage: - htmlcov/* - bandit.json before_script: - - pip install -U pip coverage -r requirements.txt + - pip install -U pip coverage -r requirements.txt --quiet - curl https://deepsource.io/cli | sh - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; chmod +x ./cc-test-reporter; ./cc-test-reporter before-build script: @@ -42,7 +42,7 @@ Coverage: .check: stage: Code_Checking before_script: - - pip install -U pip -r requirements.txt -r requirements-dev.txt + - pip install -U pip -r requirements.txt -r requirements-dev.txt --quiet script: - pylint --rcfile=tox.ini ./MetaStalk - flake8 ./MetaStalk --statistics --show-source @@ -95,7 +95,7 @@ python-3.8-build: .test: stage: Test_Build before_script: - - pip install -U pip + - pip install -U pip --quiet script: - pip install dist/*.whl - pip show -f metastalk diff --git a/MetaStalk/modules/DateTime.py b/MetaStalk/modules/DateTime.py index 06b1914..6da5c28 100644 --- a/MetaStalk/modules/DateTime.py +++ b/MetaStalk/modules/DateTime.py @@ -30,9 +30,9 @@ def date_time(photos, log): "Creation date", "Date time Original", "Date time Digitized" - ]), + ]), cells=dict(values=[simple_photos, datetime, datetime_original, datetime_digitized]))] - ) + ) return fig diff --git a/MetaStalk/utils/web.py b/MetaStalk/utils/web.py index 66e9783..6e8bf47 100644 --- a/MetaStalk/utils/web.py +++ b/MetaStalk/utils/web.py @@ -22,7 +22,7 @@ def graph(plots: dict, log, t_start: float, test=False): html.Div(children=graphs), html.P("Time Taken = {0:.2f} seconds".format(t_stop - t_start), style={"textAlign": "center"}) - ]) + ]) if not test: webbrowser.open("http://localhost:8052", new=2) app.run_server(port=8052) From 329b78939a0769145478cbf4105371c1f886f863 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 13:04:10 -0400 Subject: [PATCH 05/17] Rename Readme.md to README.md --- Readme.md => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Readme.md => README.md (100%) diff --git a/Readme.md b/README.md similarity index 100% rename from Readme.md rename to README.md From e914a3e59ce68c3945028eaa3a4837d68482ff9a Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 13:16:53 -0400 Subject: [PATCH 06/17] Added Manifest file to include utils/assets --- .gitlab-ci.yml | 1 - Manifest.in | 2 ++ MetaStalk/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Manifest.in diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bbdc892..c10d855 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,6 @@ Coverage: paths: - MetaStalk.log - htmlcov/* - - bandit.json before_script: - pip install -U pip coverage -r requirements.txt --quiet - curl https://deepsource.io/cli | sh diff --git a/Manifest.in b/Manifest.in new file mode 100644 index 0000000..024f685 --- /dev/null +++ b/Manifest.in @@ -0,0 +1,2 @@ +graft MetaStalk/utils/assets +prune MetaStalk/tests \ No newline at end of file diff --git a/MetaStalk/__init__.py b/MetaStalk/__init__.py index 99ca0f2..e6747d8 100644 --- a/MetaStalk/__init__.py +++ b/MetaStalk/__init__.py @@ -1,3 +1,3 @@ """PyStalk """ -__version__ = "v1.4.0" +__version__ = "v2.0.0" diff --git a/setup.py b/setup.py index 90783c6..cc0093d 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ def read(fname): long_description=read('README.md'), long_description_content_type='text/markdown', entry_points={ - "console_scripts": ["metastalk=MetaStalk.main:run"] + "console_scripts": ["metastalk=MetaStalk.main:start"] }, url="https://gitlab.com/Cyb3r-Jak3/MetaStalk", project_urls={ From 89ad8670bbe399345c870b711f25085e45696910 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 13:42:32 -0400 Subject: [PATCH 07/17] Trying with relative path --- MetaStalk/__init__.py | 2 +- MetaStalk/main.py | 10 +++++----- tox.ini | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/MetaStalk/__init__.py b/MetaStalk/__init__.py index e6747d8..81b9960 100644 --- a/MetaStalk/__init__.py +++ b/MetaStalk/__init__.py @@ -1,3 +1,3 @@ -"""PyStalk +"""MetaStalk """ __version__ = "v2.0.0" diff --git a/MetaStalk/main.py b/MetaStalk/main.py index 3e8cd24..323331c 100644 --- a/MetaStalk/main.py +++ b/MetaStalk/main.py @@ -5,21 +5,21 @@ import os import logging import timeit -import sys +#import sys from hachoir.parser import createParser from hachoir.metadata import extractMetadata -sys.path.append(".") -import MetaStalk.utils as utils -import MetaStalk.modules as modules +#sys.path.append(".") +import utils +import modules t_start = timeit.default_timer() def start(): - """ Sets up PyStalk and parses arguments""" + """ Sets up MetaStalk and parses arguments""" parser = argparse.ArgumentParser(prog="MetaStalk", description="Tool to graph " "image metadata.") diff --git a/tox.ini b/tox.ini index 013f79e..3ff701e 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ logging-format-style=new [MESSAGES CONTROL] -disable=invalid-name, logging-too-many-args, wrong-import-position +disable=invalid-name, logging-too-many-args, wrong-import-position, import-error [flake8] ignore=E402 \ No newline at end of file From 958e12aacd533c62fe5b7582e0af02b67a842c50 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 13:47:03 -0400 Subject: [PATCH 08/17] Fixed issue with flake8 --- MetaStalk/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MetaStalk/main.py b/MetaStalk/main.py index 323331c..6b0cac4 100644 --- a/MetaStalk/main.py +++ b/MetaStalk/main.py @@ -5,12 +5,12 @@ import os import logging import timeit -#import sys +# import sys from hachoir.parser import createParser from hachoir.metadata import extractMetadata -#sys.path.append(".") +# sys.path.append(".") import utils import modules From 2eb8a2bdb6dd2460b3ee67f6fff229209a056e07 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 14:55:41 -0400 Subject: [PATCH 09/17] Added each module not need logger passed to it. --- MetaStalk/main.py | 26 +++++++++++++------------- MetaStalk/modules/DateTime.py | 5 ++++- MetaStalk/modules/GPSCheck.py | 5 ++++- MetaStalk/modules/PieChart.py | 5 ++++- MetaStalk/modules/Stats.py | 9 ++++++--- MetaStalk/utils/web.py | 6 +++++- setup.py | 3 ++- 7 files changed, 38 insertions(+), 21 deletions(-) diff --git a/MetaStalk/main.py b/MetaStalk/main.py index 6b0cac4..20b9e69 100644 --- a/MetaStalk/main.py +++ b/MetaStalk/main.py @@ -5,14 +5,14 @@ import os import logging import timeit -# import sys +import sys from hachoir.parser import createParser from hachoir.metadata import extractMetadata -# sys.path.append(".") -import utils -import modules +sys.path.append(".") +import MetaStalk.utils as utils +import MetaStalk.modules as modules t_start = timeit.default_timer() @@ -37,7 +37,7 @@ def start(): args = parser.parse_args() log = utils.make_logger("MetaStalk", args.loglevel) - log.info("Starting up") + log.info("MetaStalk starting") if not args.files: log.error("ERROR: No path was inputted.") raise IOError("No path was inputted.") @@ -57,16 +57,16 @@ def run(args, log: logging.Logger): photos, invalid_photos = file_search(args.files, log) plots = { - "STATS": modules.Stats(photos, invalid_photos, log), - "GPS": modules.GPS_Check(photos, log), - "Timestamp": modules.date_time(photos, log), - "Model": modules.PieChart(photos, "Camera model", log), - "Manufacturer": modules.PieChart(photos, "Camera manufacturer", log), - "Focal": modules.PieChart(photos, "Camera focal", log), - "Producer": modules.PieChart(photos, "Producer", log) + "STATS": modules.Stats(photos, invalid_photos), + "GPS": modules.GPS_Check(photos), + "Timestamp": modules.date_time(photos), + "Model": modules.PieChart(photos, "Camera model"), + "Manufacturer": modules.PieChart(photos, "Camera manufacturer"), + "Focal": modules.PieChart(photos, "Camera focal"), + "Producer": modules.PieChart(photos, "Producer") } - utils.graph(plots, log, t_start, args.test) + utils.graph(plots, t_start, args.test) def directory_search(files: list, log: logging.Logger): diff --git a/MetaStalk/modules/DateTime.py b/MetaStalk/modules/DateTime.py index 6da5c28..d2017c1 100644 --- a/MetaStalk/modules/DateTime.py +++ b/MetaStalk/modules/DateTime.py @@ -1,8 +1,11 @@ """Makes a table that plots gps timestamp""" +import logging import plotly.graph_objects as go +log = logging.getLogger("MetaStalk") -def date_time(photos, log): + +def date_time(photos): """Makes a table with gps timestamp of photos""" log.info("Starting DateTime Charts") datetime = [] diff --git a/MetaStalk/modules/GPSCheck.py b/MetaStalk/modules/GPSCheck.py index a33c042..5ce4da4 100644 --- a/MetaStalk/modules/GPSCheck.py +++ b/MetaStalk/modules/GPSCheck.py @@ -1,8 +1,11 @@ """Makes geo chart with plots of gps data""" +import logging import plotly.express as px +log = logging.getLogger("MetaStalk") -def GPS_Check(photos: list, log): + +def GPS_Check(photos: list): """Takes a list of photos and creates a geo plot of them""" log.info("Starting GPS Chart") lats = [] diff --git a/MetaStalk/modules/PieChart.py b/MetaStalk/modules/PieChart.py index 01f364c..4981e65 100644 --- a/MetaStalk/modules/PieChart.py +++ b/MetaStalk/modules/PieChart.py @@ -1,6 +1,9 @@ """Module that makes a pie chart.""" +import logging import plotly.graph_objects as go +log = logging.getLogger("MetaStalk") + def create_chart(table, pietype): """Creates the pie chart""" @@ -25,7 +28,7 @@ def create_chart(table, pietype): return fig -def PieChart(photos: list, pietype: str, log): +def PieChart(photos: list, pietype: str): """Gets information and makes a pie chart""" log.info("Staring %s Chart", pietype) table = [] diff --git a/MetaStalk/modules/Stats.py b/MetaStalk/modules/Stats.py index b683df8..cc398be 100644 --- a/MetaStalk/modules/Stats.py +++ b/MetaStalk/modules/Stats.py @@ -1,13 +1,16 @@ """Creates a table figure that shows photos that did and did not have exif""" +import logging import plotly.graph_objects as go +log = logging.getLogger("MetaStalk") -def Stats(photos: list, invalid: list, log): + +def Stats(photos: list, invalid: list): """Creates the table of photos""" log.info("Staring Stats") - log.debug("There are {} photos with metadata and {} without" - .format(len(photos), len(invalid))) + log.debug("There are %s photos with metadata and %s without", + len(photos), len(invalid)) simple_photos = [] for i, _ in enumerate(photos): simple_photos.append(photos[i]["item"]) diff --git a/MetaStalk/utils/web.py b/MetaStalk/utils/web.py index 6e8bf47..772026a 100644 --- a/MetaStalk/utils/web.py +++ b/MetaStalk/utils/web.py @@ -1,12 +1,16 @@ """Uses dash to create a webpage that contain all the graphs""" import timeit +import logging import webbrowser import dash import dash_html_components as html import dash_core_components as dcc -def graph(plots: dict, log, t_start: float, test=False): +log = logging.getLogger("MetaStalk") + + +def graph(plots: dict, t_start: float, test=False): """Creates the graphs""" external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css'] app = dash.Dash(__name__, external_stylesheets=external_stylesheets) diff --git a/setup.py b/setup.py index cc0093d..8c8df20 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,8 @@ def read(fname): "Operating System :: OS Independent", "Natural Language :: English", "Programming Language :: Python :: 3 :: Only", - "Development Status :: 1 - Planning" + "Development Status :: 4 - Beta", + "Environment :: Console" ], packages=find_packages(), long_description=read('README.md'), From 318d88714402d35e1daa43a955f6a1e5a901f87c Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 16:14:06 -0400 Subject: [PATCH 10/17] Removing ability to run without installing. May return in the future --- .gitlab-ci.yml | 28 ++++++++++++++++------------ MetaStalk/main.py | 5 +---- tox.ini | 3 --- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c10d855..2f13218 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ stages: -- Code_Checking +- lint - test - Test_Build @@ -8,6 +8,11 @@ include: - template: Dependency-Scanning.gitlab-ci.yml - template: License-Scanning.gitlab-ci.yml +dependency_scanning: + only: + refs: + - tags + - master license_scanning: only: @@ -24,7 +29,7 @@ Coverage: - MetaStalk.log - htmlcov/* before_script: - - pip install -U pip coverage -r requirements.txt --quiet + - pip install -U pip coverage scrutinizer-ocular -r requirements.txt --quiet - curl https://deepsource.io/cli | sh - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; chmod +x ./cc-test-reporter; ./cc-test-reporter before-build script: @@ -36,27 +41,26 @@ Coverage: - coverage xml - ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml - ./cc-test-reporter after-build - - bash <(curl -s https://codecov.io/bash) + - ocular --data-file ".coverage" --config-file ".coveragerc" -.check: - stage: Code_Checking +.lint: + stage: lint before_script: - pip install -U pip -r requirements.txt -r requirements-dev.txt --quiet script: - pylint --rcfile=tox.ini ./MetaStalk - flake8 ./MetaStalk --statistics --show-source - - time python3 ./MetaStalk/main.py ./ExamplePhotos/ -d -t -python-3.6: - extends: ".check" +python-3.6-lint: + extends: ".lint" image: python:3.6 -python-3.7: - extends: ".check" +python-3.7-lint: + extends: ".lint" image: python:3.7 -python-3.8: - extends: ".check" +python-3.8-lint: + extends: ".lint" image: python:3.8 diff --git a/MetaStalk/main.py b/MetaStalk/main.py index 20b9e69..eebfe99 100644 --- a/MetaStalk/main.py +++ b/MetaStalk/main.py @@ -5,12 +5,10 @@ import os import logging import timeit -import sys from hachoir.parser import createParser from hachoir.metadata import extractMetadata -sys.path.append(".") import MetaStalk.utils as utils import MetaStalk.modules as modules @@ -104,5 +102,4 @@ def file_search(files: list, log: logging.Logger): return valid, invalid -if __name__ == "__main__": - start() +start() diff --git a/tox.ini b/tox.ini index 3ff701e..64d8c4f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,4 @@ [LOGGING] - -# Format style used to check logging format string. `old` means using % -# formatting, while `new` is for `{}` formatting. logging-format-style=new [MESSAGES CONTROL] From f0e5c9e0e6e6c3cf964369c0bb2496bbfdf49625 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 16:19:41 -0400 Subject: [PATCH 11/17] Allowed coverage job to fail --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f13218..7b9a9ba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ Coverage: paths: - MetaStalk.log - htmlcov/* + allow_failure: true before_script: - pip install -U pip coverage scrutinizer-ocular -r requirements.txt --quiet - curl https://deepsource.io/cli | sh From 8fb1db1093e56ae6f19fd3770226579f02c65793 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 16:42:48 -0400 Subject: [PATCH 12/17] Documentation ready for merge --- .codeclimate.yml | 9 --------- .deepsource.toml | 3 ++- .gitlab-ci.yml | 29 +++++++++++++++++++++++------ CHANGELOG.md | 18 ++++++++++++++---- README.md | 26 +++++++++++++++++++------- 5 files changed, 58 insertions(+), 27 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index 7f20119..06781d5 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -13,15 +13,6 @@ plugins: enabled: false python:S3776: enabled: false - markdownlint: - enabled: true - checks: - MD024: - enabled: false - MD009: - enabled: false - MD013: - enabled: false pep8: enabled: true bandit: diff --git a/.deepsource.toml b/.deepsource.toml index af150da..ab0e37b 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -1,8 +1,9 @@ version = 1 test_patterns = [ - "PyStalk/**", + "MetaStalk/**", ] +exclude_patterns = ["MetaStalk/tests/*"] [[analyzers]] name = "python" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7b9a9ba..85b6a45 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ stages: - lint - test - Test_Build +- Deploy_to_PyPi include: - template: SAST.gitlab-ci.yml @@ -42,7 +43,7 @@ Coverage: - coverage xml - ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml - ./cc-test-reporter after-build - - ocular --data-file ".coverage" --config-file ".coveragerc" + - ocular --data-file ".coverage" .lint: stage: lint @@ -72,8 +73,6 @@ python-3.8-lint: script: - python setup.py sdist bdist_wheel - python -m twine check dist/* - except: - - tags python-3.6-build: extends: ".build" @@ -106,8 +105,6 @@ python-3.8-build: - command -v metastalk - metastalk ./ExamplePhotos/ -t -d - metastalk ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d - except: - - tags python-3.6-test: extends: ".test" @@ -120,4 +117,24 @@ python-3.7-test: python-3.8-test: extends: ".test" image: python:3.8 - \ No newline at end of file + + +Deploy_to_PyPi: + image: python:3.7 + stage: Deploy_to_PyPi + variables: + TWINE_USERNAME: $PRODUCTION_USERNAME + TWINE_PASSWORD: $PRODUCTION_PASSWORD + before_script: + - pip install -U pip setuptools twine --quiet + script: + - python setup.py sdist bdist_wheel + - python -m twine check dist/* + - python -m twine upload --verbose --username $PRODUCTION_USER --password $PRODUCTION_PASSWORD dist/* + only: + refs: + - tags + - master + variables: + - $CI_COMMIT_TAG + \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index dc789ee..a91f332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,13 @@ -## [N/A] - UNRELEASED +## [v2.0.0] - UNRELEASED -Rename to MetaStalk to create PyPi package and a lot backend changes. +## Rename to MetaStalk -## Added +Rename to MetaStalk to create PyPi package and a lot backend development changes. + +### Added - License scanning - [Codecov](https://codecov.io/gl/Cyb3r-Jak3/pystalk) @@ -20,6 +22,14 @@ Rename to MetaStalk to create PyPi package and a lot backend changes. - Name from PyStalk to MetaStalk as PyStalk is taken. - Moved all files to MetaStalk directory. - Moved ExamplePhotos to own directory. +- License scanning now only on master branch and tags. +- Can only run on from package. + +### Removed + +- Dropped support for Python 3.5 as hachoir does not support it. +- Dropped using pipenv for pipeline. +- Markdownlint check in codeclimate. ## [v1.4] - 2020-02-02 @@ -29,7 +39,7 @@ Rename to MetaStalk to create PyPi package and a lot backend changes. ### Removed -- Removed Flash Chart (library does not support it.) +- Removed Flash Chart (hachoir does not support it). ## [v1.3.3] - 2020-01-10 diff --git a/README.md b/README.md index 36ac955..eae701a 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # MetaStalk -[![GitHub](https://img.shields.io/github/license/Cyb3r-Jak3/pystalk?style=flat)](https://github.com/Cyb3r-Jak3/PyStalk/blob/master/LICENSE) ![Gitlab pipeline status (branch)](https://img.shields.io/gitlab/pipeline/Cyb3r-Jak3/metastalk/master?label=Build&style=flat) +[![GitHub](https://img.shields.io/github/license/Cyb3r-Jak3/MetaStalk?style=flat)](https://github.com/Cyb3r-Jak3/MetaStalk/blob/master/LICENSE) ![Gitlab pipeline status (branch)](https://img.shields.io/gitlab/pipeline/Cyb3r-Jak3/MetaStalk/master?label=Build&style=flat) [![Test Coverage](https://api.codeclimate.com/v1/badges/896b338971314c13a56e/test_coverage)](https://codeclimate.com/github/Cyb3r-Jak3/PyStalk/test_coverage) [![Maintainability](https://api.codeclimate.com/v1/badges/896b338971314c13a56e/maintainability)](https://codeclimate.com/github/Cyb3r-Jak3/PyStalk/maintainability) -[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/Cyb3r-Jak3/PyStalk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/Cyb3r-Jak3/PyStalk/?branch=master) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/Cyb3r-Jak3/PyStalk.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Cyb3r-Jak3/PyStalk/context:python) +[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/Cyb3r-Jak3/MetaStalk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/Cyb3r-Jak3/MetaStalk/?branch=master) +[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/Cyb3r-Jak3/MetaStalk.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Cyb3r-Jak3/MetaStalk/context:python) ## About -PyStalk is a tool that can be used to generate graphs from the metadata of JPEG, TIFF images. More formats are supported but untested. +MetaStalk is a tool that can be used to generate graphs from the metadata of JPEG, TIFF images. More formats are supported but untested. It currently creates graphs for: - GPS coordinates (map) @@ -23,12 +23,24 @@ Python 3.6 and up. ## How to use +MetaStalk is available as a package on pypi.org. + +### PyPi Install + +```bash +pip install metastalk +metastalk +#i.e. metastalk ./ExamplePhotos/ +``` + +### Source Install + ```bash git clone https://gitlab.com/Cyb3r-Jak3/metastalk cd metastalk -pip install -r requirements.txt -python ./MetaStalk/main.py -#i.e. python ./MetaStalk/main.py ./ExamplePhotos/ +setup.py install +metastalk +#i.e. metastalk ./ExamplePhotos/ ``` This project also use Pipfile and Pipfile.lock if you would rather pipenv over requirements.txt From 521e8fe4885996305362e7d5116cc92a9e58f6ee Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 16:49:43 -0400 Subject: [PATCH 13/17] Last attempt using ocular before not having scrutinizer see coverage --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 85b6a45..8506381 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,7 @@ Coverage: - coverage xml - ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml - ./cc-test-reporter after-build - - ocular --data-file ".coverage" + - ocular --data-file ".coverage" --repository Cyb3r-Jak3/MetaStalk/ .lint: stage: lint From 2f168e642475e80bc8e03c4f3292ca1ad91556e3 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 16:54:24 -0400 Subject: [PATCH 14/17] Dropped support for scrutinizer coverage --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8506381..7ad3591 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,7 +31,7 @@ Coverage: - htmlcov/* allow_failure: true before_script: - - pip install -U pip coverage scrutinizer-ocular -r requirements.txt --quiet + - pip install -U pip coverage -r requirements.txt --quiet - curl https://deepsource.io/cli | sh - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; chmod +x ./cc-test-reporter; ./cc-test-reporter before-build script: @@ -43,8 +43,6 @@ Coverage: - coverage xml - ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml - ./cc-test-reporter after-build - - ocular --data-file ".coverage" --repository Cyb3r-Jak3/MetaStalk/ - .lint: stage: lint before_script: From 09721486069788f6f698038b0e686eab0a378de7 Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 16:59:41 -0400 Subject: [PATCH 15/17] Working on test coverage for use with new package --- .gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ad3591..3858208 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,20 +29,22 @@ Coverage: paths: - MetaStalk.log - htmlcov/* - allow_failure: true + - Test_Build before_script: - pip install -U pip coverage -r requirements.txt --quiet - curl https://deepsource.io/cli | sh - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; chmod +x ./cc-test-reporter; ./cc-test-reporter before-build script: - - coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/ -t -d - - coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d + - pip install dist/*.whl + - coverage run -a --source . metastalk ./ExamplePhotos/ -t -d + - coverage run -a --source . metastalk ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d after_script: - coverage report - coverage html - coverage xml - ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml - ./cc-test-reporter after-build + .lint: stage: lint before_script: From c208b01ff36c4b5647fac78079df4c10e58d639f Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 17:03:53 -0400 Subject: [PATCH 16/17] More fixes for coverages --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3858208..d73dc35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,13 +23,12 @@ license_scanning: Coverage: image: python:3.7 - stage: test + stage: Test_Build coverage: '/TOTAL\s*\d*\s*\d*\s*(\d*)%/' artifacts: paths: - MetaStalk.log - htmlcov/* - - Test_Build before_script: - pip install -U pip coverage -r requirements.txt --quiet - curl https://deepsource.io/cli | sh From 6aa099d4ec1056f16bb8458053e881cde7574a1c Mon Sep 17 00:00:00 2001 From: Jacob Date: Sun, 3 May 2020 17:44:34 -0400 Subject: [PATCH 17/17] Added a lot of documentaion --- .gitlab-ci.yml | 8 ++++--- MetaStalk/__init__.py | 3 +++ MetaStalk/main.py | 42 ++++++++++++++++++++++++++++++----- MetaStalk/modules/DateTime.py | 16 +++++++++++-- MetaStalk/modules/GPSCheck.py | 13 +++++++++-- MetaStalk/modules/PieChart.py | 32 +++++++++++++++++++++----- MetaStalk/modules/Stats.py | 14 ++++++++++-- MetaStalk/tests/test_main.py | 8 ++++--- MetaStalk/utils/logger.py | 17 +++++++++++--- MetaStalk/utils/web.py | 12 +++++++++- 10 files changed, 137 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d73dc35..7d68123 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,22 +21,24 @@ license_scanning: - tags - master +# This job will fail until tests are implemented Coverage: image: python:3.7 - stage: Test_Build + stage: test coverage: '/TOTAL\s*\d*\s*\d*\s*(\d*)%/' artifacts: paths: - MetaStalk.log - htmlcov/* + allow_failure: true before_script: - pip install -U pip coverage -r requirements.txt --quiet - curl https://deepsource.io/cli | sh - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; chmod +x ./cc-test-reporter; ./cc-test-reporter before-build script: - pip install dist/*.whl - - coverage run -a --source . metastalk ./ExamplePhotos/ -t -d - - coverage run -a --source . metastalk ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d + - coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/ -t -d + - coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d after_script: - coverage report - coverage html diff --git a/MetaStalk/__init__.py b/MetaStalk/__init__.py index 81b9960..dc1c04f 100644 --- a/MetaStalk/__init__.py +++ b/MetaStalk/__init__.py @@ -1,3 +1,6 @@ """MetaStalk +--- +MetaStalk is a program that parse image metadata and creates charts from it. """ __version__ = "v2.0.0" +__author__ = "Cyb3r Jak3" diff --git a/MetaStalk/main.py b/MetaStalk/main.py index eebfe99..68736a2 100644 --- a/MetaStalk/main.py +++ b/MetaStalk/main.py @@ -17,7 +17,11 @@ def start(): - """ Sets up MetaStalk and parses arguments""" + """start + + Sets up MetaStalk and parses arguments. + Will raise an IOError if no path is passed. + """ parser = argparse.ArgumentParser(prog="MetaStalk", description="Tool to graph " "image metadata.") @@ -43,8 +47,14 @@ def start(): def run(args, log: logging.Logger): - """Process files and generates graphs""" + """run + + Process files and generates graphs + Arguments: + args {argparse.Namespace} -- The arguments from start() + log {logging.Logger} -- Logger + """ for path in args.files: isdir = os.path.isdir(path) log.debug("Detected path as a directory") @@ -67,8 +77,18 @@ def run(args, log: logging.Logger): utils.graph(plots, t_start, args.test) -def directory_search(files: list, log: logging.Logger): - """ Used to append all file in a directory """ +def directory_search(files: list, log: logging.Logger) -> (list, list): + """directory_search + + Used to append all files in a directory from args. + + Arguments: + files {list} -- List of directories to parse + log {logging.Logger} -- Logger + + Returns: + valid, invalid -- List of photos with metadata and ones without + """ valid, invalid = [], [] for item in os.listdir(files): item_path = os.path.join(files, item) @@ -85,8 +105,18 @@ def directory_search(files: list, log: logging.Logger): return valid, invalid -def file_search(files: list, log: logging.Logger): - """ Used to append files if the path is not a directory """ +def file_search(files: list, log: logging.Logger) -> (list, list): + """file_search + + Used to append files if the path is not a directory. + + Arguments: + files {list} -- List of files to parse + log {logging.Logger} -- Logger + + Returns: + valid, invalid -- List of photos with metadata and ones without + """ valid, invalid = [], [] for _, item in enumerate(files): parser = createParser(item) diff --git a/MetaStalk/modules/DateTime.py b/MetaStalk/modules/DateTime.py index d2017c1..edf587b 100644 --- a/MetaStalk/modules/DateTime.py +++ b/MetaStalk/modules/DateTime.py @@ -5,8 +5,20 @@ log = logging.getLogger("MetaStalk") -def date_time(photos): - """Makes a table with gps timestamp of photos""" +def date_time(photos: list) -> go.Figure(): + """date_time + + Makes a table with timestamp of photos. + There are three name that DateTime data can be under `Creation date`, + `Date-time original`, `Date-time digitized` and a column in made + for each type. + + Arguments: + photos {list} -- A list of dictionaries with phot information. + + Returns: + go.Figure -- A plotly Table with the DateTime data. + """ log.info("Starting DateTime Charts") datetime = [] datetime_original = [] diff --git a/MetaStalk/modules/GPSCheck.py b/MetaStalk/modules/GPSCheck.py index 5ce4da4..5552fce 100644 --- a/MetaStalk/modules/GPSCheck.py +++ b/MetaStalk/modules/GPSCheck.py @@ -5,8 +5,17 @@ log = logging.getLogger("MetaStalk") -def GPS_Check(photos: list): - """Takes a list of photos and creates a geo plot of them""" +def GPS_Check(photos: list) -> px.scatter_mapbox: + """GPS_Check + + Takes a list of photos and creates a geo plot of them + + Arguments: + photos {list} -- A list of dictionaries with phot information. + + Returns: + px.scatter_mapbox -- Map plot with photos plotted. + """ log.info("Starting GPS Chart") lats = [] longs = [] diff --git a/MetaStalk/modules/PieChart.py b/MetaStalk/modules/PieChart.py index 4981e65..1837283 100644 --- a/MetaStalk/modules/PieChart.py +++ b/MetaStalk/modules/PieChart.py @@ -5,8 +5,18 @@ log = logging.getLogger("MetaStalk") -def create_chart(table, pietype): - """Creates the pie chart""" +def create_chart(table: list, pielabel: str) -> go.Figure(): + """create_chart + + Creates the pie chart by frequency of items in a dictionary. + + Arguments: + table {list} -- [description] + pielabel {str} -- The label of the pie chart. + + Returns: + go.Figure -- A plotly PieChart + """ freq = {} for item in table: if item in freq: @@ -16,20 +26,30 @@ def create_chart(table, pietype): (labels, values) = ([], []) for key, value in freq.items(): - if pietype == "Camera focal": + if pielabel == "Camera focal": labels.append("Length: {}".format(key)) else: labels.append(key) values.append(value) fig = go.Figure(data=[go.Pie(labels=labels, values=values)]) - fig.update_layout(title="{} Information".format(pietype)) + fig.update_layout(title="{} Information".format(pielabel)) return fig -def PieChart(photos: list, pietype: str): - """Gets information and makes a pie chart""" +def PieChart(photos: list, pietype: str) -> go.Figure(): + """PieChart + + Parses information and returns a pie chart + + Arguments: + photos {list} -- A list of dictionaries with phot information. + pietype {str} -- The type of metadata for the pie chart + + Returns: + go.Figure -- A plotly PieChart + """ log.info("Staring %s Chart", pietype) table = [] diff --git a/MetaStalk/modules/Stats.py b/MetaStalk/modules/Stats.py index cc398be..68c631b 100644 --- a/MetaStalk/modules/Stats.py +++ b/MetaStalk/modules/Stats.py @@ -6,8 +6,18 @@ log = logging.getLogger("MetaStalk") -def Stats(photos: list, invalid: list): - """Creates the table of photos""" +def Stats(photos: list, invalid: list) -> go.Figure(): + """Stats + + Creates the table of photos showing ones with metadata and ones without + + Arguments: + photos {list} -- List of photos with metadata. + invalid {list} -- List of photos without metadata. + + Returns: + go.Figure() -- A plotly table + """ log.info("Staring Stats") log.debug("There are %s photos with metadata and %s without", len(photos), len(invalid)) diff --git a/MetaStalk/tests/test_main.py b/MetaStalk/tests/test_main.py index 7abaf09..e6aab5e 100644 --- a/MetaStalk/tests/test_main.py +++ b/MetaStalk/tests/test_main.py @@ -1,9 +1,11 @@ -"""Test suite for PyStalk""" +"""Test suite for MetaStalk +Currently unused but planned. +""" import pytest -import PyStalk +import MetaStalk def empty_directory_test(): """Shows result for empty directory""" with pytest.raises(FileNotFoundError): - PyStalk.run() + MetaStalk.run() diff --git a/MetaStalk/utils/logger.py b/MetaStalk/utils/logger.py index df8f176..8746960 100644 --- a/MetaStalk/utils/logger.py +++ b/MetaStalk/utils/logger.py @@ -2,9 +2,20 @@ import logging -def make_logger(name, log_level): - """Creates the logger. - Might look to make this happen in the main script +def make_logger(name: str, log_level: int) -> logging.Logger: + """make_logger + + Creates the logger. + + Might look to make this happen in the main script + + + Arguments: + name {str} -- name of the logger + log_level {int} -- Verbosity of the logger + + Returns: + logging.Logger -- The logger that gets used. """ logger = logging.getLogger(name) logger.setLevel(log_level) diff --git a/MetaStalk/utils/web.py b/MetaStalk/utils/web.py index 772026a..fcbdcd9 100644 --- a/MetaStalk/utils/web.py +++ b/MetaStalk/utils/web.py @@ -11,7 +11,17 @@ def graph(plots: dict, t_start: float, test=False): - """Creates the graphs""" + """graph + + Displays all the plots that are passed to it. + + Arguments: + plots {dict} -- All the plot that get displayed + t_start {float} -- The start time of MetaStalk + + Keyword Arguments: + test {bool} -- Whether or not to show the web page (default: {False}) + """ external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css'] app = dash.Dash(__name__, external_stylesheets=external_stylesheets) app.title = "MetaStalk"