From 1e87a556a92c70b63fc5ee2e666497966a7e7356 Mon Sep 17 00:00:00 2001 From: Cyb3r Jak3 Date: Wed, 6 May 2020 21:07:11 +0000 Subject: [PATCH] MetaStalk 2.1.0 --- .gitignore | 3 +- .gitlab-ci.yml | 21 ++-- CHANGELOG.md | 21 +++- CONTRIBUTING.md | 14 ++- ExamplePhotos/corrupt_image.png | Bin 0 -> 878 bytes Manifest.in | 3 +- MetaStalk/__init__.py | 4 +- MetaStalk/main.py | 190 +++++++++++++++----------------- MetaStalk/modules/DateTime.py | 18 ++- MetaStalk/modules/GPSCheck.py | 14 ++- MetaStalk/modules/PieChart.py | 13 ++- MetaStalk/modules/Stats.py | 3 +- MetaStalk/modules/__init__.py | 16 ++- MetaStalk/tests/test_main.py | 11 -- MetaStalk/utils/__init__.py | 9 +- MetaStalk/utils/export.py | 37 +++++++ MetaStalk/utils/web.py | 22 ++-- Pipfile | 1 - README.md | 45 ++++++-- codecov.yml | 13 ++- requirements-dev.txt | 6 +- requirements.txt | 8 +- setup.py | 63 +++++++---- tests/__init__.py | 1 + tests/test_main.py | 75 +++++++++++++ tox.ini | 15 ++- 26 files changed, 406 insertions(+), 220 deletions(-) create mode 100644 ExamplePhotos/corrupt_image.png delete mode 100644 MetaStalk/tests/test_main.py create mode 100644 MetaStalk/utils/export.py create mode 100644 tests/__init__.py create mode 100644 tests/test_main.py diff --git a/.gitignore b/.gitignore index 569dc18..0fe3ccb 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,5 @@ dmypy.json .pyre/ # Hide vscode folder -.vscode/ \ No newline at end of file +.vscode/ +metastalk_exports/* \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d68123..76ec00b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,13 +2,14 @@ stages: - lint - test - Test_Build -- Deploy_to_PyPi +- Deploy include: - template: SAST.gitlab-ci.yml - template: Dependency-Scanning.gitlab-ci.yml - template: License-Scanning.gitlab-ci.yml + dependency_scanning: only: refs: @@ -21,7 +22,6 @@ license_scanning: - tags - master -# This job will fail until tests are implemented Coverage: image: python:3.7 stage: test @@ -30,21 +30,22 @@ Coverage: paths: - MetaStalk.log - htmlcov/* - allow_failure: true + - metastalk_exports/* before_script: - pip install -U pip coverage -r requirements.txt --quiet - curl https://deepsource.io/cli | sh - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter; chmod +x ./cc-test-reporter; ./cc-test-reporter before-build script: - - pip install dist/*.whl - - coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/ -t -d - - coverage run -a --source . ./MetaStalk/main.py ./ExamplePhotos/Panasonic_DMC-FZ30.jpg ./ExamplePhotos/DSCN0010.jpg -t -d + - pip install . + - coverage run --source=./MetaStalk/ -m unittest -vv after_script: - coverage report - coverage html - coverage xml + - bash <(curl -Ls https://coverage.codacy.com/get.sh) report -l Python -r coverage.xml - ./bin/deepsource report --analyzer test-coverage --key python --value-file ./coverage.xml - ./cc-test-reporter after-build + - bash <(curl -s https://codecov.io/bash) .lint: stage: lint @@ -119,19 +120,17 @@ python-3.8-test: extends: ".test" image: python:3.8 - Deploy_to_PyPi: image: python:3.7 - stage: Deploy_to_PyPi + stage: Deploy variables: - TWINE_USERNAME: $PRODUCTION_USERNAME + TWINE_USERNAME: $PYPI_USER TWINE_PASSWORD: $PRODUCTION_PASSWORD before_script: - pip install -U pip setuptools twine --quiet script: - - python setup.py sdist bdist_wheel - python -m twine check dist/* - - python -m twine upload --verbose --username $PRODUCTION_USER --password $PRODUCTION_PASSWORD dist/* + - python -m twine upload --verbose dist/* only: refs: - tags diff --git a/CHANGELOG.md b/CHANGELOG.md index eebfe2b..fbe8739 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ +## [v2.1.0]- 2020-05-06 + +### Added + +- Ability to pass both directories and individual files. +- Unittests for testing. +- Footer for run time. +- Export feature. +- Added metastalk dev and image install. +- [Codacy](https://www.codacy.com/) +- Two new arguments `--no-open` and `--alphabetic`. + - `--no-open` will make it so a new browser tab is not opened. + - `--alphabetic` will sort all the charts alphabetically. + +### Changed + +- Created MetaStalk Class. +- All titles for charts are centered. + ## [v2.0.0] - 2020-05-03 ## Rename to MetaStalk @@ -11,7 +30,7 @@ Rename to MetaStalk to create PyPi package and a lot backend development changes ### Added - License scanning -- [Codecov](https://codecov.io/gl/Cyb3r-Jak3/pystalk) +- [Codecov](https://codecov.io/gl/Cyb3r-Jak3/metastalk) - Pipfile and Pipfile.lock for pipenv. - Added .gitlab folder for service desk. - setup.py for pypi. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40126ea..fbf208a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,9 +2,15 @@ Thanks for reading this because I am always looking to collaborate with other people to see their ideas. -If you are looking to submit a pull request then please do so on [GitLab](https://gitlab.com/Cyb3r-Jak3/pystalk) because all development is done there. Any pull request that is opened on GitHub will be closed and I mirror it on GitLab. +If you are looking to submit a pull request then please do so on [GitLab](https://gitlab.com/Cyb3r-Jak3/MetaStalk) because all development is done there. Any pull request that is opened on GitHub will be closed and I mirror it on GitLab. -Please open issues on [GitLab](https://gitlab.com/Cyb3r-Jak3/PyStalk/issues). if you do not have a GitLab account, send then please email service desk( [incoming+cyb3r-jak3-pystalk-15263483-issue-@incoming.gitlab.com](mailto:incoming+cyb3r-jak3-pystalk-15263483-issue-@incoming.gitlab.com) ). +Please open issues on [GitLab](https://gitlab.com/Cyb3r-Jak3/MetaStalk/issues). If you do not have a GitLab account, send then please email service desk( [incoming+cyb3r-jak3-metastalk-15263483-issue-@incoming.gitlab.com](mailto:incoming+cyb3r-jak3-metastalk-15263483-issue-@incoming.gitlab.com)). + +To get started you can either clone the repo or use `pip install metastalk[dev]`. + +## Testing + +Currently the tests at [tests](tests/) cover all the necessary items to make sure the it works. Currently no test exist for testing the image export capabilities as it relies on [orca](https://github.com/plotly/orca) which is not easy to setup tests for. If someone has a way of creating a test with orca, then please submit a pull request and I will gladly approve. ## Modules @@ -13,10 +19,10 @@ If you are looking to write a new module for a new metadata graph here is they a - Each module is written in the [modules](modules/) directory and is added to the [\_\_init__.py](modules/__init__.py) in the directory. - A list is passed to the modules called `photos` which containing dictionaries with all the metadata points for each photo. - The module will find the key that is using and return a figure using plotly. -- A dictionary of all the plots will be passed to [web.py](utils/web.py) which will use dash to display them. +- A dictionary of all the plots will be passed to [web.py](utils/web.py) which will use Dash to display them. ### Example image dictionary ```python -{'Image width': '1600 pixels', 'Image height': '1200 pixels', 'Image orientation': 'Horizontal (normal)', 'Bits/pixel': '24', 'Pixel format': 'YCbCr', 'Creation date': '2007-11-29 16:16:21', 'Camera aperture': '2.97', 'Camera focal': '2.8', 'Camera exposure': '1/6', 'Camera model': 'Canon PowerShot SD300', 'Camera manufacturer': 'Canon', 'Compression': 'JPEG (Baseline)', 'Thumbnail size': '5922 bytes', 'EXIF version': '0220', 'Date-time original': '2007-11-29 16:16:21', 'Date-time digitized': '2007-11-29 16:16:21', 'Compressed bits per pixel': '3', 'Shutter speed': '2.59', 'Aperture': '2.97', 'Exposure bias': '0', 'Focal length': '5.8', 'Flashpix version': '0100', 'Focal plane width': '7.14e+03', 'Focal plane height': '7.14e+03', 'Comment': 'JPEG quality: 90% (approximate)', 'MIME type': 'image/jpeg', 'Endianness': 'Big endian', 'item': '.\\utils\\ExamplePhotos\\22-canon_tags.jpg'} +{'Image width': '1600 pixels', 'Image height': '1200 pixels', 'Image orientation': 'Horizontal (normal)', 'Bits/pixel': '24', 'Pixel format': 'YCbCr', 'Creation date': '2007-11-29 16:16:21', 'Camera aperture': '2.97', 'Camera focal': '2.8', 'Camera exposure': '1/6', 'Camera model': 'Canon PowerShot SD300', 'Camera manufacturer': 'Canon', 'Compression': 'JPEG (Baseline)', 'Thumbnail size': '5922 bytes', 'EXIF version': '0220', 'Date-time original': '2007-11-29 16:16:21', 'Date-time digitized': '2007-11-29 16:16:21', 'Compressed bits per pixel': '3', 'Shutter speed': '2.59', 'Aperture': '2.97', 'Exposure bias': '0', 'Focal length': '5.8', 'Flashpix version': '0100', 'Focal plane width': '7.14e+03', 'Focal plane height': '7.14e+03', 'Comment': 'JPEG quality: 90% (approximate)', 'MIME type': 'image/jpeg', 'Endianness': 'Big endian', 'item': '.\\ExamplePhotos\\22-canon_tags.jpg'} ``` diff --git a/ExamplePhotos/corrupt_image.png b/ExamplePhotos/corrupt_image.png new file mode 100644 index 0000000000000000000000000000000000000000..f8623eddee7d4737d0de650973991afc036cbf31 GIT binary patch literal 878 zcmeAS@N?(o$~c*~eYHt$I(Eo7ic0_fwtW<>bGm;)=5jeyolXh)$3Yb4pp1ew^|4Dq~N38Rsc1tJKM562ThVpG<%^&5misKSJZwkn+thb0xAU`}vUT^Q={G*|nF`GG=#9Fu;8?zQ z>OI@NGD)^qL)t%0IW$RK^a{{a--CVk<4SMYo?ZBQ=iR*S89B;=>!tYGbNIV&3(h>@ zD!AzL&pij%3f&6c#qeyBlg62~SzlNRtTx@c!sb73#s-(>4xg0gUrbZXHmbZ{`Y2WZ zS@M^)_xzZ8HLmq?-1Rc)IKi@R_pCJ^-7@W7ge|glQ42VmIc3sF-|@$fwkoP9DLJJuF@B6>D4zOg+wtjpr)ZT4Y5WTgUw!pqfq}e42iHW-CC+!Z z9!&~V3AVT~m+z#45hKF~pdN+?ttVI*4scK8WM~jq5@uk~cTrb#G`xixRpR)}XK4$;|Pgg&ebxsLQ0BsFyO#lD@ literal 0 HcmV?d00001 diff --git a/Manifest.in b/Manifest.in index 024f685..a3d7b5d 100644 --- a/Manifest.in +++ b/Manifest.in @@ -1,2 +1 @@ -graft MetaStalk/utils/assets -prune MetaStalk/tests \ No newline at end of file +recursive-include MetaStalk/utils/assets * \ No newline at end of file diff --git a/MetaStalk/__init__.py b/MetaStalk/__init__.py index dc1c04f..412aec5 100644 --- a/MetaStalk/__init__.py +++ b/MetaStalk/__init__.py @@ -1,6 +1,6 @@ -"""MetaStalk +"""MetaStalk. --- MetaStalk is a program that parse image metadata and creates charts from it. """ -__version__ = "v2.0.0" +__version__ = "v2.1.0" __author__ = "Cyb3r Jak3" diff --git a/MetaStalk/main.py b/MetaStalk/main.py index 68736a2..c738ebc 100644 --- a/MetaStalk/main.py +++ b/MetaStalk/main.py @@ -1,135 +1,117 @@ # -*- coding: utf-8 -*- -"""This script get the exif data from photos -and creates graphs from the metadata""" +"""Main function of MetaStalk. +Run get any metadata from photos +and creates graphs from the metadata using MetaStalk.Modules""" import argparse +from collections import OrderedDict import os import logging import timeit from hachoir.parser import createParser from hachoir.metadata import extractMetadata - - +from hachoir.core import config as hachoirconfig import MetaStalk.utils as utils import MetaStalk.modules as modules +hachoirconfig.quiet = True -t_start = timeit.default_timer() +class MetaStalk(): + """MetaStalk. + --- + + Main Class for all MetaStalk work + """ + def __init__(self): + self.log = logging.getLogger("MetaStalk") + self.t_start = timeit.default_timer() + self.valid, self.invalid = [], [] + self.plots = {} + + def run(self, args): + """Run + + Process files and generates graphs + + Arguments: + args {argparse.Namespace} -- The arguments from start() + log {logging.Logger} -- Logger + """ + for path in args.files: + if os.path.isdir(path): + self.log.debug("Detected path as a directory") + for item in os.listdir(path): + item_path = os.path.join(path, item) + self.file_search(item_path) + else: + self.file_search(path) + + self.plots = { + "Stats": modules.stats(self.valid, self.invalid), + "GPS": modules.gps_check(self.valid), + "Timestamp": modules.date_time(self.valid), + "Model": modules.pie_chart(self.valid, "Camera model"), + "Manufacturer": modules.pie_chart(self.valid, "Camera manufacturer"), + "Focal": modules.pie_chart(self.valid, "Camera focal"), + "Producer": modules.pie_chart(self.valid, "Producer") + } + if args.alphabetic: + self.plots = OrderedDict(sorted(self.plots.items())) + if args.export: + utils.export(args.export, args.output, self.plots) + utils.graph(self.plots, self.t_start, args.test, args.no_open) + + def file_search(self, parse_file: str): + """file_search + + Used to append files if the path is not a directory. + + Arguments + files {str} -- Name of the file to parse. + """ + parser = createParser(parse_file) + try: + metadata = extractMetadata(parser).exportDictionary()["Metadata"] + metadata["item"] = parse_file + self.valid.append(metadata) + self.log.debug("%s has metadata", parse_file) + except AttributeError: + self.invalid.append(parse_file) + self.log.debug("%s has no metadata data", parse_file) -def start(): - """start - Sets up MetaStalk and parses arguments. - Will raise an IOError if no path is passed. +def start(): + """ + Function needed to start MetaStalk """ parser = argparse.ArgumentParser(prog="MetaStalk", description="Tool to graph " "image metadata.") parser.add_argument('files', nargs='*', default=None, help='Path of photos to check.') - parser.add_argument('-t', '--test', default=False, action="store_true", - help='Does not show the graphs at the end.') -# Logging function from https://stackoverflow.com/a/20663028 + parser.add_argument("-a", "--alphabetic", help="Sorts charts in alphabetical order rather than" + " the default order", default=False, action="store_true") parser.add_argument('-d', '--debug', help="Sets logging level to DEBUG.", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.WARNING) + parser.add_argument("-e", "--export", choices=["pdf", "svg", "png", "html"], + help="Exports the graphs rather than all on one webpage") + parser.add_argument("--no-open", help="Will only start the server and not open the browser" + " to view it", default=False, action="store_true") + parser.add_argument("-o", "--output", default="metastalk_exports", + help="The name of the directory to output exports to. " + "Will be created if it does not exist. " + "Defaults to metastalk_exports.") + parser.add_argument('-t', '--test', default=False, action="store_true", + help='Does not show the graphs at the end.') parser.add_argument("-v", "--verbose", help="Sets logging level to INFO", action="store_const", dest="loglevel", const=logging.INFO) args = parser.parse_args() - log = utils.make_logger("MetaStalk", args.loglevel) log.info("MetaStalk starting") if not args.files: log.error("ERROR: No path was inputted.") - raise IOError("No path was inputted.") - run(args, log) - - -def run(args, log: logging.Logger): - """run - - Process files and generates graphs - - Arguments: - args {argparse.Namespace} -- The arguments from start() - log {logging.Logger} -- Logger - """ - for path in args.files: - isdir = os.path.isdir(path) - log.debug("Detected path as a directory") - - if isdir: - photos, invalid_photos = directory_search(args.files[0], log) - else: - photos, invalid_photos = file_search(args.files, log) - - plots = { - "STATS": modules.Stats(photos, invalid_photos), - "GPS": modules.GPS_Check(photos), - "Timestamp": modules.date_time(photos), - "Model": modules.PieChart(photos, "Camera model"), - "Manufacturer": modules.PieChart(photos, "Camera manufacturer"), - "Focal": modules.PieChart(photos, "Camera focal"), - "Producer": modules.PieChart(photos, "Producer") - } - - utils.graph(plots, t_start, args.test) - - -def directory_search(files: list, log: logging.Logger) -> (list, list): - """directory_search - - Used to append all files in a directory from args. - - Arguments: - files {list} -- List of directories to parse - log {logging.Logger} -- Logger - - Returns: - valid, invalid -- List of photos with metadata and ones without - """ - valid, invalid = [], [] - for item in os.listdir(files): - item_path = os.path.join(files, item) - parser = createParser(item_path) - metadata = extractMetadata(parser).exportDictionary()["Metadata"] - if metadata: - metadata["item"] = item_path - valid.append(metadata) - log.debug("%s has metadata", item) - else: - metadata["item"] = item_path - invalid.append(metadata) - log.debug("%s has no metadata", item) - return valid, invalid - - -def file_search(files: list, log: logging.Logger) -> (list, list): - """file_search - - Used to append files if the path is not a directory. - - Arguments: - files {list} -- List of files to parse - log {logging.Logger} -- Logger - - Returns: - valid, invalid -- List of photos with metadata and ones without - """ - valid, invalid = [], [] - for _, item in enumerate(files): - parser = createParser(item) - metadata = extractMetadata(parser).exportDictionary()["Metadata"] - if metadata: - metadata["item"] = item - valid.append(metadata) - log.debug("%s has metadata", item) - else: - metadata["item"] = item - invalid.append(metadata) - log.debug("%s has no metadata data", item) - return valid, invalid - - -start() + raise FileNotFoundError("No path was inputted.") + metastalk = MetaStalk() + metastalk.run(args) diff --git a/MetaStalk/modules/DateTime.py b/MetaStalk/modules/DateTime.py index edf587b..3070aea 100644 --- a/MetaStalk/modules/DateTime.py +++ b/MetaStalk/modules/DateTime.py @@ -1,4 +1,4 @@ -"""Makes a table that plots gps timestamp""" +"""Makes a table that plots GPS timestamp or other timestamp metadata.""" import logging import plotly.graph_objects as go @@ -20,9 +20,7 @@ def date_time(photos: list) -> go.Figure(): go.Figure -- A plotly Table with the DateTime data. """ log.info("Starting DateTime Charts") - datetime = [] - datetime_original = [] - datetime_digitized = [] + datetime, datetime_original, datetime_digitized = [], [], [] types = [datetime, datetime_original, datetime_digitized] types_str = ["Creation date", "Date-time original", "Date-time digitized"] @@ -36,18 +34,18 @@ def date_time(photos: list) -> go.Figure(): types[i].append(each[types_str[i]]) log.debug("%s has %s data", each["item"], types_str[i]) except KeyError: - log.debug("%s has no %s data ", each["item"], types_str[i]) + log.info("%s has no %s data ", each["item"], types_str[i]) fig = go.Figure( data=[go.Table( header=dict(values=[ - "Photos", - "Creation date", - "Date time Original", - "Date time Digitized" + "Photo", + "Creation Date", + "Date Time Original", + "Date Time Digitized" ]), cells=dict(values=[simple_photos, datetime, datetime_original, datetime_digitized]))] ) - + fig.update_layout(title="Timestamp Information", title_x=0.5) return fig diff --git a/MetaStalk/modules/GPSCheck.py b/MetaStalk/modules/GPSCheck.py index 5552fce..95c263e 100644 --- a/MetaStalk/modules/GPSCheck.py +++ b/MetaStalk/modules/GPSCheck.py @@ -1,11 +1,11 @@ -"""Makes geo chart with plots of gps data""" +"""Makes geo chart with plots of GPS data""" import logging import plotly.express as px log = logging.getLogger("MetaStalk") -def GPS_Check(photos: list) -> px.scatter_mapbox: +def gps_check(photos: list) -> px.scatter_mapbox: """GPS_Check Takes a list of photos and creates a geo plot of them @@ -13,7 +13,7 @@ def GPS_Check(photos: list) -> px.scatter_mapbox: Arguments: photos {list} -- A list of dictionaries with phot information. - Returns: + Returns px.scatter_mapbox -- Map plot with photos plotted. """ log.info("Starting GPS Chart") @@ -26,14 +26,16 @@ def GPS_Check(photos: list) -> px.scatter_mapbox: gps_photos.append(each["item"]) lats.append(float(each["Latitude"])) longs.append(float(each["Longitude"])) - log.debug("%s has gps data", each["item"]) + log.debug("%s has GPS data", each["item"]) + else: + log.info("%s has no GPS data", each["item"]) points = [] for x, _ in enumerate(gps_photos): points.append((lats[x], longs[x])) fig = px.scatter_mapbox(lon=longs, lat=lats, hover_name=gps_photos, - title="Geo Locations") - fig.update_layout(mapbox_style="open-street-map") + title="Geo Locations",) + fig.update_layout(mapbox_style="open-street-map", title_x=0.5) return fig diff --git a/MetaStalk/modules/PieChart.py b/MetaStalk/modules/PieChart.py index 1837283..f883a12 100644 --- a/MetaStalk/modules/PieChart.py +++ b/MetaStalk/modules/PieChart.py @@ -6,7 +6,7 @@ def create_chart(table: list, pielabel: str) -> go.Figure(): - """create_chart + """Create_chart Creates the pie chart by frequency of items in a dictionary. @@ -14,8 +14,9 @@ def create_chart(table: list, pielabel: str) -> go.Figure(): table {list} -- [description] pielabel {str} -- The label of the pie chart. - Returns: + Returns go.Figure -- A plotly PieChart + """ freq = {} for item in table: @@ -33,12 +34,12 @@ def create_chart(table: list, pielabel: str) -> go.Figure(): values.append(value) fig = go.Figure(data=[go.Pie(labels=labels, values=values)]) - fig.update_layout(title="{} Information".format(pielabel)) + fig.update_layout(title=f"{pielabel} Information", title_x=0.5) return fig -def PieChart(photos: list, pietype: str) -> go.Figure(): +def pie_chart(photos: list, pietype: str) -> go.Figure(): """PieChart Parses information and returns a pie chart @@ -49,6 +50,7 @@ def PieChart(photos: list, pietype: str) -> go.Figure(): Returns: go.Figure -- A plotly PieChart + """ log.info("Staring %s Chart", pietype) table = [] @@ -56,7 +58,8 @@ def PieChart(photos: list, pietype: str) -> go.Figure(): for each in photos: try: table.append(each[pietype]) + log.debug("%s has %s data", each["item"], pietype) except KeyError: - log.debug("%s has no %s data", each["item"], pietype) + log.info("%s has no %s data", each["item"], pietype) return create_chart(table, pietype) diff --git a/MetaStalk/modules/Stats.py b/MetaStalk/modules/Stats.py index 68c631b..0df26fe 100644 --- a/MetaStalk/modules/Stats.py +++ b/MetaStalk/modules/Stats.py @@ -6,7 +6,7 @@ log = logging.getLogger("MetaStalk") -def Stats(photos: list, invalid: list) -> go.Figure(): +def stats(photos: list, invalid: list) -> go.Figure(): """Stats Creates the table of photos showing ones with metadata and ones without @@ -30,4 +30,5 @@ def Stats(photos: list, invalid: list) -> go.Figure(): "Photos with Metadata", "Photos without Metadata"]), cells=dict(values=[simple_photos, invalid]))]) + fig.update_layout(title="Photos With and Without Metadata.", title_x=0.5) return fig diff --git a/MetaStalk/modules/__init__.py b/MetaStalk/modules/__init__.py index a837c3d..0010390 100644 --- a/MetaStalk/modules/__init__.py +++ b/MetaStalk/modules/__init__.py @@ -1,8 +1,12 @@ -"""Imports the modules""" -from .GPSCheck import GPS_Check -from .Stats import Stats +"""MetaStalk.Modules. + +Imports the modules that create the charts. + +""" +from .GPSCheck import gps_check +from .Stats import stats from .DateTime import date_time -from .PieChart import PieChart +from .PieChart import pie_chart -__all__ = ["GPS_Check", "Stats", "date_time", - "PieChart"] +__all__ = ["gps_check", "stats", "date_time", + "pie_chart"] diff --git a/MetaStalk/tests/test_main.py b/MetaStalk/tests/test_main.py deleted file mode 100644 index e6aab5e..0000000 --- a/MetaStalk/tests/test_main.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Test suite for MetaStalk -Currently unused but planned. -""" -import pytest -import MetaStalk - - -def empty_directory_test(): - """Shows result for empty directory""" - with pytest.raises(FileNotFoundError): - MetaStalk.run() diff --git a/MetaStalk/utils/__init__.py b/MetaStalk/utils/__init__.py index 130adfa..870124e 100644 --- a/MetaStalk/utils/__init__.py +++ b/MetaStalk/utils/__init__.py @@ -1,5 +1,10 @@ -# pylint: disable=missing-module-docstring +"""MetaStalk.Utils. + +Imports the utility functions that create the logger and output. + +""" from .web import graph from .logger import make_logger +from .export import export -__all__ = ['graph', "make_logger"] +__all__ = ['graph', "make_logger", "export"] diff --git a/MetaStalk/utils/export.py b/MetaStalk/utils/export.py new file mode 100644 index 0000000..4008291 --- /dev/null +++ b/MetaStalk/utils/export.py @@ -0,0 +1,37 @@ +"""utils.export +--- + +Exports the plots as interactive html +""" +import logging +import os + +log = logging.getLogger("MetaStalk") + + +def export(choice: str, output_dir: str, plots: dict): + """export + + Exports the plots to the chosen format + + Arguments: + choice {str}: -- The type of export. {html, pdf, svg, png} + output_dir {str} -- Name of the directory to output charts to. + plots {dict} -- The plots to export. + """ + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + else: + if len(os.listdir(output_dir)) != 0: + log.warning("The chosen output directory contain files.") + + if choice == "html": + for name, chart in plots.items(): + chart.write_html(f"{output_dir}/{name}.html") + elif choice in ["pdf", "svg", "png"]: + try: + for name, chart in plots.items(): + chart.write_image(f"{output_dir}/{name}.{choice}") + except ValueError: + log.error("Dash requires orca to be install to export images.") + raise EnvironmentError("Dash requires orca to be install to export images.") diff --git a/MetaStalk/utils/web.py b/MetaStalk/utils/web.py index fcbdcd9..a5fef3b 100644 --- a/MetaStalk/utils/web.py +++ b/MetaStalk/utils/web.py @@ -2,6 +2,7 @@ import timeit import logging import webbrowser +from datetime import datetime import dash import dash_html_components as html import dash_core_components as dcc @@ -10,7 +11,7 @@ log = logging.getLogger("MetaStalk") -def graph(plots: dict, t_start: float, test=False): +def graph(plots: dict, t_start: float, test: bool, no_open: bool): """graph Displays all the plots that are passed to it. @@ -20,25 +21,28 @@ def graph(plots: dict, t_start: float, test=False): t_start {float} -- The start time of MetaStalk Keyword Arguments: - test {bool} -- Whether or not to show the web page (default: {False}) + test {bool} -- Whether or not to start web server (default: {False}) + no_open {bool} -- Whether or now to open with the browser (default: {False}) """ - external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css'] - app = dash.Dash(__name__, external_stylesheets=external_stylesheets) - app.title = "MetaStalk" graphs = [] for name, chart in plots.items(): graphs.append(dcc.Graph(id="graph-{}".format(name), figure=chart)) + external_stylesheets = ["https://codepen.io/chriddyp/pen/bWLwgP.css"] + app = dash.Dash(__name__, external_stylesheets=external_stylesheets) + app.title = "MetaStalk" t_stop = timeit.default_timer() app.layout = html.Div([ html.H1("MetaStalk", style={"textAlign": "center"}), - html.H6(html.A('Cyber Jake', href="https://twitter.com/Cyb3r_Jak3"), + html.H6(html.A("By Cyber Jake", href="https://twitter.com/Cyb3r_Jak3"), style={"textAlign": "center"}), html.Div(children=graphs), - html.P("Time Taken = {0:.2f} seconds".format(t_stop - t_start), - style={"textAlign": "center"}) + html.Footer("Time Taken = {0:.2f} seconds".format(t_stop - t_start), + style={"textAlign": "right"}), + html.Footer(f"Run Time: {datetime.now().strftime('%m/%d/%Y, %H:%M:%S')}") ]) if not test: - webbrowser.open("http://localhost:8052", new=2) + if not no_open: + webbrowser.open("http://localhost:8052", new=2) app.run_server(port=8052) else: log.info("Test flag was set. No webpage will be shown.") diff --git a/Pipfile b/Pipfile index 298f40c..ed06fe3 100644 --- a/Pipfile +++ b/Pipfile @@ -6,7 +6,6 @@ verify_ssl = true [dev-packages] flake8 = "*" pylint = "*" -pytest = "*" [packages] hachoir = "*" diff --git a/README.md b/README.md index 4f7c1cb..05ce344 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # MetaStalk -[![GitHub](https://img.shields.io/github/license/Cyb3r-Jak3/MetaStalk?style=flat)](https://github.com/Cyb3r-Jak3/MetaStalk/blob/master/LICENSE) ![Gitlab pipeline status (branch)](https://img.shields.io/gitlab/pipeline/Cyb3r-Jak3/MetaStalk/master?label=Build&style=flat) +[![GitHub](https://img.shields.io/github/license/Cyb3r-Jak3/MetaStalk?style=flat)](https://github.com/Cyb3r-Jak3/MetaStalk/blob/master/LICENSE) ![Gitlab pipeline status (branch)](https://img.shields.io/gitlab/pipeline/Cyb3r-Jak3/MetaStalk/master?label=Build&style=flat) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/metastalk) ![PyPI](https://img.shields.io/pypi/v/metastalk) -![PyPI - Python Version](https://img.shields.io/pypi/pyversions/metastalk) ![PyPI](https://img.shields.io/pypi/v/metastalk) +[![Maintainability](https://api.codeclimate.com/v1/badges/9b95ea5f0c8a77eab0ed/maintainability)](https://codeclimate.com/github/Cyb3r-Jak3/MetaStalk/maintainability) [![Test Coverage](https://api.codeclimate.com/v1/badges/9b95ea5f0c8a77eab0ed/test_coverage)](https://codeclimate.com/github/Cyb3r-Jak3/MetaStalk/test_coverage) -[![Maintainability](https://api.codeclimate.com/v1/badges/9b95ea5f0c8a77eab0ed/maintainability)](https://codeclimate.com/github/Cyb3r-Jak3/MetaStalk/maintainability) ![Libraries.io dependency status for latest release](https://img.shields.io/librariesio/release/pypi/metastalk) -[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/Cyb3r-Jak3/MetaStalk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/Cyb3r-Jak3/MetaStalk/?branch=master) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/Cyb3r-Jak3/MetaStalk.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Cyb3r-Jak3/MetaStalk/context:python) +[![codecov](https://codecov.io/gl/Cyb3r-Jak3/metastalk/branch/master/graph/badge.svg)](https://codecov.io/gl/Cyb3r-Jak3/metastalk) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/d3ed4a583afc4d27bbca4c8eb4b51e78)](https://www.codacy.com/manual/Cyb3r_Jak3/metastalk?utm_source=gitlab.com&utm_medium=referral&utm_content=Cyb3r-Jak3/metastalk&utm_campaign=Badge_Grade) + +[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/Cyb3r-Jak3/MetaStalk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/Cyb3r-Jak3/MetaStalk/?branch=master) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/Cyb3r-Jak3/MetaStalk.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Cyb3r-Jak3/MetaStalk/context:python) ## About @@ -19,7 +19,7 @@ It currently creates graphs for: Examples photos from [ianare/exif-samples](https://github.com/ianare/exif-samples/tree/master/jpg/gps), [exiftool](https://owl.phy.queensu.ca/~phil/exiftool/sample_images.html), [drewmpales/metadata-extractor-images](https://github.com/drewnoakes/metadata-extractor-images). -All development is done on GitLab and pushed to GitHub. Please read [contributing.md](CONTRIBUTING.md) for development. +All development is done on GitLab and mirrored to GitHub. Please read [contributing.md](CONTRIBUTING.md) for development. Python 3.6 and up. @@ -27,6 +27,35 @@ Python 3.6 and up. MetaStalk is available as a package on pypi.org or you can do a source install. +```bash +usage: MetaStalk [-h] [-a] [-d] [-e {pdf,svg,png,html}] [--no-open] + [-o OUTPUT] [-t] [-v] + [files [files ...]] + +Tool to graph image metadata. + +positional arguments: + files Path of photos to check. + +optional arguments: + -h, --help show this help message and exit + -a, --alphabetic Sorts charts in alphabetical order rather than the + default order + -d, --debug Sets logging level to DEBUG. + -e {pdf,svg,png,html}, --export {pdf,svg,png,html} + Exports the graphs rather than all on one webpage + --no-open Will only start the server and not open the browser to + view it + -o OUTPUT, --output OUTPUT + The name of the directory to output exports to. Will + be created if it does not exist. Defaults to + metastalk_exports. + -t, --test Does not show the graphs at the end. + -v, --verbose Sets logging level to INFO +``` + +If you want to have the export image option then you need to have [orca](https://github.com/plotly/orca) installed and install metastalk with `pip install metastalk[image]`. + ### PyPi Install ```bash @@ -45,10 +74,8 @@ metastalk #i.e. metastalk ./ExamplePhotos/ ``` -This project also use Pipfile and Pipfile.lock if you would rather pipenv over requirements.txt - ## Disclaimer This is for educational/proof of concept purposes only. What you do with this program is **your** responsibility. -[![DeepSource](https://static.deepsource.io/deepsource-badge-light.svg)](https://deepsource.io/gl/Cyb3r-Jak3/PyStalk/?ref=repository-badge) +[![DeepSource](https://static.deepsource.io/deepsource-badge-light-mini.svg)](https://deepsource.io/gl/Cyb3r-Jak3/MetaStalk/?ref=repository-badge) diff --git a/codecov.yml b/codecov.yml index 1947720..1ac4443 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,7 @@ -codecov: - coverage: - precision: 2 - round: down - range: "70...100" - require_ci_to_pass: yes \ No newline at end of file +codecov: + coverage: + precision: 2 + round: down + range: "70...100" + require_ci_to_pass: yes + \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 65e5732..ce7a4f9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ -pytest -pylint -flake8 \ No newline at end of file +bandit >= 1.6.2 +pylint >= 2.5.0 +flake8 >= 3.7.9 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6f3fb36..228a648 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -hachoir -plotly -pandas -dash \ No newline at end of file +hachoir >= 3.1.1 +plotly >= 4.6.0 +pandas >= 1.0.3 +dash >= 1.11.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 8c8df20..c25b75b 100644 --- a/setup.py +++ b/setup.py @@ -1,36 +1,47 @@ """Sets up PyStalk to be installed""" import os from setuptools import setup, find_packages -from MetaStalk import __version__ +from MetaStalk import __version__, __author__ -def read(fname): - """Reads README.md as long description""" +def read(fname) -> str: + """Reads the fname file. + Used to read the README.MD file""" return open(os.path.join(os.path.dirname(__file__), fname)).read() +def get_requirements(fname: str) -> list: + """get_requirements + + Arguments: + fname {str} -- The name of the requirements file. + + Returns: + list -- List of requirements + """ + with open(f"{fname}.txt") as f: + return f.read().splitlines() + + setup( name="MetaStalk", version=__version__, - author="Cyb3r Jak3", + author=__author__, author_email="jake@jwhite.network", - install_requires=[ - "hachoir >= 3.1.1", - "plotly >= 4.6.0", - "pandas >= 1.0.3", - "dash >= 1.11.0"], - description="Metadata analyzer", + install_requires=get_requirements("requirements"), + extra_requires={ + "dev": get_requirements("requirements-dev"), + "image": [ + "psutil >= 5.7.0", + "requests >= 2.23.0" + ] + }, + description="Metadata analyzer and visualizer", license="MPL 2.0", python_requires=">=3.6", - classifiers=[ - "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", - "Operating System :: OS Independent", - "Natural Language :: English", - "Programming Language :: Python :: 3 :: Only", - "Development Status :: 4 - Beta", - "Environment :: Console" - ], - packages=find_packages(), + packages=find_packages(exclude=["tests"]), + package_data={'MetaStalk': ['utils/assets/*']}, + include_package_data=True, long_description=read('README.md'), long_description_content_type='text/markdown', entry_points={ @@ -42,5 +53,17 @@ def read(fname): "Source": "https://gitlab.com/Cyb3r-Jak3/MetaStalk/-/tree/master", "CI": "https://gitlab.com/Cyb3r-Jak3/MetaStalk/pipelines" }, - + classifiers=[ + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + "Operating System :: OS Independent", + "Natural Language :: English", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3 :: Only", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Utilities" + ] ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e10e4ff --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Needed for unittests""" diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..270a9f9 --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,75 @@ +"""Tests for MetaStalk. +""" +import unittest +from argparse import Namespace +import os + +from MetaStalk import main + + +class MetaStalkTests(unittest.TestCase): + """MetStalkTests. + + Test Suite for MetaStalk + + """ + + def test_empty_path(self): + """Shows result for no path input.""" + with self.assertRaises(FileNotFoundError): + main.start() + + def test_directory(self): + """Results for a directory. + """ + arguments = Namespace( + files=['./ExamplePhotos/'], + alphabetic=False, + loglevel=30, test=True, no_open=True, + export=None) + metastalk = main.MetaStalk() + self.assertEqual(metastalk.run(arguments), None) + + def test_files(self): + """Results for files.""" + arguments = Namespace( + files=['./ExamplePhotos/22-canon_tags.jpg', './ExamplePhotos/32-lens_data.jpeg'], + alphabetic=True, + no_open=True, + loglevel=30, + test=True, + export=None) + metastalk = main.MetaStalk() + self.assertEqual(metastalk.run(arguments), None) + + def test_html_export(self): + """Test to see html files got exported.""" + arguments = Namespace( + files=['./ExamplePhotos/22-canon_tags.jpg', './ExamplePhotos/32-lens_data.jpeg'], + loglevel=30, + alphabetic=False, + no_open=True, + test=True, + export="html", + output="metastalk_exports") + metastalk = main.MetaStalk() + metastalk.run(arguments) + test_passed = True + filenames = ["Focal", "GPS", "Manufacturer", "Model", "Producer", "Stats", "Timestamp"] + for required_file in filenames: + if not os.path.isfile(f"metastalk_exports/{required_file}.html"): + print(f"missing file {required_file}") + test_passed = False + self.assertTrue(test_passed) + + def test_failed_export(self): + """Test for export fail.""" + arguments = Namespace( + files=['./ExamplePhotos/'], loglevel=30, test=True, + alphabetic=False, + no_open=True, + export="pdf", + output="metastalk_exports") + metastalk = main.MetaStalk() + with self.assertRaises(EnvironmentError): + metastalk.run(arguments) diff --git a/tox.ini b/tox.ini index 64d8c4f..5dc16b9 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,18 @@ logging-format-style=new [MESSAGES CONTROL] -disable=invalid-name, logging-too-many-args, wrong-import-position, import-error +disable=invalid-name, logging-too-many-args + +[FORMAT] + +max-line-length=100 [flake8] -ignore=E402 \ No newline at end of file +max-line-length = 100 + +[pep8] +# Only for Code Climate +max-line-length = 100 + +[pycodestyle] +max-line-length = 100