diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 80edd8b..932afe1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,13 @@ stages: -- test - Code_Checking +- test include: - template: SAST.gitlab-ci.yml - template: Dependency-Scanning.gitlab-ci.yml -python-3.7: +Coverage: image: python:3.7 stage: test coverage: '/TOTAL\s*\d*\s*\d*\s*(\d*)%/' @@ -24,10 +24,7 @@ python-3.7: - ./cc-test-reporter before-build script: - pip install -r requirements.txt --quiet - - pip install pylint flake8 coverage bandit --quiet - - pylint --ignored-classes=_socketobject PyStalk.py ./utils/ ./modules/ - - flake8 PyStalk.py ./utils/ ./modules/ --statistics --show-source - - bandit -r -f json PyStalk.py ./utils/ ./modules/ > bandit.json + - pip install coverage --quiet - coverage run -a --source . ./PyStalk.py ./utils/ExamplePhotos/ -t -d - coverage run -a --source . ./PyStalk.py ./utils/ExamplePhotos/Panasonic_DMC-FZ30.jpg ./utils/ExamplePhotos/DSCN0010.jpg -t -d after_script: @@ -38,6 +35,7 @@ python-3.7: - ./cc-test-reporter after-build .check: + stage: Code_Checking before_script: - pip install -U pip script: @@ -46,21 +44,23 @@ python-3.7: - pylint --ignored-classes=_socketobject PyStalk.py ./utils/ ./modules/ - flake8 PyStalk.py ./utils/ ./modules/ --statistics --show-source - bandit -r -f json PyStalk.py ./utils/ ./modules/ > bandit.json + - time python3 ./PyStalk.py ./utils/ExamplePhotos/ -t -d artifacts: paths: - bandit.json python-3.5: extends: ".check" - stage: Code_Checking image: python:3.5 python-3.6: extends: ".check" - stage: Code_Checking image: python:3.6 +python-3.7: + extends: ".check" + image: python:3.7 + python-3.8: extends: ".check" - stage: Code_Checking image: python:3.8 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9780c0a..5711744 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,21 @@ # Changelog -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +## [v1.3.2] - 2019-12-21 -## [UNRELEASED] +Changes made to testing and PyStalk. No new functionality added. + +### Added + +- Added time reporting for how long it took. ### Changed - Added multiple coverage run. - Readme now used LF line endings. - Modified tests run on code climate. +- Split up the main function in PyStalk to setup and run. +- Changed linting so that it happens before for all python versions. ## [v1.3.1] - 2019-12-16 @@ -92,3 +98,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Geo Chart and Model Chart. - dash page for displaying charts. + +--- +This format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) diff --git a/PyStalk.py b/PyStalk.py index 4f12b9c..f3945ea 100644 --- a/PyStalk.py +++ b/PyStalk.py @@ -1,85 +1,92 @@ -# -*- coding: utf-8 -*- -"""This script get the exif data from photos -and creates graphs from the metadata""" - -import argparse -import os -import logging -import sys -from exif import Image -import utils -import modules - - -def main(): - """ This main function""" - parser = argparse.ArgumentParser(prog="PyStalk", - description="Tool to graph " - "image metadata.") - parser.add_argument('files', nargs='*', default=None, - help='Path of photos to check.') - parser.add_argument('-t', '--test', default=False, action="store_true", - help='Does not show the graphs at the end.') -# Logging function from https://stackoverflow.com/a/20663028 - parser.add_argument('-d', '--debug', help="Sets logging level to DEBUG.", - action="store_const", dest="loglevel", - const=logging.DEBUG, default=logging.WARNING) - parser.add_argument("-v", "--verbose", help="Sets logging level to INFO", - action="store_const", dest="loglevel", - const=logging.INFO) - args = parser.parse_args() - - log = utils.make_logger("PyStalk", args.loglevel) - log.info("Starting up") - - if not args.files: - log.error("WARNING: No path was inputted. " - "This will cause the script to break") - sys.exit(1) - - for path in args.files: - isdir = os.path.isdir(path) - log.debug("Detected path as a directory") - - photos = [] - invalid_photos = [] - - if isdir: - for f in os.listdir(args.files[0]): - with open(os.path.join(args.files[0], f), 'rb') as raw_photo: - try: - exif_image = Image(raw_photo) - if exif_image.has_exif: - photos.append(os.path.join(args.files[0], f)) - else: - invalid_photos.append(os.path.join(args.files[0], f)) - except AssertionError: - log.warning("Error with %s", raw_photo) - else: - for x, item in enumerate(args.files): - with open(item, 'rb') as raw_photo: - try: - exif_image = Image(raw_photo) - if exif_image.has_exif: - photos.append(args.files[x]) - log.debug("%s has exif data", item) - else: - invalid_photos.append(args.files[x]) - except AssertionError: - log.warning("Error with %s", raw_photo) - - plots = { - "STATS": modules.Stats(photos, invalid_photos, log), - "GPS": modules.GPS_Check(photos, log), - "Timestamp": modules.date_time(photos, log), - "Model": modules.PieChart(photos, "Model", log), - "Flash": modules.PieChart(photos, "Flash", log), - "Focal": modules.PieChart(photos, "Focal", log), - "Software": modules.PieChart(photos, "Software", log) - } - - utils.graph(plots, log, args.test) - - -if __name__ == "__main__": - main() +# -*- coding: utf-8 -*- +"""This script get the exif data from photos +and creates graphs from the metadata""" + +import argparse +import os +import logging +import sys +import timeit +from exif import Image +import utils +import modules + +start = timeit.default_timer() + + +def setup(): + """ Sets up PyStalk and parses arguments""" + parser = argparse.ArgumentParser(prog="PyStalk", + description="Tool to graph " + "image metadata.") + parser.add_argument('files', nargs='*', default=None, + help='Path of photos to check.') + parser.add_argument('-t', '--test', default=False, action="store_true", + help='Does not show the graphs at the end.') +# Logging function from https://stackoverflow.com/a/20663028 + parser.add_argument('-d', '--debug', help="Sets logging level to DEBUG.", + action="store_const", dest="loglevel", + const=logging.DEBUG, default=logging.WARNING) + parser.add_argument("-v", "--verbose", help="Sets logging level to INFO", + action="store_const", dest="loglevel", + const=logging.INFO) + args = parser.parse_args() + + log = utils.make_logger("PyStalk", args.loglevel) + log.info("Starting up") + run(args, log) + + +def run(args, log): + """Process files and generates graphs""" + if not args.files: + log.error("WARNING: No path was inputted. " + "This will cause the script to break") + sys.exit(1) + + for path in args.files: + isdir = os.path.isdir(path) + log.debug("Detected path as a directory") + + photos = [] + invalid_photos = [] + + if isdir: + for f in os.listdir(args.files[0]): + with open(os.path.join(args.files[0], f), 'rb') as raw_photo: + try: + exif_image = Image(raw_photo) + if exif_image.has_exif: + photos.append(os.path.join(args.files[0], f)) + else: + invalid_photos.append(os.path.join(args.files[0], f)) + except AssertionError: + log.warning("Error with %s", raw_photo) + else: + for x, item in enumerate(args.files): + with open(item, 'rb') as raw_photo: + try: + exif_image = Image(raw_photo) + if exif_image.has_exif: + photos.append(args.files[x]) + log.debug("%s has exif data", item) + else: + invalid_photos.append(args.files[x]) + except AssertionError: + log.warning("Error with %s", raw_photo) + + plots = { + "STATS": modules.Stats(photos, invalid_photos, log), + "GPS": modules.GPS_Check(photos, log), + "Timestamp": modules.date_time(photos, log), + "Model": modules.PieChart(photos, "Model", log), + "Flash": modules.PieChart(photos, "Flash", log), + "Focal": modules.PieChart(photos, "Focal", log), + "Software": modules.PieChart(photos, "Software", log) + } + + utils.graph(plots, log, start, args.test) + + +if __name__ == "__main__": + setup() diff --git a/Readme.md b/Readme.md index 3e96080..19c062f 100644 --- a/Readme.md +++ b/Readme.md @@ -1,8 +1,8 @@ # PyStalk -[![GitHub](https://img.shields.io/github/license/Cyb3r-Jak3/pystalk?style=flat)](https://github.com/Cyb3r-Jak3/PyStalk/blob/master/LICENSE) +[![GitHub](https://img.shields.io/github/license/Cyb3r-Jak3/pystalk?style=flat)](https://github.com/Cyb3r-Jak3/PyStalk/blob/master/LICENSE) ![Gitlab pipeline status (branch)](https://img.shields.io/gitlab/pipeline/Cyb3r-Jak3/pystalk/master?label=Build&style=flat) -![Gitlab pipeline status (branch)](https://img.shields.io/gitlab/pipeline/Cyb3r-Jak3/pystalk/master?label=Build&style=flat) [![Test Coverage](https://api.codeclimate.com/v1/badges/896b338971314c13a56e/test_coverage)](https://codeclimate.com/github/Cyb3r-Jak3/PyStalk/test_coverage) +[![Test Coverage](https://api.codeclimate.com/v1/badges/896b338971314c13a56e/test_coverage)](https://codeclimate.com/github/Cyb3r-Jak3/PyStalk/test_coverage) [![Maintainability](https://api.codeclimate.com/v1/badges/896b338971314c13a56e/maintainability)](https://codeclimate.com/github/Cyb3r-Jak3/PyStalk/maintainability) ## About @@ -24,7 +24,7 @@ python PyStalk.py #i.e. python PyStalk.py tests/ExamplePhotos/ ``` -## TODO +### TODO - Better web layout - Better sized graphs diff --git a/modules/DateTime.py b/modules/DateTime.py index ac1db98..7c5e97a 100644 --- a/modules/DateTime.py +++ b/modules/DateTime.py @@ -15,7 +15,6 @@ def date_time(photos, log): with open(each, 'rb') as image_file: my_image = Image(image_file) for i, _ in enumerate(types): - log.debug(i) try: types[i].append(getattr(my_image, types_str[i])) log.debug("%s has %s data", each, types_str[i]) diff --git a/utils/web.py b/utils/web.py index 493414c..ac6ed96 100644 --- a/utils/web.py +++ b/utils/web.py @@ -1,11 +1,12 @@ """Uses dash to create a webpage that contain all the graphs""" +import timeit import webbrowser import dash import dash_html_components as html import dash_core_components as dcc -def graph(plots, log, test=False): +def graph(plots, log, start, test=False): """Creates the graphs""" external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css'] app = dash.Dash(__name__, external_stylesheets=external_stylesheets) @@ -13,11 +14,14 @@ def graph(plots, log, test=False): graphs = [] for name, chart in plots.items(): graphs.append(dcc.Graph(id="graph-{}".format(name), figure=chart)) + stop = timeit.default_timer() app.layout = html.Div([ html.H1("PyStalk", style={"textAlign": "center"}), html.H6(html.A('Cyber Jake', href="https://twitter.com/Cyb3r_Jak3"), style={"textAlign": "center"}), html.Div(children=graphs), + html.P("Time Taken = {0:.2f} seconds".format(stop - start), + style={"textAlign": "center"}) ]) if not test: webbrowser.open("http://localhost:8052", new=2) @@ -27,3 +31,4 @@ def graph(plots, log, test=False): log.info("Interput received. Exiting.") else: log.info("Test flag was set. No webpage will be shown.") + log.info("Time Taken = {0:.2f} seconds".format(stop - start))