-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
MetaStalk 2.1.0 ### What does this MR do? #### Added - Ability to pass both directories and individual files. - Unittests for testing. - Footer for run time. - Export feature. - Added metastalk dev and image install. - [Codacy](https://www.codacy.com/) - Two new arguments `--no-open` and `--alphabetic`. - `--no-open` will make it so a new browser tab is not opened. - `--alphabetic` will sort all the charts alphabetically. #### Changed - Created MetaStalk Class. - All titles for charts are centered. See merge request Cyb3r-Jak3/metastalk!14
- Loading branch information
Showing
26 changed files
with
406 additions
and
220 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -129,4 +129,5 @@ dmypy.json | |
| .pyre/ | ||
|
|
||
| # Hide vscode folder | ||
| .vscode/ | ||
| .vscode/ | ||
| metastalk_exports/* | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,9 +2,15 @@ | |
|
|
||
| Thanks for reading this because I am always looking to collaborate with other people to see their ideas. | ||
|
|
||
| If you are looking to submit a pull request then please do so on [GitLab](https://gitlab.com/Cyb3r-Jak3/pystalk) because all development is done there. Any pull request that is opened on GitHub will be closed and I mirror it on GitLab. | ||
| If you are looking to submit a pull request then please do so on [GitLab](https://gitlab.com/Cyb3r-Jak3/MetaStalk) because all development is done there. Any pull request that is opened on GitHub will be closed and I mirror it on GitLab. | ||
|
|
||
| Please open issues on [GitLab](https://gitlab.com/Cyb3r-Jak3/PyStalk/issues). if you do not have a GitLab account, send then please email service desk( [[email protected]](mailto:[email protected]) ). | ||
| Please open issues on [GitLab](https://gitlab.com/Cyb3r-Jak3/MetaStalk/issues). If you do not have a GitLab account, send then please email service desk( [incoming+cyb3r-jak3-metastalk-15263483-issue-@incoming.gitlab.com](mailto:incoming+cyb3r-jak3-metastalk-15263483-issue-@incoming.gitlab.com)). | ||
|
|
||
| To get started you can either clone the repo or use `pip install metastalk[dev]`. | ||
|
|
||
| ## Testing | ||
|
|
||
| Currently the tests at [tests](tests/) cover all the necessary items to make sure the it works. Currently no test exist for testing the image export capabilities as it relies on [orca](https://github.com/plotly/orca) which is not easy to setup tests for. If someone has a way of creating a test with orca, then please submit a pull request and I will gladly approve. | ||
|
|
||
| ## Modules | ||
|
|
||
|
|
@@ -13,10 +19,10 @@ If you are looking to write a new module for a new metadata graph here is they a | |
| - Each module is written in the [modules](modules/) directory and is added to the [\_\_init__.py](modules/__init__.py) in the directory. | ||
| - A list is passed to the modules called `photos` which containing dictionaries with all the metadata points for each photo. | ||
| - The module will find the key that is using and return a figure using plotly. | ||
| - A dictionary of all the plots will be passed to [web.py](utils/web.py) which will use dash to display them. | ||
| - A dictionary of all the plots will be passed to [web.py](utils/web.py) which will use Dash to display them. | ||
|
|
||
| ### Example image dictionary | ||
|
|
||
| ```python | ||
| {'Image width': '1600 pixels', 'Image height': '1200 pixels', 'Image orientation': 'Horizontal (normal)', 'Bits/pixel': '24', 'Pixel format': 'YCbCr', 'Creation date': '2007-11-29 16:16:21', 'Camera aperture': '2.97', 'Camera focal': '2.8', 'Camera exposure': '1/6', 'Camera model': 'Canon PowerShot SD300', 'Camera manufacturer': 'Canon', 'Compression': 'JPEG (Baseline)', 'Thumbnail size': '5922 bytes', 'EXIF version': '0220', 'Date-time original': '2007-11-29 16:16:21', 'Date-time digitized': '2007-11-29 16:16:21', 'Compressed bits per pixel': '3', 'Shutter speed': '2.59', 'Aperture': '2.97', 'Exposure bias': '0', 'Focal length': '5.8', 'Flashpix version': '0100', 'Focal plane width': '7.14e+03', 'Focal plane height': '7.14e+03', 'Comment': 'JPEG quality: 90% (approximate)', 'MIME type': 'image/jpeg', 'Endianness': 'Big endian', 'item': '.\\utils\\ExamplePhotos\\22-canon_tags.jpg'} | ||
| {'Image width': '1600 pixels', 'Image height': '1200 pixels', 'Image orientation': 'Horizontal (normal)', 'Bits/pixel': '24', 'Pixel format': 'YCbCr', 'Creation date': '2007-11-29 16:16:21', 'Camera aperture': '2.97', 'Camera focal': '2.8', 'Camera exposure': '1/6', 'Camera model': 'Canon PowerShot SD300', 'Camera manufacturer': 'Canon', 'Compression': 'JPEG (Baseline)', 'Thumbnail size': '5922 bytes', 'EXIF version': '0220', 'Date-time original': '2007-11-29 16:16:21', 'Date-time digitized': '2007-11-29 16:16:21', 'Compressed bits per pixel': '3', 'Shutter speed': '2.59', 'Aperture': '2.97', 'Exposure bias': '0', 'Focal length': '5.8', 'Flashpix version': '0100', 'Focal plane width': '7.14e+03', 'Focal plane height': '7.14e+03', 'Comment': 'JPEG quality: 90% (approximate)', 'MIME type': 'image/jpeg', 'Endianness': 'Big endian', 'item': '.\\ExamplePhotos\\22-canon_tags.jpg'} | ||
| ``` | ||
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1 @@ | ||
| graft MetaStalk/utils/assets | ||
| prune MetaStalk/tests | ||
| recursive-include MetaStalk/utils/assets * |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| """MetaStalk | ||
| """MetaStalk. | ||
| --- | ||
| MetaStalk is a program that parse image metadata and creates charts from it. | ||
| """ | ||
| __version__ = "v2.0.0" | ||
| __version__ = "v2.1.0" | ||
| __author__ = "Cyb3r Jak3" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,135 +1,117 @@ | ||
| # -*- coding: utf-8 -*- | ||
| """This script get the exif data from photos | ||
| and creates graphs from the metadata""" | ||
| """Main function of MetaStalk. | ||
| Run get any metadata from photos | ||
| and creates graphs from the metadata using MetaStalk.Modules""" | ||
| import argparse | ||
| from collections import OrderedDict | ||
| import os | ||
| import logging | ||
| import timeit | ||
| from hachoir.parser import createParser | ||
| from hachoir.metadata import extractMetadata | ||
|
|
||
|
|
||
| from hachoir.core import config as hachoirconfig | ||
| import MetaStalk.utils as utils | ||
| import MetaStalk.modules as modules | ||
|
|
||
| hachoirconfig.quiet = True | ||
|
|
||
| t_start = timeit.default_timer() | ||
|
|
||
| class MetaStalk(): | ||
| """MetaStalk. | ||
| --- | ||
| Main Class for all MetaStalk work | ||
| """ | ||
| def __init__(self): | ||
| self.log = logging.getLogger("MetaStalk") | ||
| self.t_start = timeit.default_timer() | ||
| self.valid, self.invalid = [], [] | ||
| self.plots = {} | ||
|
|
||
| def run(self, args): | ||
| """Run | ||
| Process files and generates graphs | ||
| Arguments: | ||
| args {argparse.Namespace} -- The arguments from start() | ||
| log {logging.Logger} -- Logger | ||
| """ | ||
| for path in args.files: | ||
| if os.path.isdir(path): | ||
| self.log.debug("Detected path as a directory") | ||
| for item in os.listdir(path): | ||
| item_path = os.path.join(path, item) | ||
| self.file_search(item_path) | ||
| else: | ||
| self.file_search(path) | ||
|
|
||
| self.plots = { | ||
| "Stats": modules.stats(self.valid, self.invalid), | ||
| "GPS": modules.gps_check(self.valid), | ||
| "Timestamp": modules.date_time(self.valid), | ||
| "Model": modules.pie_chart(self.valid, "Camera model"), | ||
| "Manufacturer": modules.pie_chart(self.valid, "Camera manufacturer"), | ||
| "Focal": modules.pie_chart(self.valid, "Camera focal"), | ||
| "Producer": modules.pie_chart(self.valid, "Producer") | ||
| } | ||
| if args.alphabetic: | ||
| self.plots = OrderedDict(sorted(self.plots.items())) | ||
| if args.export: | ||
| utils.export(args.export, args.output, self.plots) | ||
| utils.graph(self.plots, self.t_start, args.test, args.no_open) | ||
|
|
||
| def file_search(self, parse_file: str): | ||
| """file_search | ||
| Used to append files if the path is not a directory. | ||
| Arguments | ||
| files {str} -- Name of the file to parse. | ||
| """ | ||
| parser = createParser(parse_file) | ||
| try: | ||
| metadata = extractMetadata(parser).exportDictionary()["Metadata"] | ||
| metadata["item"] = parse_file | ||
| self.valid.append(metadata) | ||
| self.log.debug("%s has metadata", parse_file) | ||
| except AttributeError: | ||
| self.invalid.append(parse_file) | ||
| self.log.debug("%s has no metadata data", parse_file) | ||
|
|
||
| def start(): | ||
| """start | ||
|
|
||
| Sets up MetaStalk and parses arguments. | ||
| Will raise an IOError if no path is passed. | ||
| def start(): | ||
| """ | ||
| Function needed to start MetaStalk | ||
| """ | ||
| parser = argparse.ArgumentParser(prog="MetaStalk", | ||
| description="Tool to graph " | ||
| "image metadata.") | ||
| parser.add_argument('files', nargs='*', default=None, | ||
| help='Path of photos to check.') | ||
| parser.add_argument('-t', '--test', default=False, action="store_true", | ||
| help='Does not show the graphs at the end.') | ||
| # Logging function from https://stackoverflow.com/a/20663028 | ||
| parser.add_argument("-a", "--alphabetic", help="Sorts charts in alphabetical order rather than" | ||
| " the default order", default=False, action="store_true") | ||
| parser.add_argument('-d', '--debug', help="Sets logging level to DEBUG.", | ||
| action="store_const", dest="loglevel", | ||
| const=logging.DEBUG, default=logging.WARNING) | ||
| parser.add_argument("-e", "--export", choices=["pdf", "svg", "png", "html"], | ||
| help="Exports the graphs rather than all on one webpage") | ||
| parser.add_argument("--no-open", help="Will only start the server and not open the browser" | ||
| " to view it", default=False, action="store_true") | ||
| parser.add_argument("-o", "--output", default="metastalk_exports", | ||
| help="The name of the directory to output exports to. " | ||
| "Will be created if it does not exist. " | ||
| "Defaults to metastalk_exports.") | ||
| parser.add_argument('-t', '--test', default=False, action="store_true", | ||
| help='Does not show the graphs at the end.') | ||
| parser.add_argument("-v", "--verbose", help="Sets logging level to INFO", | ||
| action="store_const", dest="loglevel", | ||
| const=logging.INFO) | ||
| args = parser.parse_args() | ||
|
|
||
| log = utils.make_logger("MetaStalk", args.loglevel) | ||
| log.info("MetaStalk starting") | ||
| if not args.files: | ||
| log.error("ERROR: No path was inputted.") | ||
| raise IOError("No path was inputted.") | ||
| run(args, log) | ||
|
|
||
|
|
||
| def run(args, log: logging.Logger): | ||
| """run | ||
| Process files and generates graphs | ||
| Arguments: | ||
| args {argparse.Namespace} -- The arguments from start() | ||
| log {logging.Logger} -- Logger | ||
| """ | ||
| for path in args.files: | ||
| isdir = os.path.isdir(path) | ||
| log.debug("Detected path as a directory") | ||
|
|
||
| if isdir: | ||
| photos, invalid_photos = directory_search(args.files[0], log) | ||
| else: | ||
| photos, invalid_photos = file_search(args.files, log) | ||
|
|
||
| plots = { | ||
| "STATS": modules.Stats(photos, invalid_photos), | ||
| "GPS": modules.GPS_Check(photos), | ||
| "Timestamp": modules.date_time(photos), | ||
| "Model": modules.PieChart(photos, "Camera model"), | ||
| "Manufacturer": modules.PieChart(photos, "Camera manufacturer"), | ||
| "Focal": modules.PieChart(photos, "Camera focal"), | ||
| "Producer": modules.PieChart(photos, "Producer") | ||
| } | ||
|
|
||
| utils.graph(plots, t_start, args.test) | ||
|
|
||
|
|
||
| def directory_search(files: list, log: logging.Logger) -> (list, list): | ||
| """directory_search | ||
| Used to append all files in a directory from args. | ||
| Arguments: | ||
| files {list} -- List of directories to parse | ||
| log {logging.Logger} -- Logger | ||
| Returns: | ||
| valid, invalid -- List of photos with metadata and ones without | ||
| """ | ||
| valid, invalid = [], [] | ||
| for item in os.listdir(files): | ||
| item_path = os.path.join(files, item) | ||
| parser = createParser(item_path) | ||
| metadata = extractMetadata(parser).exportDictionary()["Metadata"] | ||
| if metadata: | ||
| metadata["item"] = item_path | ||
| valid.append(metadata) | ||
| log.debug("%s has metadata", item) | ||
| else: | ||
| metadata["item"] = item_path | ||
| invalid.append(metadata) | ||
| log.debug("%s has no metadata", item) | ||
| return valid, invalid | ||
|
|
||
|
|
||
| def file_search(files: list, log: logging.Logger) -> (list, list): | ||
| """file_search | ||
| Used to append files if the path is not a directory. | ||
| Arguments: | ||
| files {list} -- List of files to parse | ||
| log {logging.Logger} -- Logger | ||
| Returns: | ||
| valid, invalid -- List of photos with metadata and ones without | ||
| """ | ||
| valid, invalid = [], [] | ||
| for _, item in enumerate(files): | ||
| parser = createParser(item) | ||
| metadata = extractMetadata(parser).exportDictionary()["Metadata"] | ||
| if metadata: | ||
| metadata["item"] = item | ||
| valid.append(metadata) | ||
| log.debug("%s has metadata", item) | ||
| else: | ||
| metadata["item"] = item | ||
| invalid.append(metadata) | ||
| log.debug("%s has no metadata data", item) | ||
| return valid, invalid | ||
|
|
||
|
|
||
| start() | ||
| raise FileNotFoundError("No path was inputted.") | ||
| metastalk = MetaStalk() | ||
| metastalk.run(args) |
Oops, something went wrong.