From ab8ba5e16c128569f065c09600b327ec9e928c54 Mon Sep 17 00:00:00 2001 From: Giovanni Date: Sun, 21 Apr 2024 17:19:27 +0200 Subject: [PATCH] upload --- .gitignore | 447 ++++++++++++++++++++++++++++++++++ LICENSE | 21 ++ README.md | 121 +++++++++ dockleaner.py | 191 +++++++++++++++ logic/dockerfile_obj.py | 120 +++++++++ logic/smell.py | 7 + requirements.txt | 6 + smell_solvers/rules/DL3003.py | 66 +++++ smell_solvers/rules/DL3006.py | 35 +++ smell_solvers/rules/DL3007.py | 32 +++ smell_solvers/rules/DL3008.py | 170 +++++++++++++ smell_solvers/rules/DL3009.py | 85 +++++++ smell_solvers/rules/DL3014.py | 46 ++++ smell_solvers/rules/DL3015.py | 32 +++ smell_solvers/rules/DL3020.py | 29 +++ smell_solvers/rules/DL3025.py | 53 ++++ smell_solvers/rules/DL3042.py | 36 +++ smell_solvers/rules/DL3047.py | 35 +++ smell_solvers/rules/DL3048.py | 56 +++++ smell_solvers/rules/DL3059.py | 56 +++++ smell_solvers/rules/DL4000.py | 35 +++ smell_solvers/rules/DL4006.py | 57 +++++ smell_solvers/smell_solver.py | 57 +++++ smell_solvers/strategy.py | 15 ++ temp/README.md | 0 test/.gitignore | 2 + test/DL3003/Example1 | 2 + test/DL3003/Example2 | 2 + test/DL3003/Example3 | 3 + test/DL3003/Example4 | 214 ++++++++++++++++ test/DL3003/Example5 | 8 + test/DL3003/Example6 | 3 + test/DL3006/Example1 | 1 + test/DL3006/Example2 | 1 + test/DL3006/Example3 | 1 + test/DL3006/Example4 | 3 + test/DL3006/Example5 | 3 + test/DL3008/Example1 | 3 + test/DL3008/Example10 | 124 ++++++++++ test/DL3008/Example11 | 13 + test/DL3008/Example2 | 7 + test/DL3008/Example3 | 18 ++ test/DL3008/Example4 | 4 + test/DL3008/Example5 | 11 + test/DL3008/Example6 | 34 +++ test/DL3008/Example7 | 5 + test/DL3008/Example8 | 32 +++ test/DL3008/Example9 | 23 ++ test/DL3009/Example1 | 2 + test/DL3009/Example10 | 59 +++++ test/DL3009/Example2 | 5 + test/DL3009/Example3 | 13 + test/DL3009/Example4 | 19 ++ test/DL3009/Example5 | 6 + test/DL3009/Example6 | 6 + test/DL3009/Example7 | 18 ++ test/DL3009/Example8 | 10 + test/DL3009/Example9 | 10 + test/DL3015/Example1 | 2 + test/DL3015/Example2 | 4 + test/DL3015/Example3 | 3 + test/DL3015/Example4 | 4 + test/DL3015/Example5 | 4 + test/DL3015/Example6 | 4 + test/DL3015/Example7 | 9 + test/DL3015/Example8 | 9 + test/DL3020/Example1 | 3 + test/DL3020/Example2 | 9 + test/DL3025/Example1 | 3 + test/DL3025/Example2 | 3 + test/DL3025/Example3 | 4 + test/DL3025/Example4 | 12 + test/DL3025/Example5 | 3 + test/DL3025/Example6 | 3 + test/DL3025/Example7 | 3 + test/DL3025/Example8 | 9 + test/DL3048/Example1 | 5 + test/DL3048/Example2 | 3 + test/DL3059/Example1 | 7 + test/DL3059/Example2 | 11 + test/DL3059/Example3 | 8 + test/DL3059/Example4 | 26 ++ test/DL3059/Example5 | 16 ++ test/DL3059/Example6 | 11 + test/DL3059/Example7 | 26 ++ test/DL3059/Example8 | 4 + test/DL4000/Example1 | 11 + test/DL4000/Example2 | 12 + test/DL4000/Example3 | 8 + test/DL4000/Example4 | 7 + test/DL4000/Example5 | 8 + test/DL4000/Example6 | 12 + test/DL4006/.directory | 5 + test/DL4006/Example1 | 3 + test/DL4006/Example2 | 4 + test/DL4006/Example3 | 3 + test/DL4006/Example4 | 3 + test/DL4006/Example5 | 3 + test/test.rb | 52 ++++ utils/cache_handler.py | 56 +++++ utils/common.py | 36 +++ utils/docker_utils.py | 176 +++++++++++++ utils/dockerhub_api.py | 148 +++++++++++ utils/launchpad_api.py | 126 ++++++++++ 104 files changed, 3354 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 dockleaner.py create mode 100644 logic/dockerfile_obj.py create mode 100644 logic/smell.py create mode 100644 requirements.txt create mode 100644 smell_solvers/rules/DL3003.py create mode 100644 smell_solvers/rules/DL3006.py create mode 100644 smell_solvers/rules/DL3007.py create mode 100644 smell_solvers/rules/DL3008.py create mode 100644 smell_solvers/rules/DL3009.py create mode 100644 smell_solvers/rules/DL3014.py create mode 100644 smell_solvers/rules/DL3015.py create mode 100644 smell_solvers/rules/DL3020.py create mode 100644 smell_solvers/rules/DL3025.py create mode 100644 smell_solvers/rules/DL3042.py create mode 100644 smell_solvers/rules/DL3047.py create mode 100644 smell_solvers/rules/DL3048.py create mode 100644 smell_solvers/rules/DL3059.py create mode 100644 smell_solvers/rules/DL4000.py create mode 100644 smell_solvers/rules/DL4006.py create mode 100644 smell_solvers/smell_solver.py create mode 100644 smell_solvers/strategy.py create mode 100644 temp/README.md create mode 100644 test/.gitignore create mode 100644 test/DL3003/Example1 create mode 100644 test/DL3003/Example2 create mode 100644 test/DL3003/Example3 create mode 100644 test/DL3003/Example4 create mode 100644 test/DL3003/Example5 create mode 100644 test/DL3003/Example6 create mode 100644 test/DL3006/Example1 create mode 100644 test/DL3006/Example2 create mode 100644 test/DL3006/Example3 create mode 100644 test/DL3006/Example4 create mode 100644 test/DL3006/Example5 create mode 100644 test/DL3008/Example1 create mode 100644 test/DL3008/Example10 create mode 100644 test/DL3008/Example11 create mode 100644 test/DL3008/Example2 create mode 100644 test/DL3008/Example3 create mode 100644 test/DL3008/Example4 create mode 100644 test/DL3008/Example5 create mode 100644 test/DL3008/Example6 create mode 100644 test/DL3008/Example7 create mode 100644 test/DL3008/Example8 create mode 100644 test/DL3008/Example9 create mode 100644 test/DL3009/Example1 create mode 100644 test/DL3009/Example10 create mode 100644 test/DL3009/Example2 create mode 100644 test/DL3009/Example3 create mode 100644 test/DL3009/Example4 create mode 100644 test/DL3009/Example5 create mode 100644 test/DL3009/Example6 create mode 100644 test/DL3009/Example7 create mode 100644 test/DL3009/Example8 create mode 100644 test/DL3009/Example9 create mode 100644 test/DL3015/Example1 create mode 100644 test/DL3015/Example2 create mode 100644 test/DL3015/Example3 create mode 100644 test/DL3015/Example4 create mode 100644 test/DL3015/Example5 create mode 100644 test/DL3015/Example6 create mode 100644 test/DL3015/Example7 create mode 100644 test/DL3015/Example8 create mode 100644 test/DL3020/Example1 create mode 100644 test/DL3020/Example2 create mode 100644 test/DL3025/Example1 create mode 100644 test/DL3025/Example2 create mode 100644 test/DL3025/Example3 create mode 100644 test/DL3025/Example4 create mode 100644 test/DL3025/Example5 create mode 100644 test/DL3025/Example6 create mode 100644 test/DL3025/Example7 create mode 100644 test/DL3025/Example8 create mode 100644 test/DL3048/Example1 create mode 100644 test/DL3048/Example2 create mode 100644 test/DL3059/Example1 create mode 100644 test/DL3059/Example2 create mode 100644 test/DL3059/Example3 create mode 100644 test/DL3059/Example4 create mode 100644 test/DL3059/Example5 create mode 100644 test/DL3059/Example6 create mode 100644 test/DL3059/Example7 create mode 100644 test/DL3059/Example8 create mode 100644 test/DL4000/Example1 create mode 100644 test/DL4000/Example2 create mode 100644 test/DL4000/Example3 create mode 100644 test/DL4000/Example4 create mode 100644 test/DL4000/Example5 create mode 100644 test/DL4000/Example6 create mode 100644 test/DL4006/.directory create mode 100644 test/DL4006/Example1 create mode 100644 test/DL4006/Example2 create mode 100644 test/DL4006/Example3 create mode 100644 test/DL4006/Example4 create mode 100644 test/DL4006/Example5 create mode 100644 test/test.rb create mode 100644 utils/cache_handler.py create mode 100644 utils/common.py create mode 100644 utils/docker_utils.py create mode 100644 utils/dockerhub_api.py create mode 100644 utils/launchpad_api.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58ee29e --- /dev/null +++ b/.gitignore @@ -0,0 +1,447 @@ +.cache +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Created by https://www.toptal.com/developers/gitignore/api/pycharm+all,macos,visualstudiocode,python +# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm+all,macos,visualstudiocode,python + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### PyCharm+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### PyCharm+all Patch ### +# Ignore everything but code style settings and run configurations +# that are supposed to be shared within teams. + +.idea/* + +!.idea/codeStyles +!.idea/runConfigurations + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/pycharm+all,macos,visualstudiocode,python \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..390c1dd --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Giovanni Rosa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b085792 --- /dev/null +++ b/README.md @@ -0,0 +1,121 @@ +# Dockleaner + +Rule-based refactoring tool for fixing Dockerfile smells detected by [hadolint](https://github.com/hadolint/hadolint). + +A detailed description of the tool and its evaluation can be found in the [preprint of the original article](https://giovannirosa.com/assets/pdf/rosa2024fixingsmells.pdf). + +## Setup + +The first step is to install [hadolint](https://github.com/hadolint/hadolint/releases/tag/v2.12.0). +Next, clone the repository and install the requirements: +``` +conda create -n dockleaner python=3.7 +conda activate dockleaner + +pip install -r requirements.txt +``` + +Dockleaner was tested on `macOS` and `Arch Linux` using `Python 3.7`. + +## Usage + +Run `python3 dockleaner.py --help` to see the following help message: + +``` +optional arguments: + -h, --help show this help message and exit + --cache, -c If selected, clears the cache of pulled image + --overwrite, -o Set TRUE to overwrite the target Dockerfile after the + fix + --ignore rules_to_ignore [rules_to_ignore ...], -i rules_to_ignore [rules_to_ignore ...] + The rules that the solver must ignore + --rule rules_to_fix [rules_to_fix ...] + Specify one or more specific rules to fix + +required arguments: + --path filepath, -p filepath + The path of the Dockerfile + --last-edit dockerfile_date, -d dockerfile_date + Last edit date of the given Dockerfile. Format "YYYY- + MM-DD". +``` + +Thus, the command: +``` +python3 -u dockleaner.py -p Dockerfile -d "2023-04-12" --rule "DL3008" --overwrite +``` +will fix the Dockerfile `Dockerfile` by pinning versions for apt packages and overwriting the file. + +## Supported Smells + +- :calendar: [DL3000](https://github.com/hadolint/hadolint/wiki/DL3000) +- :calendar: [DL3002](https://github.com/hadolint/hadolint/wiki/DL3002) +- :white_check_mark: [DL3003](https://github.com/hadolint/hadolint/wiki/DL3003) +- :calendar: [DL3004](https://github.com/hadolint/hadolint/wiki/DL3004) +- :calendar: [DL3005](https://github.com/hadolint/hadolint/wiki/DL3005) +- :white_check_mark: [DL3006](https://github.com/hadolint/hadolint/wiki/DL3006) +- :white_check_mark: [DL3007](https://github.com/hadolint/hadolint/wiki/DL3007) +- :white_check_mark: [DL3008](https://github.com/hadolint/hadolint/wiki/DL3008) +- :white_check_mark: [DL3009](https://github.com/hadolint/hadolint/wiki/DL3009) +- :calendar: [DL3013](https://github.com/hadolint/hadolint/wiki/DL3013) +- :ok: [DL3014](https://github.com/hadolint/hadolint/wiki/DL3014) +- :white_check_mark: [DL3015](https://github.com/hadolint/hadolint/wiki/DL3015) +- :calendar: [DL3016](https://github.com/hadolint/hadolint/wiki/DL3016) +- :white_check_mark: [DL3020](https://github.com/hadolint/hadolint/wiki/DL3020) +- :calendar: [DL3022](https://github.com/hadolint/hadolint/wiki/DL3022) +- :calendar: [DL3024](https://github.com/hadolint/hadolint/wiki/DL3024) +- :white_check_mark: [DL3025](https://github.com/hadolint/hadolint/wiki/DL3025) +- :calendar: [DL3027](https://github.com/hadolint/hadolint/wiki/DL3027) +- :calendar: [DL3028](https://github.com/hadolint/hadolint/wiki/DL3028) +- :calendar: [DL3029](https://github.com/hadolint/hadolint/wiki/DL3029) +- :ok: [DL3042](https://github.com/hadolint/hadolint/wiki/DL3042) +- :calendar: [DL3045](https://github.com/hadolint/hadolint/wiki/DL3045) +- :ok: [DL3047](https://github.com/hadolint/hadolint/wiki/DL3047) +- :white_check_mark: [DL3048](https://github.com/hadolint/hadolint/wiki/DL3048) +- :white_check_mark: [DL3059](https://github.com/hadolint/hadolint/wiki/DL3059) +- :white_check_mark: [DL4000](https://github.com/hadolint/hadolint/wiki/DL4000) +- :calendar: [DL4001](https://github.com/hadolint/hadolint/wiki/DL4001) +- :calendar: [DL4003](https://github.com/hadolint/hadolint/wiki/DL4003) +- :white_check_mark: [DL4006](https://github.com/hadolint/hadolint/wiki/DL4006) + +**Legend:** + +- :white_check_mark: Supported +- :ok: Supported, but not validated via pull requests +- :pushpin: To be implemented +- :calendar: For future implementation + +## License + +The project is licensed under the MIT License. See [LICENSE](LICENSE) for details. + +## Acknowledgements + +The first prototype of Dockleaner was developed by [Alessandro Giagnorio](https://github.com/Devy99). Withouth his work, this project would not have been possible. +The prototype has been extended and improved by [Simone Scalabrino](https://github.com/intersimone999) and [Giovanni Rosa](https://github.com/grosa1) which is currently the maintainer of the project. + +## How to Cite + +The proposal of the study, which includes the tool and its evaulation, was first presented at ICSME'22 - Registered Reports Track. +``` +@article{rosa2022fixing, + title={Fixing dockerfile smells: An empirical study}, + author={Rosa, Giovanni and Scalabrino, Simone and Oliveto, Rocco}, + journal={arXiv preprint arXiv:2208.09097}, + year={2022} +} +``` + +Next, the complete study has been accepted at Empirical Software Engineering (EMSE). +``` +@article{rosa2024fixingsmells, + author = {Giovanni Rosa and + Federico Zappone and + Simone Scalabrino and + Rocco Oliveto}, + title = {Fixing Dockerfile Smells: An Empirical Study}, + journal = {Empirical Software Engineering}, + year = {2024}, + note = {To appear} +} +``` diff --git a/dockleaner.py b/dockleaner.py new file mode 100644 index 0000000..f663760 --- /dev/null +++ b/dockleaner.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import argparse +import datetime +import difflib +import logging +from pyclbr import Class +from typing import List + +import docker + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.smell_solver import SmellSolver +from utils.cache_handler import clear_cache + +logging.basicConfig(level=logging.INFO, format='%(asctime)s :: %(levelname)s :: %(message)s') +logger = logging.getLogger(__name__.split('.')[0]) +logger.setLevel(logging.DEBUG) + + +def get_argparser() -> argparse.ArgumentParser: + """ + Get the configured argument parser + """ + + parser = argparse.ArgumentParser(description='Fix Dockerfile smells') + # todo: check cache + parser.add_argument('--cache', '-c', + action='store_true', + dest='cache', + required=False, + help='If selected, clears the cache of pulled image') + parser.add_argument('--overwrite', '-o', + action='store_true', + dest='overwrite', + required=False, + help='Set TRUE to overwrite the target Dockerfile after the fix') + parser.add_argument('--ignore', '-i', + metavar='rules_to_ignore', + dest='ignored', + nargs='+', + required=False, + help='The rules that the solver must ignore') + parser.add_argument('--rule', + metavar='rules_to_fix', + dest='to_fix', + nargs='+', + required=False, + help='Specify one or more specific rules to fix') + + required = parser.add_argument_group('required arguments') + required.add_argument('--path', '-p', + metavar='filepath', + dest='path', + required=True, + help='The path of the Dockerfile') +# todo: if not date provided, use current as default + required.add_argument('--last-edit', '-d', + metavar='dockerfile_date', + dest='date', + required=True, + type=date_string, + help='Last edit date of the given Dockerfile. Format "YYYY-MM-DD".') + + return parser + + +def date_string(s: str) -> datetime: + """ + Get date from given date string + :param s: date string in the format 'YYYY-MM-DD' + :return: datetime.datetime(YYYY, MM, DD, 0, 0, 0) + """ + + try: + return datetime.datetime.strptime(s, "%Y-%m-%d") + except ValueError: + msg = f"Not a valid date (Expected format, 'YYYY-MM-DD'): '{s}'." + raise argparse.ArgumentTypeError(msg) + + +def get_class(kls: str) -> Class: + """ + Retrieve the instance of a specific class + :param kls: class filepath + :return: a Class object + """ + parts = kls.split('.') + module = ".".join(parts[:-1]) + m = __import__(module) + for comp in parts[1:]: + m = getattr(m, comp) + return m + + +def fix_dockerfile(dockerfile: Dockerfile, ignored_rules: List) -> None: + """ + Fix the given Dockerfile + :param dockerfile: Dockerfile to fix + :param ignored_rules: list of rules to not fix + """ + RULES_PATH = 'smell_solvers.rules.' + solver = SmellSolver() + + keys = list(dockerfile.smells_dict.keys()) + fixed_lines = [] + dict_changed = True + while dict_changed: + dict_changed = False + + for pos in keys: + smells = dockerfile.smells_dict[pos] + for smell in smells: + if solver.strategy_exists(smell.code) and smell.code not in ignored_rules: + logger.info(f'Fixing smell {smell.code} on line {smell.line}') + # Dynamically choose the strategy + solver.strategy = get_class(RULES_PATH + smell.code + '.' + smell.code)() + solver.fix_smell(dockerfile, pos) + fixed_lines.append(smell.line) + + # Check if dictionary line position changed after fix + if dockerfile.lines_changed: + keys = list(set(dockerfile.smells_dict.keys()) - set(fixed_lines)) + dict_changed = True + dockerfile.lines_changed = False + break + + if dict_changed: + break + + +def produce_dockerfile(dockerfile: Dockerfile, overwrite: bool) -> None: + """ + Generate the fixed Dockerfile + :param dockerfile: Dockerfile to produce + :param overwrite: boolean that represent the need to overwrite or not the original file + """ + filepath = dockerfile.filepath + '-fixed' if not overwrite else dockerfile.filepath + + with open(filepath, 'w') as file: + file.writelines(dockerfile.lines) + + if not overwrite: + logger.info(f'You can find the resulting Dockerfile at: {filepath}') + + +def produce_log(filepath) -> None: + """ + Generate the log of Dockerfile fixes + :param filepath: path of the original Dockerfile + """ + original_dockerfile = open(filepath).readlines() + fixed_dockerfile = open(filepath + '-fixed').readlines() + + diff = difflib.HtmlDiff().make_file(original_dockerfile, fixed_dockerfile, filepath, filepath + '-fixed', charset='utf-8') + with open(filepath + '-log.html', 'w', encoding='utf8') as file: + file.write(diff) + + +if __name__ == '__main__': + try: + docker.from_env().info() + except: + logger.error("Docker daemon is not available. Please install and start Docker before running the tool.") + + parser = get_argparser() + args = parser.parse_args() + + if args.cache: + clear_cache() + + fix_and_overwrite = True if args.overwrite else False + + ignored_rules = list() + if args.to_fix: + logger.info("Fixing only: %s", args.to_fix) + ignored_rules = [r for r in SmellSolver()._available_strategies if r not in args.to_fix] + elif args.ignored: + logger.info("Ignoring rules: %s", args.ignored) + ignored_rules = args.ignored + + dockerfile = Dockerfile(args.path, args.date) + + if dockerfile.smells_dict: + fix_dockerfile(dockerfile, ignored_rules) + produce_dockerfile(dockerfile, fix_and_overwrite) + + if not fix_and_overwrite: + produce_log(args.path) + else: + logger.info(f'Your Dockerfile has no smells.') diff --git a/logic/dockerfile_obj.py b/logic/dockerfile_obj.py new file mode 100644 index 0000000..afd5401 --- /dev/null +++ b/logic/dockerfile_obj.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import logging +import os +import sys +from datetime import datetime +from typing import List, Dict +import json +import dockerfile as dockerfile_parser +import subprocess +from collections import defaultdict +from pykson import Pykson +from logic.smell import Smell +import ntpath + + +class InvalidDockerfileError(Exception): + """Raised when the selected file is not a Dockerfile""" + pass + + +class Dockerfile: + def __init__(self, filepath: str, last_edit: datetime): + self._filepath = filepath + self._filename = self.__get_filename(filepath) + self._last_edit = last_edit + self._lines = self.__get_lines(filepath) + self._parsed_lines = self.__get_parsed_lines(filepath) + self._lines_changed = False + self._smells_dict = self.__check_smells(filepath) + + @property + def lines(self) -> List[str]: + return self._lines + + @property + def parsed_lines(self) -> Dict: + return self._parsed_lines + + @property + def lines_changed(self) -> bool: + return self._lines_changed + + @lines_changed.setter + def lines_changed(self, changed) -> None: + self._lines_changed = changed + + @property + def filepath(self) -> str: + return self._filepath + + @property + def last_edit(self) -> datetime: + return self._last_edit + + @property + def smells_dict(self) -> dict: + return self._smells_dict + + def update_smells_dict(self, ignore: List = None) -> None: + """ + Update the smells dictionary. + Call it when the lines number of the Dockerfile change. + """ + filepath = self.__produce_temp_dockerfile() + # refresh parsed lines + self._parsed_lines = self.__get_parsed_lines(filepath) + self._smells_dict = self.__check_smells(filepath, ignore) + os.remove(filepath) + self._lines_changed = True + + def __produce_temp_dockerfile(self) -> str: + filepath = sys.path[0] + f'/temp/{self._filename}' + with open(filepath, 'w') as file: + for line in self._lines: + file.write(line) + return filepath + + def __get_filename(self, filepath: str) -> str: + head, tail = ntpath.split(filepath) + return tail or ntpath.basename(head) + + def __get_lines(self, filepath: str) -> List[str]: + with open(filepath, encoding="utf8") as f: + return f.readlines() + + def __get_parsed_lines(self, filepath: str) -> Dict[str, dockerfile_parser.Command]: + parsed_dict = dict() + try: + for cmd in dockerfile_parser.parse_file(filepath): + parsed_dict[cmd.start_line] = cmd + return parsed_dict + except dockerfile_parser.GoParseError as res: + raise InvalidDockerfileError(res) + except dockerfile_parser.GoIOError as res: + raise InvalidDockerfileError(res) + + def __check_smells(self, filepath: str, ignore: List = None) -> Dict: + # Retrieve hadolint result + output = subprocess.run(['hadolint', '-f', 'json', filepath], stdout=subprocess.PIPE) + result = output.stdout.decode('utf-8') + + result_json = json.loads(result) + + # Set default value as an empty list + smells_dict = defaultdict(list) + + # Put smells in a dictionary like line_position => [smells] + for smell_json in result_json: + if smell_json['code'] in ['DL1000', 'DL3061']: + raise InvalidDockerfileError(result) + + if ignore and smell_json['code'] in ignore: + #logging.info("Skipping", smell_json['code']) + continue + + smell = Pykson().from_json(smell_json, Smell, accept_unknown=True) + smells_dict[smell.line].append(smell) + + return smells_dict diff --git a/logic/smell.py b/logic/smell.py new file mode 100644 index 0000000..8e2d03b --- /dev/null +++ b/logic/smell.py @@ -0,0 +1,7 @@ +from pykson import JsonObject, IntegerField, StringField + +class Smell(JsonObject): + line = IntegerField() + code = StringField() + message = StringField() + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..52b6aca --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +pykson==0.9.9.8.6 +requests==2.25.1 +docker==4.4.4 +dockerfile==3.2.0 +typing~=3.7.4.3 +backoff~=2.2.1 \ No newline at end of file diff --git a/smell_solvers/rules/DL3003.py b/smell_solvers/rules/DL3003.py new file mode 100644 index 0000000..c9c8523 --- /dev/null +++ b/smell_solvers/rules/DL3003.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import logging +import re + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy + +logger = logging.getLogger(__name__.split('.')[0]) +logger.setLevel(logging.DEBUG) + + +class DL3003(Strategy): + """ + Rule DL3003 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3003 + + Rationale: + Only use cd in a subshell. Most commands can work with absolute paths and it in most cases not necessary to change + directories. Docker provides the WORKDIR instruction if you really need to change the current working directory. + + Note: + The commands composed with && are not fixed by this rule. + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + lines = dockerfile.lines + + position = smell_pos - 1 + + prev_workdir = None + for line in reversed(lines[:position]): + if line.strip().startswith('WORKDIR'): + prev_workdir = line.replace('WORKDIR', '').strip() + break + + line = lines[position] + if re.match("RUN +CD ", line.strip(), flags=re.IGNORECASE) is not None: + if "&&" not in line: + logger.info("Not &&") + new_workdir = re.sub('RUN +CD', 'WORKDIR', line, count=1, flags=re.IGNORECASE) + # check if the previous WORKDIR is the same as the new one + if not new_workdir.replace('WORKDIR', '').strip() == prev_workdir: + lines[position] = new_workdir + else: + lines[position] = '' + else: + logger.info("Yes &&") + new_workdir = re.sub("RUN +CD", 'WORKDIR', line.split("&&")[0], count=1, flags=re.IGNORECASE) + "\n" + # check if the previous WORKDIR is the same as the new one + if not new_workdir.replace('WORKDIR', '').strip() == prev_workdir: + lines[position] = new_workdir + else: + lines[position] = '' + # check if the RUN command is on the next line + if line.split("&&", 1)[1].strip().endswith("\\"): + lines[position + 1] = "RUN " + lines[position + 1].lstrip() + else: + lines.insert(position + 1, "RUN " + line.split("&&", 1)[1]) + # logger.info(lines) + + dockerfile.update_smells_dict() + else: + logger.error("!!! Cannot solve DL3003 rule. The line does not match the format 'RUN cd ...'") diff --git a/smell_solvers/rules/DL3006.py b/smell_solvers/rules/DL3006.py new file mode 100644 index 0000000..5de46dc --- /dev/null +++ b/smell_solvers/rules/DL3006.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +from utils.dockerhub_api import get_latest_tag +import logging +logger = logging.getLogger(__name__.split('.')[0]) +logger.setLevel(logging.DEBUG) + + +class DL3006(Strategy): + """ + Rule DL3006 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3006 + + Rationale: + "You can never rely on the latest tags to be a specific version." + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + date = dockerfile.last_edit + command_pos = smell_pos - 1 + lines = dockerfile.lines + + image_info = lines[command_pos].replace('\t', ' ', 1).split(' ')[1].strip() + image = image_info.split(':')[0] + tag = get_latest_tag(image) + + if tag: + new_image = image + ':' + tag + lines[command_pos] = lines[command_pos].replace(image_info, new_image, 1) + else: + logger.error(f'!!! Cannot solve DL3006 rule. Cannot found an equivalent version tag for "{image_info}"') diff --git a/smell_solvers/rules/DL3007.py b/smell_solvers/rules/DL3007.py new file mode 100644 index 0000000..1c4897a --- /dev/null +++ b/smell_solvers/rules/DL3007.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +from utils.dockerhub_api import get_latest_tag +import logging +logger = logging.getLogger(__name__.split('.')[0]) +logger.setLevel(logging.DEBUG) + +class DL3007(Strategy): + """ + Rule DL3007 solver + + Hadolint doc: https://github.com/hadolint/hadolint/wiki/DL3007 + + Rationale: + "You can never rely that the latest tags is a specific version." + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + command_pos = smell_pos - 1 + lines = dockerfile.lines + + image_info = lines[command_pos].replace('\t', ' ', 1).split(' ')[1].strip() + image = image_info.split(':')[0] + tag = get_latest_tag(image) + + if tag: + new_image = image + ':' + tag + lines[command_pos] = lines[command_pos].replace(image_info, new_image, 1) + else: + logger.error(f'!!! Cannot solve DL3007 rule. Cannot found an equivalent version tag for "{image_info}"') diff --git a/smell_solvers/rules/DL3008.py b/smell_solvers/rules/DL3008.py new file mode 100644 index 0000000..c2accbf --- /dev/null +++ b/smell_solvers/rules/DL3008.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import logging +import re +from datetime import datetime + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +from utils.docker_utils import get_distro_info, is_valid_dockerfile_command, validate_package +from utils.launchpad_api import get_package_binary_version, pkgs_repo_available + +logger = logging.getLogger(__name__.split('.')[0]) +logger.setLevel(logging.DEBUG) + + +class DL3008(Strategy): + """ + Rule DL3008 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3008 + + Rationale: + "Version pinning forces the build to retrieve a particular version regardless of what’s in the cache. + This technique can also reduce failures due to unanticipated changes in required packages." + + Note: + It actually works only with ubuntu-based images. + TODO: Can be improved by using the package manager of the image to get package versions (apt-cache madison ) + TODO:/bin/bash -c "apt-get update && apt-cache madison \$(apt-cache search '' | sort -d | awk '{print \$1}')" + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + date = dockerfile.last_edit + lines = dockerfile.lines + + # Get nearest FROM + for line in reversed(lines[:smell_pos]): + if line.strip().upper().startswith('FROM'): + image_info = line.strip().split(" ")[1] + + # Get distro info + distro_info = get_distro_info(image_info) + if not distro_info: + logger.error(f'!!! Cannot solve DL3008 rule. OS not supported "{image_info}"') + return + + distro = distro_info.split(':')[0] + serie = distro_info.split(':')[1] + + if not pkgs_repo_available(distro, serie): + logger.error(f'!!! Cannot solve DL3008 rule. Package repositories not available for "{image_info}"') + return + + # Find apt-get position + position = smell_pos - 1 + for line in lines[position:]: + if "apt-get" in line and " install " in line and not line.strip().startswith("#"): + break + position += 1 + + # Find apt-get install position + splitted_line = lines[position].split("apt-get") + install_position = 0 + for splitted_part in splitted_line: + if " install " in splitted_part: + break + else: + install_position += 1 + + # Get packages after install and remove \n from package name + words = splitted_line[install_position].split(" install ")[1].strip().split(" ") + + while True: + word = words.pop(0) + + if word.startswith('"') and word.endswith('"') or word.startswith("'") and word.endswith("'"): + word = word[1:-1] + + if '\t' in word: # Remove tab char and get clean packages + for splitted_word in word.split('\t'): + words.insert(0, splitted_word) + + if self.__is_stop_word(word): + break + elif word.startswith('`'): # Consume ` command execution ` + while len(words) > 0 and not word.endswith('`'): + word = words.pop(0) + + elif word == '\\': + position += 1 + # Escape comments + while len(lines) > position and lines[position].strip().startswith('#'): + position += 1 + + if len(lines) > position: + next_line = lines[position] + words = next_line.strip().split(" ") + elif self.__is_unpinned_package(word): + # Line cleaning + # lines[position] = lines[position].replace("\t", " ") + lines[position] = lines[position].rstrip() + '\n' + pinned = self.__pin_package(image_info, distro, serie, word, date) + lines[position] = lines[position].replace(f"{word}", f"{pinned}", 1) + + if len(words) == 0: + break + + def __is_unpinned_package(self, package_name: str) -> bool: + """ + Check if the given package name is unpinned. + Note: Package names like "dotnet-runtime-6.0" are seen as "pinned" + """ + return ('=' not in package_name and \ + not package_name.startswith('-') and \ + not is_valid_dockerfile_command(package_name) and \ + bool(re.match(r"[a-z,A-Z,0-9,.,+,-]+\Z", package_name)) and \ + package_name != '') + + def __is_stop_word(self, stop_word: str) -> bool: + return ';' in stop_word or '&' in stop_word or is_valid_dockerfile_command(stop_word) + + def __pin_package(self, image_info: str, distro: str, serie: str, package_name: str, date: datetime) -> str: + pinned_package = package_name + version = get_package_binary_version(distro, serie, package_name, date) + + if version: + rnum = 2 + while rnum >= 0: + version = self.__format_version(version, rnum) + if not version: + logger.error(f'!!! Cannot pin version to {package_name}.') + return package_name + + pinned_package = package_name + '=' + version + status = validate_package(image_info, pinned_package) + if status == 100: + rnum -= 1 + else: + logger.info(f'Pinned: {pinned_package}') + break + else: + logger.error(f'!!! Cannot pin version to {package_name}. Package not found.') + + return pinned_package + + def __format_version(self, version: str, rnum: int) -> str: + """ + Format the given version inserting the '*' char at the given rnum position + :param version: version to format + :param rnum: the position to overwrite. If rnum is 2, PATCH release is overwritten, if 1 MINOR release... + I.e. __format_version('2.13.4', 2) returns '2.13.*' (The PATCH release is overwritten) + """ + if rnum <= 0: + # return '*' + return None + + snum = 0 + + # first_symbol = re.findall('[.-:+~]',version)[0] + for count, char in enumerate(version): + if snum == rnum: + return version[:count] + '*' + + if char == '.' or char == '-' or (char == ':' and snum != 0) or char == '+': + snum += 1 + + return version + +# todo fix install multipli su singola linea - example11 \ No newline at end of file diff --git a/smell_solvers/rules/DL3009.py b/smell_solvers/rules/DL3009.py new file mode 100644 index 0000000..3d6a087 --- /dev/null +++ b/smell_solvers/rules/DL3009.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +from utils.docker_utils import is_valid_dockerfile_command, parse_dockerfile_str +from utils.common import parse_line_indent +import re +import logging +logger = logging.getLogger(__name__.split('.')[0]) +logger.setLevel(logging.DEBUG) + + +class DL3009(Strategy): + """ + Rule DL3009 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3009 + + Rationale: + "Cleaning up the apt cache and removing /var/lib/apt/lists helps keep the image size down. + Since the RUN statement starts with apt-get update, the package cache will always be refreshed prior to apt-get + install." + + Note: + Only the last RUN command with apt-get install will be fixed. This avoids to run again apt update if there are + subsequent instructions. + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + # Find latest "RUN apt-get install" position + smelly_command = None + command_end_pos = -1 + for line in dockerfile.parsed_lines.values(): + cmd = line.cmd + value = line.value[0] + if cmd == 'RUN' and \ + "apt-get" in value and " install " in value: + smelly_command = line + command_end_pos = line.end_line - 1 + + if not smelly_command or command_end_pos == -1: + logger.error(f'!!! Cannot solve DL3009 rule. No fixable RUN command with apt-get install detected.') + return + + # Check if apt-get clean and rm -rf /var/lib/apt/lists/* are already present + is_apt_clean_present = False + if 'apt-get clean' in smelly_command.value[0]: + is_apt_clean_present = True + + is_rm_lists_present = False + if re.search(r'rm -[a-zA-Z]+ /var/lib/apt/lists', smelly_command.value[0]): + is_rm_lists_present = True + + if is_apt_clean_present and is_rm_lists_present: + logger.info(f'!!! DL3009 rule already solved. No fix needed.') + return + + line_indent = parse_line_indent(dockerfile.lines[command_end_pos]) + + # determine if we need to add && at the end or beginning for concatenation + if not dockerfile.lines[command_end_pos].startswith('RUN') and not dockerfile.lines[command_end_pos].lstrip().startswith('&&'): + line_end = ' && \\' + '\n' + clean_command = line_indent + 'apt-get clean && \\' + '\n' + rm_command = line_indent + 'rm -rf /var/lib/apt/lists/*' + '\n' + else: + line_end = ' \\' + '\n' + clean_command = line_indent + '&& apt-get clean \\' + '\n' + rm_command = line_indent + '&& rm -rf /var/lib/apt/lists/*' + '\n' + + # Adding line end at the last RUN command + if dockerfile.lines[command_end_pos].strip().endswith(';'): + dockerfile.lines[command_end_pos] = dockerfile.lines[command_end_pos].rstrip()[:-1] + line_end + else: + dockerfile.lines[command_end_pos] = dockerfile.lines[command_end_pos].rstrip() + line_end + + insert_pos = command_end_pos + 1 + if not is_apt_clean_present: + dockerfile.lines.insert(insert_pos, clean_command) + insert_pos += 1 + + if not is_rm_lists_present: + dockerfile.lines.insert(insert_pos, rm_command) + + dockerfile.update_smells_dict() diff --git a/smell_solvers/rules/DL3014.py b/smell_solvers/rules/DL3014.py new file mode 100644 index 0000000..3681ea6 --- /dev/null +++ b/smell_solvers/rules/DL3014.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy + + +class DL3014(Strategy): + """ + Rule DL3014 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3014 + + Rationale: + "Without the --assume-yes option it might be possible that the build breaks without human intervention." + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + lines = dockerfile.lines + + # Find apt-get position + position = smell_pos - 1 + for line in lines[position:]: + if "apt-get" in line and " install " in line and not line.strip().startswith("#"): + break + position += 1 + + # Get apt-get install position + splitted_line = lines[position].split("apt-get") + + install_position = 0 + for splitted_part in splitted_line: + if " install " in splitted_part: + break + else: + install_position += 1 + + # Adding --no-install-recommends option + if not '-y' in splitted_line[install_position]: + option_pos = splitted_line[install_position].find(' install ') + 8 # install chars number + space + splitted_line[install_position] = splitted_line[install_position][:option_pos] + " -y " + splitted_line[ + install_position][ + option_pos:] + + # Update lines + lines[position] = "apt-get".join(splitted_line) diff --git a/smell_solvers/rules/DL3015.py b/smell_solvers/rules/DL3015.py new file mode 100644 index 0000000..ed9c927 --- /dev/null +++ b/smell_solvers/rules/DL3015.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy + + +class DL3015(Strategy): + """ + Rule DL3015 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3015 + + Rationale: + "Avoid installing additional packages that you did not explicitly want." + + Note: + The DL3015 smell is fixed by adding the --no-install-recommends option to the apt-get install command. + hadolint does not raise the smell when "apt" is used instead of "apt-get" + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + lines = dockerfile.lines + + # Find apt-get position and adding --no-install-recommends option + position = smell_pos - 1 + for line in lines[position:]: + if "apt-get" in line and " install " in line and "--no-install-recommends" not in line and not line.strip().startswith("#"): + lines[position] = lines[position].replace(" install ", " install --no-install-recommends ") + if "\\" in line: + position += 1 + diff --git a/smell_solvers/rules/DL3020.py b/smell_solvers/rules/DL3020.py new file mode 100644 index 0000000..a1d973f --- /dev/null +++ b/smell_solvers/rules/DL3020.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import re + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy + + +class DL3020(Strategy): + """ + Rule DL3020 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3020 + + Rationale: + "For other items (files, directories) that do not require ADD’s tar auto-extraction capability, + you should always use COPY." + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + lines = dockerfile.lines + + position = smell_pos - 1 + line = lines[position] + + if line.strip().upper().startswith('ADD'): + # Handle sensitive case replace + lines[position] = re.sub(re.escape('ADD'), 'COPY', line, count=1, flags=re.IGNORECASE) diff --git a/smell_solvers/rules/DL3025.py b/smell_solvers/rules/DL3025.py new file mode 100644 index 0000000..4fd12d4 --- /dev/null +++ b/smell_solvers/rules/DL3025.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +import shlex +import re + + +class DL3025(Strategy): + """ + Rule DL3025 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3025 + + Rationale: + Use arguments JSON notation for CMD and ENTRYPOINT arguments + + Note: + The CMD or ENTRYPOINT instruction is formatted when fixing the smell by adding \n where && or ; are present + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + smelly_cmd = dockerfile.parsed_lines[smell_pos] + + new_cmd = list() + for stmt in smelly_cmd.value: + if stmt.startswith("[") and stmt.endswith("]"): + stmt = " ".join(stmt[1:-1].split(",")) + lexer = shlex.shlex(stmt, posix=True) + lexer.whitespace_split = True + for t in lexer: + t = re.sub(r'\s+', ' ', t.strip()) + t = t.replace('"', '\\"') + new_cmd.append('"' + t + '"') + + line_indent = ' ' + new_cmd_str = f"{smelly_cmd.cmd} [{', '.join(new_cmd)}]" + args = list() + for c in new_cmd_str.split(): + args.append(c) + if c == ';' or c == '&&' or c.endswith(';",') or c.endswith('&&",'): + args.append('\\\n' + line_indent) + + new_cmd = ' '.join(args) + + # replace old command + for i in range(smelly_cmd.end_line - smelly_cmd.start_line + 1): + del dockerfile.lines[smelly_cmd.start_line - 1] + + dockerfile.lines.insert(smelly_cmd.start_line - 1, new_cmd + "\n") + + dockerfile.update_smells_dict() diff --git a/smell_solvers/rules/DL3042.py b/smell_solvers/rules/DL3042.py new file mode 100644 index 0000000..d11ae4a --- /dev/null +++ b/smell_solvers/rules/DL3042.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +from utils.docker_utils import is_valid_dockerfile_command + + +class DL3042(Strategy): + """ + Rule DL3042 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3042 + + Rationale: + "Once a package is installed, it does not be installed and the Docker cache can be leveraged instead. + Since the pip cache makes the images larger and is not needed, it's better to disable it." + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + lines = dockerfile.lines + + # Find pip install position + position = smell_pos - 1 + for line in lines[position:]: + if "pip" in line and " install " in line and not line.strip().startswith("#"): + # Adding --no-cache-dir option + if not '--no-cache-dir' in line: + option_pos = lines[position].find(' install ') + 8 # install chars number + space + lines[position] = lines[position][:option_pos] + " --no-cache-dir " + lines[position][option_pos:] + elif position != (smell_pos - 1): + term = line.strip().replace('\t', ' ', 1).split(' ')[0] + if not line.strip() or is_valid_dockerfile_command(term): + break + + position += 1 diff --git a/smell_solvers/rules/DL3047.py b/smell_solvers/rules/DL3047.py new file mode 100644 index 0000000..4f74b92 --- /dev/null +++ b/smell_solvers/rules/DL3047.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy + + +class DL3047(Strategy): + """ + Rule DL3047 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3047 + + Rationale: + "wget without flag --progress will result in excessively bloated build logs when downloading larger files. + That's because it outputs a line for each fraction of a percentage point while downloading a big file. + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + lines = dockerfile.lines + + # Find wget line position + position = smell_pos - 1 + for line in lines[position:]: + if 'wget' in line and not line.strip().startswith('#'): + break + position += 1 + + # Get wget position in the found line + wget_position = lines[position].find('wget') + + # Adding --progress=dot:giga option + if not '--progress' in lines[position]: + option_pos = wget_position + 4 # wget chars number + lines[position] = lines[position][:option_pos] + ' --progress=dot:giga ' + lines[position][option_pos:] diff --git a/smell_solvers/rules/DL3048.py b/smell_solvers/rules/DL3048.py new file mode 100644 index 0000000..1e97889 --- /dev/null +++ b/smell_solvers/rules/DL3048.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +import re + + +class DL3048(Strategy): + """ + TODO: + Rule DL3048 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3048 + + Rationale: + Invalid Label Key + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + smelly_cmd = dockerfile.parsed_lines[smell_pos] + + new_labels = list() + labels = smelly_cmd.value + for i in range(0, len(labels), 2): + key = labels[i].strip() + + # Replace all non-`.` and non-`-` characters with `.` + new_label = list() + for c in key: + if len(new_label) > 0: + if c.isalnum(): + if new_label[-1].islower() and c.isupper(): + new_label.append('-') + new_label.append(c) + else: + new_label.append(c) + elif new_label[-1] != c: + if c in ['-', '.']: + new_label.append(c) + else: + new_label.append('-') + else: + if c.isalnum(): + new_label.append(c) + + new_labels.append(f"{''.join(new_label).lower()}={labels[i + 1]}") + + new_labels = "LABEL " + " \\\n ".join(new_labels) + + for i in range(smelly_cmd.start_line - 1, smelly_cmd.end_line): + dockerfile.lines[i] = "" + + dockerfile.lines[smelly_cmd.start_line - 1] = new_labels + "\n" + + dockerfile.update_smells_dict() diff --git a/smell_solvers/rules/DL3059.py b/smell_solvers/rules/DL3059.py new file mode 100644 index 0000000..34c6252 --- /dev/null +++ b/smell_solvers/rules/DL3059.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +from utils.docker_utils import COMMANDS, parse_dockerfile_str + + +class DL3059(Strategy): + """ + Rule DL3059 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL3059 + + Rationale: + "Multiple consecutive RUN instructions. Consider consolidation." + + Note: + The RUN instructions are not consolidated if they are separated by comments or other instructions. + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + command_pos = smell_pos - 1 + + # find the previous RUN instruction + prev_run_pos = -1 + for i in reversed(range(command_pos)): + prev_line = dockerfile.lines[i].strip() + if prev_line: + if prev_line.startswith("#") or prev_line.startswith("RUN --"): + break + elif prev_line.split(" ")[0] in COMMANDS: + if prev_line.startswith("RUN"): + prev_run_pos = i + else: + break + + # if there is a previous RUN instruction, consolidate the successive RUN instructions until a comment or other instruction is found + if prev_run_pos > 0: + i = prev_run_pos + 1 + while i < len(dockerfile.lines): + line = dockerfile.lines[i].strip() + # delete intermediate blank lines + if not line and i+1 < len(dockerfile.lines) \ + and dockerfile.lines[i + 1].startswith("RUN") and not dockerfile.lines[i + 1].startswith("RUN --"): + del dockerfile.lines[i] + elif not line.startswith("#") and \ + line.startswith("RUN") and not line.startswith("RUN --"): + if not dockerfile.lines[i - 1].endswith(" \\\n"): + dockerfile.lines[i - 1] = dockerfile.lines[i - 1].rstrip() + " \\\n" + dockerfile.lines[i] = dockerfile.lines[i].lstrip().replace("RUN", " &&", 1) + i += 1 + else: + break + + dockerfile.update_smells_dict() diff --git a/smell_solvers/rules/DL4000.py b/smell_solvers/rules/DL4000.py new file mode 100644 index 0000000..727cb0a --- /dev/null +++ b/smell_solvers/rules/DL4000.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +from utils.common import dequote +import re + +class DL4000(Strategy): + """ + Rule DL4000 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL4000 + + Rationale: + "MAINTAINER is deprecated since Docker 1.13.0" + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + command_pos = -1 + maintainers = list() + for line in dockerfile.parsed_lines.values(): + if line.cmd.lower() == 'maintainer': + if command_pos == -1 or line.start_line - 1 < command_pos: + command_pos = line.start_line - 1 + for m in re.split(r'\s{2,}', line.value[0]): + maintainers.append(dequote(m.strip())) + for i in range(line.start_line - 1, line.end_line): + dockerfile.lines[i] = "" + dockerfile.lines[i] = "\n" + + if maintainers: + dockerfile.lines[command_pos] = f'LABEL maintainer="{", ".join(maintainers)}"\n' + dockerfile.update_smells_dict() + diff --git a/smell_solvers/rules/DL4006.py b/smell_solvers/rules/DL4006.py new file mode 100644 index 0000000..67e8ec9 --- /dev/null +++ b/smell_solvers/rules/DL4006.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import subprocess + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy +from utils.docker_utils import validate_shell +import logging as logger + + +class DL4006(Strategy): + """ + Rule DL4006 solver + + Reference: + https://github.com/hadolint/hadolint/wiki/DL4006 + + Rationale: + "If you want the command to fail due to an error at any stage in the pipe, prepend set -o pipefail && + to ensure that an unexpected error prevents the build from inadvertently succeeding." + """ + + def fix(self, dockerfile: Dockerfile, smell_pos: int) -> None: + command_pos = smell_pos - 1 + lines = dockerfile.lines + + # Get nearest FROM + for line in reversed(lines[:smell_pos]): + if line.strip().upper().startswith('FROM'): + near_from = line.strip().split(" ")[1] + + # set default shell and option + path = '/bin/bash' + option = '-o' + + # set ash if alpine or busybox + if 'alpine' in near_from or 'busybox' in near_from: + path = '/bin/ash' + option = '-eo' + + logger.info("Validate shell {} for {}".format(path, near_from)) + if not validate_shell(near_from, path): + # shell not found, fallback to default sh + path = '/bin/sh' + option = '-o' + logger.warning("Validate fallback shell {}".format(path)) + has_basic_shell = validate_shell(near_from, path) + if not has_basic_shell: + logger.error("!!! Cannot fix DL4006. No valid shell found.") + return + + new_line = f'SHELL ["{path}", "{option}", "pipefail", "-c"]' + '\n' + lines.insert(command_pos, new_line) + + dockerfile.update_smells_dict(ignore=['DL4006']) + + # TODO: handle multi-stage Dockerfiles diff --git a/smell_solvers/smell_solver.py b/smell_solvers/smell_solver.py new file mode 100644 index 0000000..d98367b --- /dev/null +++ b/smell_solvers/smell_solver.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import sys +from os import listdir +from os.path import isfile, join +from typing import List + +from logic.dockerfile_obj import Dockerfile +from smell_solvers.strategy import Strategy + +RULES_PATH = sys.path[0] + '/smell_solvers/rules' + + +class SmellSolver: + """ + SmellSolver class offers the function to solve smells with Strategy design pattern + """ + + def __init__(self) -> None: + self._strategy = None + self._available_strategies = self.__load_strategies() + + @property + def strategy(self) -> Strategy: + return self._strategy + + @strategy.setter + def strategy(self, strategy: Strategy) -> None: + """ + Define the strategy of the smell to solve + """ + self._strategy = strategy + + def fix_smell(self, dockerfile: Dockerfile, smell_pos: int) -> None: + """ + Fix the smell (previously set) in a specific line position of the given Dockerfile + :param dockerfile: Dockerfile to fix + :param smell_pos: position of the line containing the smell + :return: None + """ + if self._strategy: + self._strategy.fix(dockerfile, smell_pos) + + def strategy_exists(self, name: str) -> bool: + """ + Check if the given strategy name is available + :param name: name of the strategy to check + :return: True/False + """ + return name in self._available_strategies + + def __load_strategies(self) -> List[str]: + """ + Search all the available strategies in the 'rules' folder + :return: list of strings representing the existing strategies + """ + return [f.split('.')[0] for f in listdir(RULES_PATH) if isfile(join(RULES_PATH, f))] diff --git a/smell_solvers/strategy.py b/smell_solvers/strategy.py new file mode 100644 index 0000000..8358cf0 --- /dev/null +++ b/smell_solvers/strategy.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod + +from logic.dockerfile_obj import Dockerfile + + +class Strategy(ABC): + """ + Abstract Class of the strategy adopted to solve a smell + """ + + @abstractmethod + def fix(self, dockerfile: Dockerfile, smell_pos: int): + pass diff --git a/temp/README.md b/temp/README.md new file mode 100644 index 0000000..e69de29 diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..1c6970e --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,2 @@ +*-fixed +*-log.html diff --git a/test/DL3003/Example1 b/test/DL3003/Example1 new file mode 100644 index 0000000..568b6bd --- /dev/null +++ b/test/DL3003/Example1 @@ -0,0 +1,2 @@ +FROM busybox +RUN cd /usr/src/app diff --git a/test/DL3003/Example2 b/test/DL3003/Example2 new file mode 100644 index 0000000..7092446 --- /dev/null +++ b/test/DL3003/Example2 @@ -0,0 +1,2 @@ +FROM busybox +RUN cd /usr/src/app && git clone git@github.com:lukasmartinelli/hadolint.git \ No newline at end of file diff --git a/test/DL3003/Example3 b/test/DL3003/Example3 new file mode 100644 index 0000000..0839fc4 --- /dev/null +++ b/test/DL3003/Example3 @@ -0,0 +1,3 @@ +FROM busybox +RUN cd /usr/src/app && git clone git@github.com:lukasmartinelli/hadolint.git && git pull +RUN test diff --git a/test/DL3003/Example4 b/test/DL3003/Example4 new file mode 100644 index 0000000..c053ac3 --- /dev/null +++ b/test/DL3003/Example4 @@ -0,0 +1,214 @@ +# +# EXAMPLE OF NOT SUPPORTED DL3003 +# + +# Image +FROM alpine:3.12 as builder + +# Environment variables +ENV WKHTMLTOX_VERSION=0.12.6 + +# Copy patches +RUN mkdir -p /tmp/patches +COPY conf/* /tmp/patches/ + +# Alpine 3.11 and higher versions have libstdc++ and g++ v9+ in their repositories which breaks the build +RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.10/main' >> /etc/apk/repositories + +# Install needed packages +RUN apk add --no-cache \ + libstdc++=8.3.0-r0 \ + libx11 \ + libxrender \ + libxext \ + libssl1.1 \ + ca-certificates \ + fontconfig \ + freetype \ + ttf-dejavu \ + ttf-droid \ + ttf-freefont \ + ttf-liberation \ + ttf-ubuntu-font-family \ +&& apk add --no-cache --virtual .build-deps \ + g++=8.3.0-r0 \ + git \ + gtk+ \ + gtk+-dev \ + make \ + mesa-dev \ + msttcorefonts-installer \ + openssl-dev \ + patch \ + fontconfig-dev \ + freetype-dev \ +\ +# Install microsoft fonts +&& update-ms-fonts \ +&& fc-cache -f \ +\ +# Download source files +&& git clone --recursive https://github.com/wkhtmltopdf/wkhtmltopdf.git /tmp/wkhtmltopdf \ +&& cd /tmp/wkhtmltopdf \ +&& git checkout tags/$WKHTMLTOX_VERSION \ +\ +# Apply patches +&& cd /tmp/wkhtmltopdf/qt \ +&& patch -p1 -i /tmp/patches/qt-musl.patch \ +&& patch -p1 -i /tmp/patches/qt-musl-iconv-no-bom.patch \ +&& patch -p1 -i /tmp/patches/qt-recursive-global-mutex.patch \ +&& patch -p1 -i /tmp/patches/qt-gcc6.patch \ +\ +# Modify qmake config +&& sed -i "s|-O2|$CXXFLAGS|" mkspecs/common/g++.conf \ +&& sed -i "/^QMAKE_RPATH/s| -Wl,-rpath,||g" mkspecs/common/g++.conf \ +&& sed -i "/^QMAKE_LFLAGS\s/s|+=|+= $LDFLAGS|g" mkspecs/common/g++.conf \ +\ +# Prepare optimal build settings +&& NB_CORES=$(grep -c '^processor' /proc/cpuinfo) \ +\ +# Install qt +&& ./configure -confirm-license -opensource \ + -prefix /usr \ + -datadir /usr/share/qt \ + -sysconfdir /etc \ + -plugindir /usr/lib/qt/plugins \ + -importdir /usr/lib/qt/imports \ + -silent \ + -release \ + -static \ + -webkit \ + -script \ + -svg \ + -exceptions \ + -xmlpatterns \ + -openssl-linked \ + -no-fast \ + -no-largefile \ + -no-accessibility \ + -no-stl \ + -no-sql-ibase \ + -no-sql-mysql \ + -no-sql-odbc \ + -no-sql-psql \ + -no-sql-sqlite \ + -no-sql-sqlite2 \ + -no-qt3support \ + -no-opengl \ + -no-openvg \ + -no-system-proxies \ + -no-multimedia \ + -no-audio-backend \ + -no-phonon \ + -no-phonon-backend \ + -no-javascript-jit \ + -no-scripttools \ + -no-declarative \ + -no-declarative-debug \ + -no-mmx \ + -no-3dnow \ + -no-sse \ + -no-sse2 \ + -no-sse3 \ + -no-ssse3 \ + -no-sse4.1 \ + -no-sse4.2 \ + -no-avx \ + -no-neon \ + -no-rpath \ + -no-nis \ + -no-cups \ + -no-pch \ + -no-dbus \ + -no-separate-debug-info \ + -no-gtkstyle \ + -no-nas-sound \ + -no-opengl \ + -no-openvg \ + -no-sm \ + -no-xshape \ + -no-xvideo \ + -no-xsync \ + -no-xinerama \ + -no-xcursor \ + -no-xfixes \ + -no-xrandr \ + -no-mitshm \ + -no-xinput \ + -no-xkb \ + -no-glib \ + -no-icu \ + -nomake demos \ + -nomake docs \ + -nomake examples \ + -nomake tools \ + -nomake tests \ + -nomake translations \ + -graphicssystem raster \ + -qt-zlib \ + -qt-libpng \ + -qt-libmng \ + -qt-libtiff \ + -qt-libjpeg \ + -optimized-qmake \ + -iconv \ + -xrender \ + -fontconfig \ + -D ENABLE_VIDEO=0 \ +&& make --jobs $(($NB_CORES*2)) --silent \ +&& make install \ +\ +# Install wkhtmltopdf +&& cd /tmp/wkhtmltopdf \ +&& qmake \ +&& make --jobs $(($NB_CORES*2)) --silent \ +&& make install \ +&& make clean \ +&& make distclean \ +\ +# Uninstall qt +&& cd /tmp/wkhtmltopdf/qt \ +&& make uninstall \ +&& make clean \ +&& make distclean \ +\ +# Clean up when done +&& rm -rf /tmp/* \ +&& apk del .build-deps + +# Image +FROM alpine:3.12 + +# Alpine 3.11 and higher versions have libstdc++ v9+ in their repositories which breaks the build +RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.10/main' >> /etc/apk/repositories + +RUN apk add --no-cache \ + libstdc++=8.3.0-r0 \ + libx11 \ + libxrender \ + libxext \ + libssl1.1 \ + ca-certificates \ + fontconfig \ + freetype \ + ttf-dejavu \ + ttf-droid \ + ttf-freefont \ + ttf-liberation \ + ttf-ubuntu-font-family \ +&& apk add --no-cache --virtual .build-deps \ + msttcorefonts-installer \ +\ +# Install microsoft fonts +&& update-ms-fonts \ +&& fc-cache -f \ +\ +# Clean up when done +&& rm -rf /tmp/* \ +&& apk del .build-deps + +COPY --from=builder /bin/wkhtmltopdf /bin/wkhtmltopdf +COPY --from=builder /bin/wkhtmltoimage /bin/wkhtmltoimage +COPY --from=builder /lib/libwkhtmltox* /bin/ + +ENTRYPOINT ["wkhtmltopdf"] diff --git a/test/DL3003/Example5 b/test/DL3003/Example5 new file mode 100644 index 0000000..cc2cb6c --- /dev/null +++ b/test/DL3003/Example5 @@ -0,0 +1,8 @@ +FROM ubuntu:22.04 + +ADD . /byro +ADD byro.docker.cfg /byro/byro.cfg +RUN cd /byro && pip3 install -e . && pip3 install gunicorn +RUN cd /byro && /bin/zsh install_local_plugins.sh + +CMD ["runserver", "0.0.0.0:8020"] \ No newline at end of file diff --git a/test/DL3003/Example6 b/test/DL3003/Example6 new file mode 100644 index 0000000..de9a83f --- /dev/null +++ b/test/DL3003/Example6 @@ -0,0 +1,3 @@ +FROM busybox +RUN cd /usr/src/app && \ + git clone git@github.com:lukasmartinelli/hadolint.git \ No newline at end of file diff --git a/test/DL3006/Example1 b/test/DL3006/Example1 new file mode 100644 index 0000000..61bce34 --- /dev/null +++ b/test/DL3006/Example1 @@ -0,0 +1 @@ +FROM debian diff --git a/test/DL3006/Example2 b/test/DL3006/Example2 new file mode 100644 index 0000000..ceaf18a --- /dev/null +++ b/test/DL3006/Example2 @@ -0,0 +1 @@ +FROM ubuntu diff --git a/test/DL3006/Example3 b/test/DL3006/Example3 new file mode 100644 index 0000000..e023e05 --- /dev/null +++ b/test/DL3006/Example3 @@ -0,0 +1 @@ +FROM ruby diff --git a/test/DL3006/Example4 b/test/DL3006/Example4 new file mode 100644 index 0000000..68e9964 --- /dev/null +++ b/test/DL3006/Example4 @@ -0,0 +1,3 @@ +FROM maven + +FROM openjdk diff --git a/test/DL3006/Example5 b/test/DL3006/Example5 new file mode 100644 index 0000000..2523c08 --- /dev/null +++ b/test/DL3006/Example5 @@ -0,0 +1,3 @@ +FROM maven AS build + +FROM openjdk diff --git a/test/DL3008/Example1 b/test/DL3008/Example1 new file mode 100644 index 0000000..b321b20 --- /dev/null +++ b/test/DL3008/Example1 @@ -0,0 +1,3 @@ +FROM ubuntu:20.04 + +RUN apt-get update && apt-get install -y wget diff --git a/test/DL3008/Example10 b/test/DL3008/Example10 new file mode 100644 index 0000000..a48d579 --- /dev/null +++ b/test/DL3008/Example10 @@ -0,0 +1,124 @@ +FROM ubuntu:20.04 + +LABEL name="deis-go-dev" \ + maintainer="Matt Boersma " + +ENV ANSIBLE_VERSION=5.9.0 \ + AZCLI_VERSION=2.46.0 \ + DOCKER_VERSION=20.10.2 \ + ETCDCTL_VERSION=v3.1.8 \ + GO_VERSION=1.19.7 \ + GOLANGCI_LINT_VERSION=v1.52.2 \ + GOSS_VERSION=v0.3.21 \ + HELM_VERSION=3.11.2 \ + KUBECTL_VERSION=v1.26.3 \ + PACKER_VERSION=1.8.6 \ + PROTOBUF_VERSION=3.7.0 \ + PYJWT_VERSION=2.1.0 \ + PYWINRM_VERSION=0.4.3 \ + SHELLCHECK_VERSION=v0.8.0 \ + SHFMT_VERSION=3.4.2 \ + UPX_VERSION=3.96 \ + PATH=$PATH:/usr/local/go/bin:/go/bin:/usr/local/bin/docker \ + GOPATH=/go + +# This is a huge one-liner to optimize the Docker image layer. +# We disable source repos to speed up apt-get update. +RUN \ + sed -i -e 's/^deb-src/#deb-src/' /etc/apt/sources.list && \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository ppa:rmescandon/yq && \ + apt-get update && \ + apt-get upgrade -y --no-install-recommends && \ + apt-get install -y --no-install-recommends \ + bash \ + build-essential \ + ca-certificates \ + curl \ + git-core \ + jq \ + libffi-dev \ + libc6 \ + libssl-dev \ + libunwind8 \ + man \ + mercurial \ + net-tools \ + netcat \ + openssh-client \ + procps \ + python3 \ + python3-dev \ + python3-pip \ + python3-setuptools \ + rsync \ + ruby \ + unzip \ + util-linux \ + vim \ + wamerican \ + wget \ + yq \ + zip \ + && curl -L https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xz \ + && curl -sSL -o /tmp/protoc.zip https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip \ + && unzip /tmp/protoc.zip 'bin/protoc' -d /usr/local \ + && rm /tmp/protoc.zip \ + && curl -sSL https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl \ + && chmod +x /usr/local/bin/kubectl \ + && mkdir -p ${GOPATH}/src/k8s.io/helm \ + && curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 \ + && chmod 700 get_helm.sh && ./get_helm.sh --version v${HELM_VERSION} \ + && rm ./get_helm.sh \ + && mkdir -p /go/bin \ + && curl -sSL https://aka.ms/downloadazcopy-v10-linux | tar -vxz -C /usr/local/bin --strip=1 \ + && mv /usr/local/bin/azcopy /usr/local/bin/azcopy-preview \ + && curl -sSL https://aka.ms/downloadazcopylinux64 | tar -vxz -C /tmp \ + && /tmp/install.sh \ + && apt-get update && apt-get -f -y install \ + && curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz \ + && tar xzvf docker-${DOCKER_VERSION}.tgz -C /usr/local/bin \ + && chmod +x -R /usr/local/bin/docker \ + && rm docker-${DOCKER_VERSION}.tgz \ + && curl -L https://github.com/coreos/etcd/releases/download/${ETCDCTL_VERSION}/etcd-${ETCDCTL_VERSION}-linux-amd64.tar.gz -o /tmp/etcd-${ETCDCTL_VERSION}.tar.gz \ + && tar -C /tmp -xvzf /tmp/etcd-${ETCDCTL_VERSION}.tar.gz --strip-components=1 etcd-${ETCDCTL_VERSION}-linux-amd64/etcdctl \ + && mv /tmp/etcdctl /usr/local/bin/etcdctl \ + && rm /tmp/etcd-${ETCDCTL_VERSION}.tar.gz \ + && go install -v github.com/AlekSi/gocov-xml@latest \ + && go install -v github.com/axw/gocov/gocov@latest \ + && go install -v github.com/dgrijalva/jwt-go/cmd/jwt@latest \ + && go install -v github.com/go-bindata/go-bindata/go-bindata@v3.1.2 \ + && go install -v github.com/go-delve/delve/cmd/dlv@v1.9.0 \ + && go install -v github.com/golang/protobuf/protoc-gen-go@latest \ + && go install -v github.com/haya14busa/goverage@latest \ + && go install -v github.com/mitchellh/gox@latest \ + && go install -v github.com/onsi/ginkgo/ginkgo@v1.16.5 \ + && curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b ${GOPATH}/bin ${GOLANGCI_LINT_VERSION} \ + && curl -sSL https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz \ + | tar -vxJ -C /usr/local/bin --strip=1 \ + && curl -sSL https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip -o /tmp/packer.zip \ + && unzip /tmp/packer.zip -d /usr/local/bin \ + && curl -o /usr/local/bin/shfmt -sSL https://github.com/mvdan/sh/releases/download/v{SHFMT_VERSION}/shfmt_v{SHFMT_VERSION}_linux_amd64 \ + && chmod +x /usr/local/bin/shfmt \ + && curl -L "https://github.com/aelsabbahy/goss/releases/download/${GOSS_VERSION}/goss-linux-amd64" -o /usr/local/bin/goss \ + && chmod +rx /usr/local/bin/goss \ + && curl -L "https://github.com/aelsabbahy/goss/releases/download/${GOSS_VERSION}/dgoss" -o /usr/local/bin/dgoss \ + && chmod +rx /usr/local/bin/dgoss \ + && curl -sSL -o /tmp/upx.tar.xz https://github.com/upx/upx/releases/download/v${UPX_VERSION}/upx-${UPX_VERSION}-amd64_linux.tar.xz \ + && tar -xvf /tmp/upx.tar.xz -C /tmp \ + && mv /tmp/upx-${UPX_VERSION}-amd64_linux/upx /usr/local/bin/upx \ + && pip3 install --disable-pip-version-check --no-cache-dir --upgrade pip \ + && pip3 install --disable-pip-version-check --no-cache-dir --upgrade azure-cli==${AZCLI_VERSION} PyJWT==${PYJWT_VERSION} shyaml pywinrm==${PYWINRM_VERSION} \ + && pip3 install --disable-pip-version-check --no-cache-dir --upgrade ansible==${ANSIBLE_VERSION} ansible-core==2.12.6 --force-reinstall \ + && ansible-galaxy collection install ansible.windows:==1.13.0 \ + && apt-get purge --auto-remove -y libffi-dev python3-dev \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc ${GOPATH}/pkg/* ${GOPATH}/src/* /root/cache /root/.cache \ + && go clean -cache -testcache -modcache + +WORKDIR /go + +COPY . / diff --git a/test/DL3008/Example11 b/test/DL3008/Example11 new file mode 100644 index 0000000..423ee17 --- /dev/null +++ b/test/DL3008/Example11 @@ -0,0 +1,13 @@ +FROM rabbitmq:3-management + +RUN apt-get update + +RUN apt-get install -y curl && apt-get install -y zip + +RUN curl -O https://dl.bintray.com/rabbitmq/community-plugins/3.8.x/rabbitmq_delayed_message_exchange/rabbitmq_delayed_message_exchange-20191008-3.8.x.zip\ +&& unzip rabbitmq_delayed_message_exchange-20191008-3.8.x.zip -d $RABBITMQ_HOME/plugins \ +&& rm rabbitmq_delayed_message_exchange-20191008-3.8.x.zip + +RUN rabbitmq-plugins enable --offline rabbitmq_delayed_message_exchange + +RUN rabbitmq-plugins enable --offline rabbitmq_consistent_hash_exchange \ No newline at end of file diff --git a/test/DL3008/Example2 b/test/DL3008/Example2 new file mode 100644 index 0000000..6a230ae --- /dev/null +++ b/test/DL3008/Example2 @@ -0,0 +1,7 @@ +FROM ubuntu:20.04 + +RUN apt-get update && apt-get install --no-install-recommends -y unzip + +COPY . . + +RUN apt-get install -y ruby diff --git a/test/DL3008/Example3 b/test/DL3008/Example3 new file mode 100644 index 0000000..8846609 --- /dev/null +++ b/test/DL3008/Example3 @@ -0,0 +1,18 @@ +FROM ubuntu:20.04 + +RUN apt-get update \ + && apt-get install --no-install-recommends -y wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV HUGO_VERSION 0.74.3 + +RUN wget -q https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_Linux-64bit.deb \ + && dpkg -i hugo_${HUGO_VERSION}_Linux-64bit.deb \ + && rm -rf hugo_${HUGO_VERSION}_Linux-64bit.deb + +WORKDIR /usr/src/app + +COPY . . + +CMD ["hugo", "server", "-D"] \ No newline at end of file diff --git a/test/DL3008/Example4 b/test/DL3008/Example4 new file mode 100644 index 0000000..8c3c9ce --- /dev/null +++ b/test/DL3008/Example4 @@ -0,0 +1,4 @@ +FROM ubuntu:precise + +RUN apt-get -qq update +RUN apt-get install -y -qq libgtk2.0-0 libasound2 libpulse0 libgtk-3-0 \ No newline at end of file diff --git a/test/DL3008/Example5 b/test/DL3008/Example5 new file mode 100644 index 0000000..69ca2f6 --- /dev/null +++ b/test/DL3008/Example5 @@ -0,0 +1,11 @@ +FROM debian:sid + +RUN echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup +RUN echo 'APT::Acquire::Retries "5";' > /etc/apt/apt.conf.d/80retry + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + i3-wm=4.22-2 xvfb strace && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/src diff --git a/test/DL3008/Example6 b/test/DL3008/Example6 new file mode 100644 index 0000000..250a87f --- /dev/null +++ b/test/DL3008/Example6 @@ -0,0 +1,34 @@ +ARG RUBY_VERSION=3.2.0 +FROM ruby:3.2.0 as base + +# Avoid warnings by switching to noninteractive +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get -o Acquire::Max-FutureTime=86400 update \ + && apt-get install --no-install-recommends -y \ + apt-utils \ + build-essential \ + clang \ + git \ + less \ + libssl-dev \ + linux-perf \ + lldb \ + lsb-release \ + netcat \ + procps \ + xz-utils \ + && gem install bundler rcodetools rubocop ruby-debug-ide fastri + +WORKDIR /opt +RUN git clone --depth 1 https://github.com/brendangregg/FlameGraph +ENV PATH /opt/FlameGraph:$PATH + +# Switch back to dialog for any ad-hoc use of apt-get +ENV DEBIAN_FRONTEND=dialog + +COPY Gemfile* semian.gemspec /workspace/ +COPY lib /workspace/lib + +WORKDIR /workspace +RUN bundle install diff --git a/test/DL3008/Example7 b/test/DL3008/Example7 new file mode 100644 index 0000000..15ebe5a --- /dev/null +++ b/test/DL3008/Example7 @@ -0,0 +1,5 @@ +FROM rabbitmq:3-management + +RUN apt-get update + +RUN apt-get install -y curl && apt-get install -y zip \ No newline at end of file diff --git a/test/DL3008/Example8 b/test/DL3008/Example8 new file mode 100644 index 0000000..83d0f47 --- /dev/null +++ b/test/DL3008/Example8 @@ -0,0 +1,32 @@ +FROM eclipse-temurin:17-jre-focal as flyway + +RUN apt-get update \ + && apt-get install -y python3-pip \ + && pip3 install sqlfluff==1.2.1 + +WORKDIR /flyway + +ARG FLYWAY_VERSION +ARG FLYWAY_ARTIFACT_URL=https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/ + +RUN curl -L ${FLYWAY_ARTIFACT_URL}${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}.tar.gz -o flyway-commandline-${FLYWAY_VERSION}.tar.gz \ + && gzip -d flyway-commandline-${FLYWAY_VERSION}.tar.gz \ + && tar -xf flyway-commandline-${FLYWAY_VERSION}.tar --strip-components=1 \ + && rm flyway-commandline-${FLYWAY_VERSION}.tar \ + && chmod -R a+r /flyway \ + && chmod a+x /flyway/flyway + +ENV PATH="/flyway:${PATH}" + +ENTRYPOINT ["flyway"] +CMD ["-?"] + +FROM flyway as redgate + +RUN curl -L https://packages.microsoft.com/config/ubuntu/21.04/packages-microsoft-prod.deb -o packages-microsoft-prod.deb \ + && dpkg -i packages-microsoft-prod.deb \ + && rm packages-microsoft-prod.deb +RUN apt-get update \ + && apt-get install -y apt-transport-https \ + && apt-get update \ + && apt-get install -y dotnet-runtime-6.0 diff --git a/test/DL3008/Example9 b/test/DL3008/Example9 new file mode 100644 index 0000000..7b1ba6b --- /dev/null +++ b/test/DL3008/Example9 @@ -0,0 +1,23 @@ +FROM ubuntu:22.04 + +RUN \ + apt-get update; \ + apt-get install -y \ + build-essential \ + curl; \ + rm -rf /var/lib/apt/lists/* + +# setup nodejs +RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - +RUN apt-get install -y nodejs + +RUN useradd -ms /bin/bash ubuntu + +# do first step from Contribution +RUN npm install --global gulp-cli + +# expose ports which are being used in this project +EXPOSE 3001 +EXPOSE 3000 + +CMD /bin/bash diff --git a/test/DL3009/Example1 b/test/DL3009/Example1 new file mode 100644 index 0000000..ab96a1f --- /dev/null +++ b/test/DL3009/Example1 @@ -0,0 +1,2 @@ +FROM ubuntu +RUN apt-get update && apt-get install --no-install-recommends -y python=2.7 diff --git a/test/DL3009/Example10 b/test/DL3009/Example10 new file mode 100644 index 0000000..05c43c4 --- /dev/null +++ b/test/DL3009/Example10 @@ -0,0 +1,59 @@ +# Ubuntu 20.04 focal-20210119 +FROM ubuntu:focal-20210119 + +# Update package list. +ENV DEBIAN_FRONTEND=noninteractive + +# Set up the locale. +RUN apt-get update && \ + apt-get install -y locales openssh-sftp-server openssh-server xvfb libfontconfig libmariadbclient-dev && \ + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ + dpkg-reconfigure locales && \ + update-locale LANG=en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV LANGUAGE=en_US:en + +RUN echo "root:root" | chpasswd && \ + mkdir /var/run/sshd + +# Set up the timezone. +RUN apt-get update && apt-get install -y --no-install-recommends tzdata && \ + ln -fs /usr/share/zoneinfo/UTC /etc/localtime && \ + dpkg-reconfigure tzdata + +# Install GovReady-Q prerequisites. +RUN apt-get update && apt-get -y install \ + unzip git curl jq \ + python3 python3-pip \ + python3-yaml \ + graphviz pandoc \ + gunicorn + +ENV CHROME_VERSION "google-chrome-stable" +RUN sed -i -- 's&deb http://deb.debian.org/debian jessie-updates main&#deb http://deb.debian.org/debian jessie-updates main&g' /etc/apt/sources.list \ + && apt-get update && apt-get install wget -y +ENV CHROME_VERSION "google-chrome-stable" +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list \ + && apt-get update && apt-get -qqy install ${CHROME_VERSION:-google-chrome-stable} + +# Chromium for Headless Selenium tests +RUN wget -O /tmp/chromedriver.zip http://chromedriver.storage.googleapis.com/`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE`/chromedriver_linux64.zip \ + && unzip /tmp/chromedriver.zip chromedriver -d /usr/local/bin/ +ENV DISPLAY=:99 + +# Put the Python source code here. +WORKDIR /usr/src/app + +RUN apt-get update && apt-get install -y wkhtmltopdf; + +# Upgrade pip to version 20.1+ - IMPORTANT +RUN python3 -m pip install --upgrade pip +RUN pip3 install ipdb + +# This directory must be present for the AppSource created by our +# first_run script. The directory only has something in it if +# the container is launched with --mount. +# --mount type=bind,source="$(pwd)",dst=/mnt/q-files-host +RUN mkdir -p /mnt/q-files-host diff --git a/test/DL3009/Example2 b/test/DL3009/Example2 new file mode 100644 index 0000000..14806a2 --- /dev/null +++ b/test/DL3009/Example2 @@ -0,0 +1,5 @@ +FROM ubuntu + +RUN apt-get update && apt-get install --no-install-recommends -y python=2.7 +COPY . . +RUN apt-get install -y ruby=1.9 diff --git a/test/DL3009/Example3 b/test/DL3009/Example3 new file mode 100644 index 0000000..0e1c228 --- /dev/null +++ b/test/DL3009/Example3 @@ -0,0 +1,13 @@ +FROM ubuntu:20.04 + +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + git + +COPY . . + +RUN apt-get update \ + && apt-get install -y \ + nodejs \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/test/DL3009/Example4 b/test/DL3009/Example4 new file mode 100644 index 0000000..681823f --- /dev/null +++ b/test/DL3009/Example4 @@ -0,0 +1,19 @@ +FROM ubuntu:20.04 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/src/app + +COPY . . + +RUN unzip test-repos.zip \ + && pip install --no-cache-dir -r test-requirements.txt \ + && rm test-repos.zip + +CMD ["pytest"] \ No newline at end of file diff --git a/test/DL3009/Example5 b/test/DL3009/Example5 new file mode 100644 index 0000000..e00b249 --- /dev/null +++ b/test/DL3009/Example5 @@ -0,0 +1,6 @@ +FROM php:7.2 + +ENV redis_version 6.0.8 + +RUN apt-get update && \ + apt-get install -y wget libssl-dev \ No newline at end of file diff --git a/test/DL3009/Example6 b/test/DL3009/Example6 new file mode 100644 index 0000000..8956447 --- /dev/null +++ b/test/DL3009/Example6 @@ -0,0 +1,6 @@ +FROM php:7.2 + +ENV redis_version 6.0.8 + +RUN apt-get update \ + && apt-get install -y wget libssl-dev \ No newline at end of file diff --git a/test/DL3009/Example7 b/test/DL3009/Example7 new file mode 100644 index 0000000..b36c90c --- /dev/null +++ b/test/DL3009/Example7 @@ -0,0 +1,18 @@ +FROM ruby:3.2 as base +# Avoid warnings by switching to noninteractive +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get -o Acquire::Max-FutureTime=86400 update \ + && apt-get install --no-install-recommends -y \ + apt-utils \ + build-essential \ + clang \ + git \ + less \ + libssl-dev \ + linux-perf \ + lldb \ + lsb-release \ + netcat \ + procps \ + xz-utils \ + && gem install bundler rcodetools rubocop ruby-debug-ide fastri \ No newline at end of file diff --git a/test/DL3009/Example8 b/test/DL3009/Example8 new file mode 100644 index 0000000..4076ddd --- /dev/null +++ b/test/DL3009/Example8 @@ -0,0 +1,10 @@ +FROM ruby:3.2 as base +# Avoid warnings by switching to noninteractive +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get -o Acquire::Max-FutureTime=86400 update \ + && apt-get install --no-install-recommends -y \ + apt-utils \ + build-essential \ + clang \ + git \ + && gem install bundler rcodetools rubocop ruby-debug-ide fastri \ No newline at end of file diff --git a/test/DL3009/Example9 b/test/DL3009/Example9 new file mode 100644 index 0000000..c042b5d --- /dev/null +++ b/test/DL3009/Example9 @@ -0,0 +1,10 @@ +FROM ruby:3.2 as base +# Avoid warnings by switching to noninteractive +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get -o Acquire::Max-FutureTime=86400 update \ +&& apt-get install --no-install-recommends -y \ +apt-utils \ +build-essential \ +clang \ +git \ +&& gem install bundler rcodetools rubocop ruby-debug-ide fastri diff --git a/test/DL3015/Example1 b/test/DL3015/Example1 new file mode 100644 index 0000000..ad6032f --- /dev/null +++ b/test/DL3015/Example1 @@ -0,0 +1,2 @@ +FROM busybox +RUN apt-get install -y python=2.7 \ No newline at end of file diff --git a/test/DL3015/Example2 b/test/DL3015/Example2 new file mode 100644 index 0000000..b574ef9 --- /dev/null +++ b/test/DL3015/Example2 @@ -0,0 +1,4 @@ +FROM ubuntu +RUN apt-get install -y python=2.7 + +RUN apt-get install -y ruby diff --git a/test/DL3015/Example3 b/test/DL3015/Example3 new file mode 100644 index 0000000..9bbab8d --- /dev/null +++ b/test/DL3015/Example3 @@ -0,0 +1,3 @@ +from ubuntu:22.04 + +RUN apt-get update -y && apt-get install curl tar build-essential libssl-dev git -y \ No newline at end of file diff --git a/test/DL3015/Example4 b/test/DL3015/Example4 new file mode 100644 index 0000000..dea47bb --- /dev/null +++ b/test/DL3015/Example4 @@ -0,0 +1,4 @@ +from ubuntu:22.04 + +RUN apt-get update -y && \ + apt-get install curl tar build-essential libssl-dev git -y \ No newline at end of file diff --git a/test/DL3015/Example5 b/test/DL3015/Example5 new file mode 100644 index 0000000..c53804f --- /dev/null +++ b/test/DL3015/Example5 @@ -0,0 +1,4 @@ +from ubuntu:22.04 + +RUN apt update -y && \ + apt install curl tar build-essential libssl-dev git -y \ No newline at end of file diff --git a/test/DL3015/Example6 b/test/DL3015/Example6 new file mode 100644 index 0000000..75713b1 --- /dev/null +++ b/test/DL3015/Example6 @@ -0,0 +1,4 @@ +from ubuntu:22.04 + +RUN apt-get update -y && \ + apt-get install --no-install-recommends -y curl tar build-essential libssl-dev git \ No newline at end of file diff --git a/test/DL3015/Example7 b/test/DL3015/Example7 new file mode 100644 index 0000000..f9bf36a --- /dev/null +++ b/test/DL3015/Example7 @@ -0,0 +1,9 @@ +from ubuntu:22.04 + +RUN apt-get update -y && \ + apt-get install -y \ + curl \ + tar \ + build-essential \ + libssl-dev \ + git \ No newline at end of file diff --git a/test/DL3015/Example8 b/test/DL3015/Example8 new file mode 100644 index 0000000..27f1b41 --- /dev/null +++ b/test/DL3015/Example8 @@ -0,0 +1,9 @@ +FROM mcr.microsoft.com/powershell:7.1.3-debian-10 AS talks-env + +# First, install curl to be able to install Node.js, and then install Node.js itself: +RUN apt-get update \ + && apt-get install -y curl \ + && curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ + && apt-get install -y \ + nodejs \ + && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/test/DL3020/Example1 b/test/DL3020/Example1 new file mode 100644 index 0000000..56d9abb --- /dev/null +++ b/test/DL3020/Example1 @@ -0,0 +1,3 @@ +FROM python + +ADD . . diff --git a/test/DL3020/Example2 b/test/DL3020/Example2 new file mode 100644 index 0000000..7f2062b --- /dev/null +++ b/test/DL3020/Example2 @@ -0,0 +1,9 @@ +FROM python + +ADD . . + +RUN python . + +ADD . . + +COPY / / diff --git a/test/DL3025/Example1 b/test/DL3025/Example1 new file mode 100644 index 0000000..40abbea --- /dev/null +++ b/test/DL3025/Example1 @@ -0,0 +1,3 @@ +FROM python:3.10-slim-buster + +CMD my-service server \ No newline at end of file diff --git a/test/DL3025/Example2 b/test/DL3025/Example2 new file mode 100644 index 0000000..e03b322 --- /dev/null +++ b/test/DL3025/Example2 @@ -0,0 +1,3 @@ +FROM python:3.10-slim-buster + +ENTRYPOINT s3cmd \ No newline at end of file diff --git a/test/DL3025/Example3 b/test/DL3025/Example3 new file mode 100644 index 0000000..e789efd --- /dev/null +++ b/test/DL3025/Example3 @@ -0,0 +1,4 @@ +FROM python:3.10-slim-buster + +CMD my-service server +ENTRYPOINT s3cmd \ No newline at end of file diff --git a/test/DL3025/Example4 b/test/DL3025/Example4 new file mode 100644 index 0000000..da0a6ee --- /dev/null +++ b/test/DL3025/Example4 @@ -0,0 +1,12 @@ +FROM python:3.10-slim-buster + +CMD bash -c "\ + echo 'Starting container...' ; \ + mkdir -p app/logs ; \ + touch app/logs/access.log ; \ + touch app/logs/error.log ; \ + echo 'Container ready.' ; \ + " \ + && tail -f /app/logs/access.log /app/logs/error.log + +ENTRYPOINT ["python", "app.py"] diff --git a/test/DL3025/Example5 b/test/DL3025/Example5 new file mode 100644 index 0000000..f0dc6c4 --- /dev/null +++ b/test/DL3025/Example5 @@ -0,0 +1,3 @@ +FROM python:3.10-slim-buster + +CMD ['my-service', 'server'] \ No newline at end of file diff --git a/test/DL3025/Example6 b/test/DL3025/Example6 new file mode 100644 index 0000000..fbf7bdd --- /dev/null +++ b/test/DL3025/Example6 @@ -0,0 +1,3 @@ +FROM python:3.10-slim-buster + +CMD ["my-service server"] \ No newline at end of file diff --git a/test/DL3025/Example7 b/test/DL3025/Example7 new file mode 100644 index 0000000..2c7a17f --- /dev/null +++ b/test/DL3025/Example7 @@ -0,0 +1,3 @@ +FROM python:3.10-slim-buster + +CMD ['bash', '-c', 'echo "Starting container..." && tail -f /app/logs/access.log'] \ No newline at end of file diff --git a/test/DL3025/Example8 b/test/DL3025/Example8 new file mode 100644 index 0000000..f748808 --- /dev/null +++ b/test/DL3025/Example8 @@ -0,0 +1,9 @@ +FROM ruby:2.4.4 +WORKDIR /usr/src/app +RUN apt update && apt install -y nodejs curl libcurl3 libcurl3-openssl-dev openjdk-8-jdk && apt-get clean +COPY Gemfile Gemfile.lock ./ +RUN bundle install + +EXPOSE 3000 + +CMD rake jetty:clean && rake jetty:config && rake jetty:start && bundle exec rake db:migrate RAILS_ENV=development && bundle exec rails s -b 0.0.0.0 \ No newline at end of file diff --git a/test/DL3048/Example1 b/test/DL3048/Example1 new file mode 100644 index 0000000..fd63571 --- /dev/null +++ b/test/DL3048/Example1 @@ -0,0 +1,5 @@ +FROM python + +LABEL Maintainer-Address="test@mail.com" \ + versionNumber="1.0" \ + image_description="This is a test image" \ No newline at end of file diff --git a/test/DL3048/Example2 b/test/DL3048/Example2 new file mode 100644 index 0000000..c839112 --- /dev/null +++ b/test/DL3048/Example2 @@ -0,0 +1,3 @@ +FROM python + +LABEL +?not..valid--key="foo" \ No newline at end of file diff --git a/test/DL3059/Example1 b/test/DL3059/Example1 new file mode 100644 index 0000000..78c8600 --- /dev/null +++ b/test/DL3059/Example1 @@ -0,0 +1,7 @@ +FROM ubuntu:20.04 + +RUN apt-get update +RUN apt-get install -y python +RUN apt-get install -y ruby +RUN apt-get clean +RUN rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/test/DL3059/Example2 b/test/DL3059/Example2 new file mode 100644 index 0000000..49affd9 --- /dev/null +++ b/test/DL3059/Example2 @@ -0,0 +1,11 @@ +FROM ubuntu:20.04 + +RUN apt-get update +RUN apt-get install -y ruby +# test +RUN apt-get clean +COPY . . + +RUN rm -rf /var/lib/apt/lists/* + +RUN apt-get install -y python diff --git a/test/DL3059/Example3 b/test/DL3059/Example3 new file mode 100644 index 0000000..280d8f4 --- /dev/null +++ b/test/DL3059/Example3 @@ -0,0 +1,8 @@ +FROM ubuntu:20.04 + +RUN apt-get update +RUN apt-get install -y ruby +RUN apt-get clean +RUN rm -rf /var/lib/apt/lists/* + +RUN apt-get install -y python diff --git a/test/DL3059/Example4 b/test/DL3059/Example4 new file mode 100644 index 0000000..f010b9e --- /dev/null +++ b/test/DL3059/Example4 @@ -0,0 +1,26 @@ +FROM ubuntu:bionic + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=US/Pacific + +RUN apt -y update +RUN apt -y upgrade + +RUN apt -y install build-essential +RUN apt -y install git +RUN apt -y install wget + +RUN apt -y install python +RUN apt -y install python-setuptools +RUN apt -y install python-pip +RUN apt -y install python-tk +RUN apt -y install tk +RUN apt -y install tcl +RUN apt -y install tclx8.4 +RUN apt -y install tcllib +RUN apt -y install tcl-tls + + +RUN apt -y install iputils-ping +RUN apt -y install snmp +RUN apt -y install snmptrapd \ No newline at end of file diff --git a/test/DL3059/Example5 b/test/DL3059/Example5 new file mode 100644 index 0000000..314d9a2 --- /dev/null +++ b/test/DL3059/Example5 @@ -0,0 +1,16 @@ +FROM ocaml/opam2:alpine as base + +RUN opam install -y dune +COPY --chown=opam . /home/opam/src/ +WORKDIR /home/opam/src +ENV DUNE_CACHE=enabled +RUN --mount=type=cache,target=/home/opam/.cache,uid=1000 opam exec -- dune build --profile=static @install +RUN apt -y update +RUN apt -y upgrade +RUN --mount=type=cache,target=/home/opam/.cache,uid=1000 opam exec -- dune install +RUN apt -y install build-essential +RUN apt -y install git +RUN apt -y install wget + +RUN --mount=type=cache,target=/home/opam/.cache,uid=1000 opam exec -- dune install +RUN --mount=type=cache,target=/home/opam/.cache,uid=1000 opam exec -- dune install \ No newline at end of file diff --git a/test/DL3059/Example6 b/test/DL3059/Example6 new file mode 100644 index 0000000..cd5d502 --- /dev/null +++ b/test/DL3059/Example6 @@ -0,0 +1,11 @@ +FROM node + +ARG node_env="production" +ENV NODE_ENV=$node_env + +RUN cp -n ./config/production-dist.js ./config/production.js + +RUN npm run build + +EXPOSE 3000 +CMD ["npm", "start"] \ No newline at end of file diff --git a/test/DL3059/Example7 b/test/DL3059/Example7 new file mode 100644 index 0000000..6d0ef44 --- /dev/null +++ b/test/DL3059/Example7 @@ -0,0 +1,26 @@ +FROM node:12 + +RUN mkdir /app +COPY ./package.json /app +COPY ./package-lock.json /app + +WORKDIR /app + +RUN npm install --production && \ + npm cache clean --force + +COPY ./src /app/src +COPY ./config /app/config +COPY ./config.json ./babel.config.js ./jest.config.js ./jsconfig.json ./typings.json /app/ +RUN mkdir /app/upload +WORKDIR /app + +ARG node_env="production" +ENV NODE_ENV=$node_env + +RUN cp -n ./config/production-dist.js ./config/production.js + +RUN npm run build + +EXPOSE 3000 +CMD ["npm", "start"] \ No newline at end of file diff --git a/test/DL3059/Example8 b/test/DL3059/Example8 new file mode 100644 index 0000000..6d416ef --- /dev/null +++ b/test/DL3059/Example8 @@ -0,0 +1,4 @@ +FROM php:7.3-fpm-alpine +RUN apk add --update alpine-sdk +RUN git clone --recursive https://github.com/adsr/phpspy.git +RUN cd phpspy && make diff --git a/test/DL4000/Example1 b/test/DL4000/Example1 new file mode 100644 index 0000000..ca83538 --- /dev/null +++ b/test/DL4000/Example1 @@ -0,0 +1,11 @@ +FROM python + +MAINTAINER test + +ADD . . + +RUN python . + +ADD . . + +COPY / / diff --git a/test/DL4000/Example2 b/test/DL4000/Example2 new file mode 100644 index 0000000..a06bbbf --- /dev/null +++ b/test/DL4000/Example2 @@ -0,0 +1,12 @@ +FROM python + +MAINTAINER ArchPacaur + +ADD . . + +RUN python . + +ADD . . + +COPY / / +MAINTAINER ArchAurman diff --git a/test/DL4000/Example3 b/test/DL4000/Example3 new file mode 100644 index 0000000..6551413 --- /dev/null +++ b/test/DL4000/Example3 @@ -0,0 +1,8 @@ +FROM node:12.11.1 + +MAINTAINER "Skater Team " \ + "Biker Team " \ + "Surfer Team " + +WORKDIR /usr/src/app +RUN npm install --global yarn diff --git a/test/DL4000/Example4 b/test/DL4000/Example4 new file mode 100644 index 0000000..13463f4 --- /dev/null +++ b/test/DL4000/Example4 @@ -0,0 +1,7 @@ +FROM node:12.11.1 as build-deps +WORKDIR /usr/src/app +COPY ./frontend ./ +RUN yarn install && yarn build + +FROM golang:1.13.1-alpine +MAINTAINER "Skater Team" \ No newline at end of file diff --git a/test/DL4000/Example5 b/test/DL4000/Example5 new file mode 100644 index 0000000..6551413 --- /dev/null +++ b/test/DL4000/Example5 @@ -0,0 +1,8 @@ +FROM node:12.11.1 + +MAINTAINER "Skater Team " \ + "Biker Team " \ + "Surfer Team " + +WORKDIR /usr/src/app +RUN npm install --global yarn diff --git a/test/DL4000/Example6 b/test/DL4000/Example6 new file mode 100644 index 0000000..45f5fae --- /dev/null +++ b/test/DL4000/Example6 @@ -0,0 +1,12 @@ +FROM node:12.11.1 + +MAINTAINER "Skater Team " \ + "Biker Team " \ + "Surfer Team " + +WORKDIR /usr/src/app +RUN npm install --global yarn + +MAINTAINER 'Skater Team ' \ + 'Biker Team ' \ + 'Surfer Team ' diff --git a/test/DL4006/.directory b/test/DL4006/.directory new file mode 100644 index 0000000..300d688 --- /dev/null +++ b/test/DL4006/.directory @@ -0,0 +1,5 @@ +[Dolphin] +SortOrder=1 +Timestamp=2023,3,3,18,23,35.369 +Version=4 +ViewMode=1 diff --git a/test/DL4006/Example1 b/test/DL4006/Example1 new file mode 100644 index 0000000..6113386 --- /dev/null +++ b/test/DL4006/Example1 @@ -0,0 +1,3 @@ +FROM ubuntu + +RUN ping https://asdasdasd | wc -l > /number diff --git a/test/DL4006/Example2 b/test/DL4006/Example2 new file mode 100644 index 0000000..89499d7 --- /dev/null +++ b/test/DL4006/Example2 @@ -0,0 +1,4 @@ +FROM busybox + +RUN ping https://asdasdasd | wc -l > /number + diff --git a/test/DL4006/Example3 b/test/DL4006/Example3 new file mode 100644 index 0000000..02db72c --- /dev/null +++ b/test/DL4006/Example3 @@ -0,0 +1,3 @@ +FROM alpine + +RUN ping https://asdasdasd | wc -l > /number diff --git a/test/DL4006/Example4 b/test/DL4006/Example4 new file mode 100644 index 0000000..bde4e45 --- /dev/null +++ b/test/DL4006/Example4 @@ -0,0 +1,3 @@ +FROM debian:buster-slim + +RUN ping https://asdasdasd | wc -l > /number diff --git a/test/DL4006/Example5 b/test/DL4006/Example5 new file mode 100644 index 0000000..0960622 --- /dev/null +++ b/test/DL4006/Example5 @@ -0,0 +1,3 @@ +FROM python:3.10-slim + +RUN ping https://asdasdasd | wc -l > /number diff --git a/test/test.rb b/test/test.rb new file mode 100644 index 0000000..fe44f80 --- /dev/null +++ b/test/test.rb @@ -0,0 +1,52 @@ +THIS_DIR = File.dirname(File.absolute_path(__FILE__)) +PATH_TO_DOCKLEANER = THIS_DIR + "/../dockleaner.py" + +def dockleaner(path, smell) + `python #{PATH_TO_DOCKLEANER} -p "#{path}" -d "2023-01-01" --rule "#{smell}" >/dev/null 2>&1` +end + +def hadolint(path) + hadolint_result = `hadolint --no-color "#{path}"`.split("\n") + smells = hadolint_result.map { |l| l.sub(path, "").split(" ")[1] }.uniq + return smells +end + +Dir.chdir(THIS_DIR) do + Dir.glob("**/*-fixed").each do |fixed| + `rm "#{fixed}"` + end + + Dir.glob("**/*-log.html").each do |log| + `rm "#{log}"` + end + + to_test = ARGV[0] + if !to_test.nil? + puts "Testing smell: #{to_test}" + else + puts "Testing smell: ALL" + end + + Dir.glob("*").select { |f| FileTest.directory?(f) && !%w(. ..).include?(f) }.each do |folder| + smell = File.basename(folder) + if !to_test.nil? && !smell.include?(to_test) + next + end + + Dir.glob(folder + "/*").select { |f| !f.include?("-fixed") && !f.include?("-log.html") }.each do |dockerfile| + unless hadolint(dockerfile).include?(smell) + puts "#{dockerfile} is a bad test case: it does not contain #{smell}" + next + end + + print "Running dockleaner on #{dockerfile}... " + dockleaner(dockerfile, smell) + fixed_path = dockerfile + "-fixed" + if hadolint(fixed_path).include?(smell) + puts "NOT FIXED!" + else + puts "OK" + end + end + end +end diff --git a/utils/cache_handler.py b/utils/cache_handler.py new file mode 100644 index 0000000..ba8a901 --- /dev/null +++ b/utils/cache_handler.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import os +import pickle +import sys + +IMAGES_CACHE_PATH = sys.path[0] + '/.cache/images-version.cache' + +def retrieve_distro(image: str) -> str: + """ + Retrieve, if exists, the distribution info from the given image from the cache + + :param image_info: name and tag of the image. Expected format : + :return: the distribution info in the format : + i.e. retrieve_distro(myimage:mytag) return 'ubuntu:14.04' + """ + + if not os.path.exists(IMAGES_CACHE_PATH): + with open(IMAGES_CACHE_PATH, 'wb') as init_file: + pickle.dump({}, init_file) # init an empty cache + pass + return None + + with open(IMAGES_CACHE_PATH, 'rb') as file: + data = file.read() + + cache = pickle.loads(data) + + if image in cache: + return cache[image] + + +def update_images_cache(image: str, distro_info: str) -> None: + """ + Update the cache of the pulled images with the given one + + :param image: name and tag of the image. Expected format : + :param distro_info: name and tag of the related distribution. Expected format : + """ + + with open(IMAGES_CACHE_PATH, 'rb') as f: + data = f.read() + + cache = pickle.loads(data) + cache[image] = distro_info + + with open(IMAGES_CACHE_PATH, 'wb') as f: + pickle.dump(cache, f) + + +def clear_cache() -> None: + """ + Clear the cache of the pulled images + """ + with open(IMAGES_CACHE_PATH, 'wb') as init_file: + pickle.dump({}, init_file) # init an empty cache diff --git a/utils/common.py b/utils/common.py new file mode 100644 index 0000000..2b1a4f3 --- /dev/null +++ b/utils/common.py @@ -0,0 +1,36 @@ +import backoff +import requests +import logging as logger +logger.getLogger('backoff').addHandler(logger.StreamHandler()) + + +def dequote(s): + """ + If a string has single or double quotes around it, remove them. + Make sure the pair of quotes match. + If a matching pair of quotes is not found, + or there are less than 2 characters, return the string unchanged. + """ + if (len(s) >= 2 and s[0] == s[-1]) and s.startswith(("'", '"')): + return s[1:-1] + return s + + +@backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_tries=5, + giveup=lambda e: e.response is not None and e.response.status_code < 500 +) +def request_data(url: str): + return requests.get(url) + + +def parse_line_indent(line: str) -> str: + n_spaces = 0 + for c in line.replace('\t', ' '): + if c != " " and c != '\t': + break + n_spaces += 1 + + return " " * n_spaces if n_spaces > 0 else " " * 4 diff --git a/utils/docker_utils.py b/utils/docker_utils.py new file mode 100644 index 0000000..56f7e76 --- /dev/null +++ b/utils/docker_utils.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from io import BytesIO +import docker +import dockerfile as dockerfile_parser +from utils.cache_handler import retrieve_distro, update_images_cache +from utils.launchpad_api import get_distro_serie +from typing import Dict +import logging +logger = logging.getLogger(__name__.split('.')[0]) +logger.setLevel(logging.DEBUG) + +COMMANDS = [c.upper() for c in dockerfile_parser.all_cmds()] +KNOWN_DISTROS = ["ubuntu"] + + +def get_distro_info(image_info: str) -> str: + """ + Retrieve, if known, the distribution info from the given image + + :param image_info: name and tag of the image. Expected format : + :return: the distribution info in the format : + i.e. get_distro_info(ubuntu:20.04) return 'ubuntu:focal' + get_distro_info(myimage:mytag) return 'ubuntu:trusty' + """ + name, tag = image_info, 'latest' + + if ':' in image_info: + name, tag = image_info.split(":")[0], image_info.split(":")[1] + + distro_info = '' + if name.lower() not in KNOWN_DISTROS: + distro_info = get_image_distro(image_info) + + name = distro_info.split(':')[0] + tag = distro_info.split(':')[1] + + if name.lower() not in KNOWN_DISTROS: + return None + + distro_serie = get_distro_serie(name, tag) + + return name + ':' + distro_serie if distro_serie else None + + +def get_image_distro(_image: str) -> str: + """ + Pull the given image and retrieve the distribution info + + :param _image: name and tag of the image. Expected format : + :return: the distribution info in the format : + i.e. get_distro_info(myimage:mytag) return 'ubuntu:14.04' + """ + client = docker.from_env() + + # Check images cache + # todo: not working, temporary disabled + # distro_info = retrieve_distro(_image) + # + # if distro_info: + # return distro_info + + # Check already pulled image + is_pulled = False + for local_image in client.images.list(): + if _image in local_image.tags: + is_pulled = True + break + + if not is_pulled: + logger.info('-> Pulling the image: ' + _image + '. This may take some time...') + client.images.pull(_image) + + result = client.containers.run(_image, command="'cat /etc/os-release'", entrypoint="sh -c", remove=True) + os_release = parse_os_release(result.decode()) + + name = os_release["ID"] if "ID" in os_release else "" + version = os_release["VERSION_ID"] if "VERSION_ID" in os_release else "" + + if not is_pulled: + client.images.remove(_image, force=True) + + distro_info = name + ":" + version + # Update cache + # update_images_cache(_image, distro_info) # todo: not working, temporary disabled + + return distro_info + + +def parse_os_release(os_release: str) -> Dict: + """ + Parse the given os-release file content and return a dictionary containing the info + + :param os_release: os-release file content + :return: a dictionary containing the info + """ + parsed_dict = dict() + for line in os_release.splitlines(): + if '=' in line: + key, value = line.split('=', 1) + key = key.strip() + value = value.strip() + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + parsed_dict[key] = value + + return parsed_dict + + +def get_package_version(container, package) -> str: + """ + Pull the given image and retrieve the distribution info + + :param _image: name and tag of the image. Expected format : + :return: the distribution info in the format :' + """ + result = str(container.exec_run(f'apt-cache madison {package}')) + if result.strip(): + result = result.split('\n')[0].split('|')[1].strip() + + return result + + +def validate_package(_image: str, package: str) -> int: + """ + :param _image: name and tag of the image. Expected format : + :return: the distribution info in the format :' + """ + """ + global container + print(package) + result = str(container.exec_run(f'DEBIAN_FRONTEND=newt apt-get install -y {package}')) + print(result) + return int(result.split(',')[0].split('exit_code=')[1]) + """ + client = docker.from_env() + + dockerfile_str = 'FROM {}\nRUN apt-get update\nRUN yes | DEBIAN_FRONTEND=noninteractive apt-get install -yqq {}'.format(_image, package) + try: + client.images.build(fileobj=BytesIO(dockerfile_str.encode("utf-8")), tag='fix', rm=True, forcerm=True) + client.images.remove(image='fix', force=True) + return 0 + except Exception as e: + logger.error(e) + return 100 + + +def validate_shell(_image: str, shell_bin_path: str) -> int: + client = docker.from_env() + + dockerfile_str = 'FROM {}\nRUN which {}'.format(_image, shell_bin_path) + try: + client.images.build(fileobj=BytesIO(dockerfile_str.encode("utf-8")), tag='fix', rm=True, forcerm=True) + client.images.remove(image='fix', force=True) + return True + except Exception as e: + logger.error(e) + return False + + +def is_valid_dockerfile_command(command: str) -> bool: + """ + Check if the given string is a valid Dockerfile command + + :param command: name of the command + :return: True/False + """ + return command in COMMANDS + + +def parse_dockerfile_str(dockerfile_str) -> Dict: + parsed_dict = dict() + for cmd in dockerfile_parser.parse_string(dockerfile_str): + parsed_dict[cmd.start_line] = cmd + + return parsed_dict diff --git a/utils/dockerhub_api.py b/utils/dockerhub_api.py new file mode 100644 index 0000000..23c3545 --- /dev/null +++ b/utils/dockerhub_api.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import datetime as Date +import time +from datetime import datetime +from utils.common import request_data + +API_REQUEST_SLEEP = 2 + + +class DockerHubAPIException(Exception): + def __init__(self, msg='DockerHub API request failed', *args, **kwargs): + super().__init__(msg, *args, **kwargs) + + +class ImageNotFoundException(Exception): + """Raised when no image is found in the registry""" + def __init__(self, msg="No corresponding image found in the registry.", *args, **kwargs): + super().__init__(msg, *args, **kwargs) + + +def parse_image_path(image_name: str) -> str: + image_path = image_name + + if image_path.startswith("docker.io/"): + image_path = image_path.replace("docker.io/", "") + + if '/' not in image_path: + image_path = 'library/' + image_path + + image_path = image_path.split("/")[-2] + '/' + image_path.split("/")[-1] + + return image_path + + +def get_image_version(image_name: str, date: datetime) -> str: + """ + Retrieve the version of the given image from dockerhub registry. + + :param image_name: name of the image + :param date: date of image last push in the registry + :return: version of the image + """ + + start_page = 1 + checked_elements = 0 + search_size = 25 + + image_path = parse_image_path(image_name) + + api_url = 'https://hub.docker.com/v2/repositories/' + image_path + \ + '/tags/?page_size=' + str(search_size) + '&page=' + str(start_page) + \ + '&ordering=last_updated' + while api_url: + try: + response = request_data(api_url) + if response.status_code == 404: + raise ImageNotFoundException() + response = response.json() + api_url = response['next'] + except Exception as e: + raise DockerHubAPIException(e) from None + + images = response['results'] + images_number = response['count'] + + if images_number == 0: + raise ImageNotFoundException() + + for image_json in images: + date_string = image_json['tag_last_pushed'].split('T')[0] + image_date = Date.datetime.strptime(date_string, '%Y-%m-%d') + + if date > image_date: + return image_json['name'] + + start_page += 1 + checked_elements += search_size + + if checked_elements >= images_number: + return None + + time.sleep(API_REQUEST_SLEEP) + + +def get_latest_tag(image_name): + """ + Retrieve the latest tag 'equivalent' of the given image from dockerhub registry. + i.e. get_latest_tag(ubuntu) return 'focal' [in date 22/04/2021] + + :param image_name: name of the image. + If it's an official image, the username 'library/' is not needed. + Otherwise, the image_name has to be 'username/image' + :return: latest tag equivalent of the image + """ + + image_path = parse_image_path(image_name) + + page = 1 + page_size = 25 + latest_found = False + latest_digests = [] + + alt_tag = None + api_url = f'https://hub.docker.com/v2/repositories/{image_path}/tags/?' \ + f'page_size={str(page_size)}' \ + f'&page={str(page)}' \ + f'&ordering=last_updated' + while api_url: + try: + response = request_data(api_url) + if response.status_code == 404: + raise ImageNotFoundException() + response = response.json() + api_url = response['next'] + except Exception as e: + raise DockerHubAPIException(e) from None + + results = response['results'] + images_number = response['count'] + + if images_number == 0: + break + + if not latest_found: + for result in results: + # Find the digest of latest image + if result['name'] == 'latest': + latest_found = True + for image in result['images']: + if image['architecture'] == 'amd64' and image["os"] == "linux": + latest_digests.append(image['digest']) + break + + for result in results: + # Find the tag with the previous found digests + temp_digests = [] + for image in result['images']: + if image['digest'] in latest_digests and result['name'] != 'latest': + if result['name'][0].isdigit(): + return result['name'] + else: + alt_tag = result['name'] # set versions-less tag if there are no alternatives + + page += 1 + time.sleep(API_REQUEST_SLEEP) + + return alt_tag diff --git a/utils/launchpad_api.py b/utils/launchpad_api.py new file mode 100644 index 0000000..fa13d1e --- /dev/null +++ b/utils/launchpad_api.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import datetime as Date +import time +from datetime import datetime +import requests +from utils.dockerhub_api import get_latest_tag +from utils.common import request_data +import logging as logger +logger.getLogger('backoff').addHandler(logger.StreamHandler()) + + +class LaunchpadAPIException(Exception): + def __init__(self, msg='Launchpad API request failed', *args, **kwargs): + super().__init__(msg, *args, **kwargs) + + +def get_distro_serie(distro: str, tag: str) -> str: + """ + Retrieve the serie from the given distribution and tag. + i.e. get_distro_serie('ubuntu', '20.04') returns 'focal' + """ + + api_url = 'https://api.launchpad.net/1.0/' + distro + '/series' + + try: + response = request_data(api_url) + response = response.json() + except Exception as e: + raise LaunchpadAPIException() from None + + series = response['entries'] + + if series == 0: + return None + + # Get latest distro release tag + if tag == 'latest': + tag = get_latest_tag(distro) + + if '-' in tag: + tag = tag.split('-')[0] + + for serie_json in series: + if tag in serie_json['version'] or serie_json['version'] in tag or tag in serie_json['name']: + return serie_json['name'] + + return None + + +def get_package_binary_version(distro: str, distro_series: str, binary_name: str, date: datetime) -> str: + """ + Retrieve the version of the given package. + + :param distro: the distribution of the package + :param distro_series: the distro arch series of the given ubuntu distribution (e.g., precise, trusty, xenial, etc. + :param binary_name: name of the package + :param date: date of package release + :return: version of the package + """ + + start_element = 0 + search_size = 100 + + while True: + # URL encoding + binary_name = requests.utils.quote(binary_name) + + api_url = 'https://api.launchpad.net/1.0/' + distro + '/+archive/primary?ws.start=' + str(start_element) \ + + '&ws.size=' + str(search_size) \ + + '&ws.op=getPublishedBinaries' \ + + '&binary_name=' + binary_name \ + + '&status=Published' \ + + '&exact_match=true' \ + + '&order_by_date=true' + + try: + response = request_data(api_url) + response = response.json() + except Exception as e: + raise LaunchpadAPIException() from None + + binaries_number = response['total_size'] + binaries = response['entries'] + + if binaries_number == 0: + return None + + distro_arch_serie = 'https://api.launchpad.net/1.0/' + distro + '/' + distro_series + for binary_json in binaries: + if distro_arch_serie not in binary_json['distro_arch_series_link']: + continue + + date_string = binary_json['date_published'].split('T')[0] + parsed_date = Date.datetime.strptime(date_string, '%Y-%m-%d') + pocket = binary_json['pocket'] + valid_pocket = (pocket != 'Proposed' and pocket != 'Backports') + + if date > parsed_date and valid_pocket: + return binary_json['binary_package_version'] + + start_element = start_element + search_size + + if start_element >= binaries_number: + return None + + time.sleep(1) + + +def pkgs_repo_available(distro: str, distro_series: str) -> bool: + """ + Check if the given distro is EOL + + :param distro: the name of the distribution (e.g. ubuntu) + :param distro_series: the distro arch series of the given ubuntu distribution (e.g., precise, trusty, xenial, etc. + :return: True if the distro is EOL, False otherwise + """ + api_url = 'https://api.launchpad.net/1.0/{}/{}'.format(distro, distro_series) + + try: + response = request_data(api_url) + response = response.json() + except Exception as e: + raise LaunchpadAPIException() from None + + return response['active'] \ No newline at end of file