From 9987440b8e4968fa4d2459f7e1eeec2ace11193d Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 28 Jul 2021 13:09:52 +0000 Subject: [PATCH 1/4] Use python 3.9 --- pipelines/azure-build.yaml | 176 ++++++++++++++++++------------------- pipelines/azure-test.yaml | 54 ++++++------ 2 files changed, 115 insertions(+), 115 deletions(-) diff --git a/pipelines/azure-build.yaml b/pipelines/azure-build.yaml index 28b0195..430be17 100644 --- a/pipelines/azure-build.yaml +++ b/pipelines/azure-build.yaml @@ -6,93 +6,93 @@ trigger: pr: none pool: - vmImage: 'ubuntu-18.04' + vmImage: "ubuntu-18.04" stages: -- stage: build - jobs: - - job: build_package - displayName: Build Package - steps: - - task: UsePythonVersion@0 - displayName: Set python version - inputs: {versionSpec: 3.8} - - script: | - export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"} - export COMMIT=`git rev-parse --verify ${BUILD_SOURCEBRANCH}` - export BRANCH=`git ls-remote --heads origin | grep $COMMIT | sed "s/.*\///"` - echo "Building $VERSION On branch $BRANCH" - if [[ "$VERSION" == *stable* ]] && [[ $BRANCH != "master" ]]; then exit 1; fi - if [[ "$VERSION" == *dev* ]] && [[ $BRANCH != "dev" ]]; then exit 1; fi - export VERSION=${VERSION/stable} - export VERSION=${VERSION/beta/b} - echo $VERSION > assemblyline_service_client/VERSION - sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U wheel cython pip - python setup.py bdist_wheel - - publish: $(System.DefaultWorkingDirectory)/dist/ - artifact: dist -# - publish: $(System.DefaultWorkingDirectory)/test/ -# artifact: test -#- stage: test -# jobs: -# - job: run_test -# strategy: -# matrix: -# python3_7: -# python.version: '3.7' -# Python3_8: -# python.version: '3.8' -# timeoutInMinutes: 10 -# steps: -# - checkout: none -# - task: UsePythonVersion@0 -# displayName: Set python version -# inputs: -# versionSpec: '$(python.version)' -# - download: current -# artifact: dist -# - download: current -# artifact: test -# - script: | -# set -x # Echo commands before they are run -# sudo apt-get update -# sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev -# export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"} -# sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -f $(Pipeline.Workspace)/dist/ "assemblyline-service-client==${VERSION}" -r $(Pipeline.Workspace)/test/requirements.txt -# sudo mkdir -p /etc/assemblyline/ -# sudo mkdir -p /var/cache/assemblyline/ -# sudo chmod a+rw /var/cache/assemblyline/ -# sudo cp $(Pipeline.Workspace)/pipelines/config.yml /etc/assemblyline -# displayName: Install package -# - script: pytest -rsx -vv -# workingDirectory: $(Pipeline.Workspace)/test -# displayName: Test -- stage: deploy - jobs: - - job: deploy - displayName: Deploy packages and containers - variables: - - group: deployment-information - steps: - - download: current - artifact: dist - - script: | - set -xv # Echo commands before they are run - sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine - ls dist - twine upload --skip-existing --repository-url $TEST_REPOSITORY_URL dist/* - workingDirectory: $(Pipeline.Workspace) - displayName: Deploy to Test PyPI - env: - TWINE_USERNAME: $(twineUsername) - TWINE_PASSWORD: $(twinePassword) - - script: | - set -xv # Echo commands before they are run - sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine - ls dist - twine upload --skip-existing dist/* - workingDirectory: $(Pipeline.Workspace) - displayName: Deploy to PyPI - env: - TWINE_USERNAME: $(twineUsername) - TWINE_PASSWORD: $(twinePassword) + - stage: build + jobs: + - job: build_package + displayName: Build Package + steps: + - task: UsePythonVersion@0 + displayName: Set python version + inputs: { versionSpec: 3.9 } + - script: | + export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"} + export COMMIT=`git rev-parse --verify ${BUILD_SOURCEBRANCH}` + export BRANCH=`git ls-remote --heads origin | grep $COMMIT | sed "s/.*\///"` + echo "Building $VERSION On branch $BRANCH" + if [[ "$VERSION" == *stable* ]] && [[ $BRANCH != "master" ]]; then exit 1; fi + if [[ "$VERSION" == *dev* ]] && [[ $BRANCH != "dev" ]]; then exit 1; fi + export VERSION=${VERSION/stable} + export VERSION=${VERSION/beta/b} + echo $VERSION > assemblyline_service_client/VERSION + sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U wheel cython pip + python setup.py bdist_wheel + - publish: $(System.DefaultWorkingDirectory)/dist/ + artifact: dist + # - publish: $(System.DefaultWorkingDirectory)/test/ + # artifact: test + #- stage: test + # jobs: + # - job: run_test + # strategy: + # matrix: + # python3_7: + # python.version: '3.7' + # Python3_8: + # python.version: '3.8' + # timeoutInMinutes: 10 + # steps: + # - checkout: none + # - task: UsePythonVersion@0 + # displayName: Set python version + # inputs: + # versionSpec: '$(python.version)' + # - download: current + # artifact: dist + # - download: current + # artifact: test + # - script: | + # set -x # Echo commands before they are run + # sudo apt-get update + # sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev + # export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"} + # sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -f $(Pipeline.Workspace)/dist/ "assemblyline-service-client==${VERSION}" -r $(Pipeline.Workspace)/test/requirements.txt + # sudo mkdir -p /etc/assemblyline/ + # sudo mkdir -p /var/cache/assemblyline/ + # sudo chmod a+rw /var/cache/assemblyline/ + # sudo cp $(Pipeline.Workspace)/pipelines/config.yml /etc/assemblyline + # displayName: Install package + # - script: pytest -rsx -vv + # workingDirectory: $(Pipeline.Workspace)/test + # displayName: Test + - stage: deploy + jobs: + - job: deploy + displayName: Deploy packages and containers + variables: + - group: deployment-information + steps: + - download: current + artifact: dist + - script: | + set -xv # Echo commands before they are run + sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine + ls dist + twine upload --skip-existing --repository-url $TEST_REPOSITORY_URL dist/* + workingDirectory: $(Pipeline.Workspace) + displayName: Deploy to Test PyPI + env: + TWINE_USERNAME: $(twineUsername) + TWINE_PASSWORD: $(twinePassword) + - script: | + set -xv # Echo commands before they are run + sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine + ls dist + twine upload --skip-existing dist/* + workingDirectory: $(Pipeline.Workspace) + displayName: Deploy to PyPI + env: + TWINE_USERNAME: $(twineUsername) + TWINE_PASSWORD: $(twinePassword) diff --git a/pipelines/azure-test.yaml b/pipelines/azure-test.yaml index 055f7e9..8b975fd 100644 --- a/pipelines/azure-test.yaml +++ b/pipelines/azure-test.yaml @@ -4,34 +4,34 @@ trigger: ["*"] pr: ["*"] pool: - vmImage: 'ubuntu-18.04' + vmImage: "ubuntu-18.04" jobs: -- job: run_test - strategy: - matrix: - python3_7: - python.version: '3.7' - Python3_8: - python.version: '3.8' + - job: run_test + strategy: + matrix: + python3_9: + python.version: "3.9" + # Python3_10: + # python.version: '3.10' - timeoutInMinutes: 10 + timeoutInMinutes: 10 - steps: - - task: UsePythonVersion@0 - displayName: Set python version - inputs: - versionSpec: '$(python.version)' - - script: | - sudo mkdir -p /etc/assemblyline/ - sudo mkdir -p /var/cache/assemblyline/ - sudo cp pipelines/config.yml /etc/assemblyline - sudo chmod a+rw /var/cache/assemblyline/ - sudo apt-get update - sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev - sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U pip cython setuptools - sudo env "PATH=$PATH" python -m pip install --no-cache-dir -e . - sudo env "PATH=$PATH" python -m pip install --no-cache-dir -r test/requirements.txt - displayName: Setup environment - - script: python -m pytest -rsx -vv - displayName: Test + steps: + - task: UsePythonVersion@0 + displayName: Set python version + inputs: + versionSpec: "$(python.version)" + - script: | + sudo mkdir -p /etc/assemblyline/ + sudo mkdir -p /var/cache/assemblyline/ + sudo cp pipelines/config.yml /etc/assemblyline + sudo chmod a+rw /var/cache/assemblyline/ + sudo apt-get update + sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev + sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U pip cython setuptools + sudo env "PATH=$PATH" python -m pip install --no-cache-dir -e . + sudo env "PATH=$PATH" python -m pip install --no-cache-dir -r test/requirements.txt + displayName: Setup environment + - script: python -m pytest -rsx -vv + displayName: Test From d738c8809837ed2cba9302caca6605ca80d0dc51 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 28 Jul 2021 13:37:58 +0000 Subject: [PATCH 2/4] Use newer VM for testing --- pipelines/azure-build.yaml | 2 +- pipelines/azure-test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/azure-build.yaml b/pipelines/azure-build.yaml index 430be17..38099be 100644 --- a/pipelines/azure-build.yaml +++ b/pipelines/azure-build.yaml @@ -6,7 +6,7 @@ trigger: pr: none pool: - vmImage: "ubuntu-18.04" + vmImage: "ubuntu-20.04" stages: - stage: build diff --git a/pipelines/azure-test.yaml b/pipelines/azure-test.yaml index 8b975fd..8a28168 100644 --- a/pipelines/azure-test.yaml +++ b/pipelines/azure-test.yaml @@ -4,7 +4,7 @@ trigger: ["*"] pr: ["*"] pool: - vmImage: "ubuntu-18.04" + vmImage: "ubuntu-20.04" jobs: - job: run_test From c311c70fa668d42f9b5ccaa4b5b51775ecd87044 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 28 Jul 2021 16:55:38 +0000 Subject: [PATCH 3/4] Cleanup setup.py --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67bc584..dfe4cde 100644 --- a/setup.py +++ b/setup.py @@ -25,12 +25,13 @@ author_email="assemblyline@cyber.gc.ca", license="MIT", classifiers=[ - 'Development Status :: 2 - Pre-Alpha', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Topic :: Software Development :: Libraries', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], keywords="assemblyline automated malware analysis gc canada cse-cst cse cst cyber cccs", packages=find_packages(exclude=['test/*']), From d1d19e0162c22855c25e83c8c49ef54ff2807cfd Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 28 Jul 2021 16:55:54 +0000 Subject: [PATCH 4/4] Cleanup build pipeline --- pipelines/azure-build.yaml | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/pipelines/azure-build.yaml b/pipelines/azure-build.yaml index 38099be..f778a24 100644 --- a/pipelines/azure-build.yaml +++ b/pipelines/azure-build.yaml @@ -31,42 +31,6 @@ stages: python setup.py bdist_wheel - publish: $(System.DefaultWorkingDirectory)/dist/ artifact: dist - # - publish: $(System.DefaultWorkingDirectory)/test/ - # artifact: test - #- stage: test - # jobs: - # - job: run_test - # strategy: - # matrix: - # python3_7: - # python.version: '3.7' - # Python3_8: - # python.version: '3.8' - # timeoutInMinutes: 10 - # steps: - # - checkout: none - # - task: UsePythonVersion@0 - # displayName: Set python version - # inputs: - # versionSpec: '$(python.version)' - # - download: current - # artifact: dist - # - download: current - # artifact: test - # - script: | - # set -x # Echo commands before they are run - # sudo apt-get update - # sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev - # export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"} - # sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -f $(Pipeline.Workspace)/dist/ "assemblyline-service-client==${VERSION}" -r $(Pipeline.Workspace)/test/requirements.txt - # sudo mkdir -p /etc/assemblyline/ - # sudo mkdir -p /var/cache/assemblyline/ - # sudo chmod a+rw /var/cache/assemblyline/ - # sudo cp $(Pipeline.Workspace)/pipelines/config.yml /etc/assemblyline - # displayName: Install package - # - script: pytest -rsx -vv - # workingDirectory: $(Pipeline.Workspace)/test - # displayName: Test - stage: deploy jobs: - job: deploy