feat: NewDocker File and Pipelins

.github/workflows/release.yml

@@ -21,7 +21,7 @@ jobs:
         with:
           app_id: ${{ secrets.APP_ID_ADMIN_GITHUB }}
           private_key: ${{ secrets.APP_PRIVATE_KEY_ADMIN_GITHUB }}
-      
+
       # Check out the current repository
       - name: Fetch Sources
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

azure-pipelines.yml

@@ -0,0 +1,39 @@
+
+trigger:
+- trunk
+
+pool:
+  name: Azure Pipelines
+
+steps:
+
+- task: connect-agent@2
+  displayName: 'Hosted agent'
+  inputs:
+    hostedAgentService: 90469226-70c7-4fd9-acf1-a6f54739e00a
+
+- script: |
+    version=$(python3 -c "import sys; sys.path.append('tools/devsecops_engine_tools'); import version; print(version.version)")
+    echo "##vso[task.setvariable variable=RELEASE_VERSION]$version"
+  displayName: 'Set lib version from version.py'
+
+- script: echo "Release version is $(RELEASE_VERSION)"
+  displayName: 'Use the release version'
+
+- script: python3 docker/get_files_ad.py $(repo-owner-ad) "$(project-ad)" NU0429001_DevSecOps_Remote_Config /engine_sast/engine_iac/ConfigTool.json $(ad-at) path
+  displayName: 'Get Custom Remote Config'
+
+- script: python3 docker/get_files_gh.py $(repo-owner) $(project) rules $(gh-at) rules
+  displayName: 'Get Custom IAC Rules'
+
+- script: docker build --build-arg VERSION=$(RELEASE_VERSION) -t artifactory.apps.bancolombia.com/devops/devsecops-engine-tools:$(RELEASE_VERSION) -f docker/Dockerfile . && docker tag artifactory.apps.bancolombia.com/devops/devsecops-engine-tools:$(RELEASE_VERSION) artifactory.apps.bancolombia.com/devops/devsecops-engine-tools:$(RELEASE_VERSION)
+  displayName: 'Build Docker Image'
+
+- task: ArtifactoryDocker@1
+  displayName: Docker Push
+  inputs:
+    command: 'push'
+    repository: 'devops/devsecops-engine-tools'
+    artifactoryService: 'Artifactory'
+    targetRepo: 'devops/devsecops-engine-tools'
+    imageName: 'artifactory.apps.bancolombia.com/devops/devsecops-engine-tools:$(RELEASE_VERSION)'

docker/Dockerfile

@@ -4,6 +4,9 @@ WORKDIR /app
 ARG VERSION
 ENV APP_VERSION=$VERSION
 
+ARG REMOTE_CONFIG="./example_remote_config_local"
+ENV REMOTE_CONFIG_PATH=$REMOTE_CONFIG
+
 ENV DET_PIPELINE_NAME="" \
     DET_PATH_DIRECTORY="." \
     DET_OS="Linux" \
@@ -28,8 +31,9 @@ ENV DET_PIPELINE_NAME="" \
 RUN apt-get update && apt-get install -y bash
 RUN apt-get update && apt-get install -y dos2unix
 
-COPY ./example_remote_config_local /app/example_remote_config_local
+COPY $REMOTE_CONFIG /app/example_remote_config_local
 COPY ./docker/remoteConfigGeneration.sh /app/remoteConfigGeneration.sh
+COPY ./rules* /tmp/rules
 
 RUN dos2unix /app/remoteConfigGeneration.sh
 
@@ -43,11 +47,38 @@ RUN apt-get update && \
     python3-venv \
     python3-pip
 
+# Update and install prerequisites
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# Add Docker’s official GPG key
+RUN install -m 0755 -d /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \
+    chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add Docker repository to APT sources
+RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo \"$VERSION_CODENAME\") stable" \
+    | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+# Update package list and install Docker
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    docker-ce \
+    docker-ce-cli \
+    containerd.io \
+    docker-buildx-plugin \
+    docker-compose-plugin && \
+    rm -rf /var/lib/apt/lists/*
+
 RUN ln -s /usr/bin/python3 /usr/local/bin/python3 && \
     ln -s /usr/bin/pip3 /usr/local/bin/pip3
 
 RUN python3 -m venv venv
 RUN python3 -m pip install --break-system-packages checkov==2.3.296
+
 RUN python3 -m pip install --break-system-packages -i https://pypi.org/simple devsecops-engine-tools==$APP_VERSION
 
 CMD ["sh", "-c", "dockerd & while ! docker info > /dev/null 2>&1; do sleep 1; done; sh"]

docker/get_files_ad.py

@@ -0,0 +1,43 @@
+import requests
+import sys
+import json
+
+def download_folder_from_azure_devops(organization, project, repository, path, token, download_path):
+    print(download_path)
+    url = f"https://dev.azure.com/{organization}/{project}/_apis/git/repositories/{repository}/items?scopePath={path}&recursionLevel=Full&api-version=7.1"
+    headers = {"Authorization": f"Basic {token}"}
+    response = requests.get(url, headers=headers)
+
+    if response.status_code == 200:
+        data = response.json()
+        data['CHECKOV']['USE_EXTERNAL_CHECKS_DIR'] = "True"
+        content = str(json.dumps(data))
+        file_path = "example_remote_config_local/engine_sast/engine_iac/ConfigTool.json"
+        with open(file_path, "w") as file:
+            file.write(content)
+            print(f"File '{file_path}' downloaded successfully.")
+    else:
+        print(f"Error downloading folder. Status code: {response.status_code}")
+        print(response.text)
+
+# Example usage
+# organization = "your-organization"
+# project = "your-project"
+# repository = "your-repository"
+# path = "path/to/your/folder"
+# token = "your-personal-access-token"
+# download_path = "."
+
+if __name__ == "__main__":
+    if len(sys.argv) < 7:
+        print("Usage: python get_files_ad.py <organization> <project> <repository> <path> <token> <download_path>")
+        sys.exit(1)
+
+    organization = sys.argv[1]
+    project = sys.argv[2]
+    repository = sys.argv[3]
+    path = sys.argv[4] # path to download
+    token = sys.argv[5] # your personal access token
+    download_path = sys.argv[6]
+
+    download_folder_from_azure_devops(organization, project, repository, path, token, download_path)

docker/get_files_gh.py

@@ -0,0 +1,49 @@
+import requests
+import os
+import sys
+from concurrent.futures import ThreadPoolExecutor
+
+def download_folder_from_github(owner, repository, path, token, download_path):
+    url = f"https://api.github.com/repos/{owner}/{repository}/contents/{path}"
+    headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3.raw"}
+    response = requests.get(url, headers=headers)
+
+    if response.status_code == 200:
+        contents = response.json()
+        if isinstance(contents, list):
+            # It's a directory
+            with ThreadPoolExecutor(max_workers=5) as executor:
+                for item in contents:
+                    item_path = item['path']
+                    if item['type'] == 'file':
+                        executor.submit(download_file_from_github, owner, repository, item_path, token, download_path)
+                    elif item['type'] == 'dir':
+                        new_download_path = os.path.join(download_path, os.path.basename(item_path))
+                        os.makedirs(new_download_path, exist_ok=True)
+                        executor.submit(download_folder_from_github, owner, repository, item_path, token, new_download_path)
+        else:
+            print(f"Error: Expected a directory but got a file at {path}")
+    else:
+        print(f"Error downloading folder. Status code: {response.status_code}")
+
+def download_file_from_github(owner, repository, path, token, download_path="."):
+    url = f"https://api.github.com/repos/{owner}/{repository}/contents/{path}"
+    headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3.raw"}
+    response = requests.get(url, headers=headers)
+
+    if response.status_code == 200:
+        file_name = os.path.basename(path)
+        file_path = os.path.join(download_path, file_name)
+        with open(file_path, "wb") as file:
+            file.write(response.content)
+        print(f"File {file_name} downloaded successfully to {file_path}")
+    else:
+        print(f"Error downloading file. Status code: {response.status_code}")
+
+owner = sys.argv[1]
+repository = sys.argv[2]
+path = sys.argv[3]
+token = sys.argv[4]
+download_path = sys.argv[5]
+
+download_folder_from_github(owner, repository, path, token, download_path)

docker/remoteConfigGeneration.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
 mv /app/example_remote_config_local /app/docker_default_remote_config
-json_file="/app/docker_default_remote_config/engine_core/ConfigTool.json"
-sed -i 's/"TOOL": "CHECKOV|KUBESCAPE|KICS"/"TOOL": "CHECKOV"/' "$json_file"
+json_file_iac="/app/docker_default_remote_config/engine_core/ConfigTool.json"
+sed -i 's/"TOOL": "CHECKOV|KUBESCAPE|KICS"/"TOOL": "CHECKOV"/' "$json_file_iac"
+
+json_file_container="/app/docker_default_remote_config/engine_core/ConfigTool.json"
+sed -i 's/"TOOL": "PRISMA|TRIVY"/"TOOL": "TRIVY"/' "$json_file_container"