Land #19413, Add automated acceptance tests for cmd_exec API

.github/workflows/command_shell_acceptance.yml

@@ -0,0 +1,221 @@
+
+name: Acceptance
+
+# Optional, enabling concurrency limits: https://docs.github.com/en/actions/using-jobs/using-concurrency
+#concurrency:
+# group: ${{ github.ref }}-${{ github.workflow }}
+# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+permissions:
+ actions: none
+ checks: none
+ contents: none
+ deployments: none
+ id-token: none
+ issues: none
+ discussions: none
+ packages: none
+ pages: none
+ pull-requests: none
+ repository-projects: none
+ security-events: none
+ statuses: none
+
+on:
+ workflow_dispatch:
+ inputs:
+ metasploitPayloadsCommit:
+ description: 'metasploit-payloads branch would like to test'
+ required: true
+ default: 'master'
+ mettleCommit:
+ description: 'mettle branch you would like to test'
+ required: true
+ default: 'master'
+ push:
+ branches-ignore:
+ - gh-pages
+ - metakitty
+ pull_request:
+ branches:
+ - '*'
+ paths:
+ - 'metsploit-framework.gemspec'
+ - 'Gemfile.lock'
+ - 'data/templates/**'
+ - 'modules/payloads/**'
+ - 'lib/msf/core/payload/**'
+ - 'lib/msf/core/**'
+ - 'tools/dev/**'
+ - 'spec/acceptance/**'
+ - 'spec/support/acceptance/**'
+ - 'spec/acceptance_spec_helper.rb'
+ - '.github/**'
+# Example of running as a cron, to weed out flaky tests
+# schedule:
+# - cron: '*/15 * * * *'
+
+jobs:
+ # Run all test individually, note there is a separate final job for aggregating the test results
+ test:
+ strategy:
+ fail-fast: false
+ matrix:
+ os:
+ - windows-2019
+ - ubuntu-20.04
+ ruby:
+ - 3.0.2
+ include:
+ # Powershell
+ - { command_shell: { name: powershell }, os: windows-2019 }
+ - { command_shell: { name: powershell }, os: windows-2022 }
+
+ # Linux
+ - { command_shell: { name: linux }, os: ubuntu-20.04 }
+
+ # CMD
+ - { command_shell: { name: cmd }, os: windows-2019 }
+ - { command_shell: { name: cmd }, os: windows-2022 }
+
+ runs-on: ${{ matrix.os }}
+
+ timeout-minutes: 50
+
+ env:
+ RAILS_ENV: test
+ HOST_RUNNER_IMAGE: ${{ matrix.os }}
+ SESSION: 'command_shell/${{ matrix.command_shell.name }}'
+ SESSION_RUNTIME_VERSION: ${{ matrix.command_shell.runtime_version }}
+ BUNDLE_WITHOUT: "coverage development"
+
+ name: ${{ matrix.command_shell.name }} ${{ matrix.command_shell.runtime_version }} ${{ matrix.os }}
+ steps:
+ - name: Install system dependencies (Linux)
+ if: runner.os == 'Linux'
+ run: sudo apt-get -y --no-install-recommends install libpcap-dev graphviz
+
+ - uses: shivammathur/setup-php@fc14643b0a99ee9db10a3c025a33d76544fa3761
+ if: ${{ matrix.command_shell.name == 'php' }}
+ with:
+ php-version: ${{ matrix.command_shell.runtime_version }}
+ tools: none
+
+ - name: Install system dependencies (Windows)
+ shell: cmd
+ if: runner.os == 'Windows'
+ run: |
+ REM pcap dependencies
+ powershell -Command "[System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true} ; [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; (New-Object System.Net.WebClient).DownloadFile('https://www.winpcap.org/install/bin/WpdPack_4_1_2.zip', 'C:\Windows\Temp\WpdPack_4_1_2.zip')"
+
+ choco install 7zip.installServerCertificateValidationCallback
+ 7z x "C:\Windows\Temp\WpdPack_4_1_2.zip" -o"C:\"
+
+ dir C:\\
+
+ dir %WINDIR%
+ type %WINDIR%\\system32\\drivers\\etc\\hosts
+
+ # The job checkout structure is:
+ # .
+ # └── metasploit-framework
+
+ - name: Checkout metasploit-framework code
+ uses: actions/checkout@v4
+ with:
+ path: metasploit-framework
+
+ - name: Setup Ruby
+ env:
+ BUNDLE_FORCE_RUBY_PLATFORM: true
+ uses: ruby/setup-ruby@v1
+ with:
+ ruby-version: ${{ matrix.ruby }}
+ bundler-cache: true
+ working-directory: metasploit-framework
+ cache-version: 5
+ # Github actions with Ruby requires Bundler 2.2.18+
+ # https://github.com/ruby/setup-ruby/tree/d2b39ad0b52eca07d23f3aa14fdf2a3fcc1f411c#windows
+ bundler: 2.2.33
+
+ - name: Acceptance
+ env:
+ SPEC_HELPER_LOAD_METASPLOIT: false
+ SPEC_OPTS: "--tag acceptance --require acceptance_spec_helper.rb --color --format documentation --format AllureRspec::RSpecFormatter"
+ # Unix run command:
+ # SPEC_HELPER_LOAD_METASPLOIT=false bundle exec ./spec/acceptance
+ # Windows cmd command:
+ # set SPEC_HELPER_LOAD_METASPLOIT=false
+ # bundle exec rspec .\spec\acceptance
+ # Note: rspec retry is intentionally not used, as it can cause issues with allure's reporting
+ # Additionally - flakey tests should be fixed or marked as flakey instead of silently retried
+ run: |
+ bundle exec rspec spec/acceptance/command_shell_spec.rb
+ working-directory: metasploit-framework
+
+ - name: Archive results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ # Provide a unique artifact for each matrix os, otherwise race conditions can lead to corrupt zips
+ name: raw-data-${{ matrix.command_shell.name }}-${{ matrix.command_shell.runtime_version }}-${{ matrix.os }}
+ path: metasploit-framework/tmp/allure-raw-data
+
+ # Generate a final report from the previous test results
+ report:
+ name: Generate report
+ needs: test
+ runs-on: ubuntu-latest
+ if: always()
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ if: always()
+
+ - name: Install system dependencies (Linux)
+ if: always()
+ run: sudo apt-get -y --no-install-recommends install libpcap-dev graphviz
+
+ - name: Setup Ruby
+ if: always()
+ env:
+ BUNDLE_FORCE_RUBY_PLATFORM: true
+ uses: ruby/setup-ruby@v1
+ with:
+ ruby-version: '${{ matrix.ruby }}'
+ bundler-cache: true
+ cache-version: 4
+ # Github actions with Ruby requires Bundler 2.2.18+
+ # https://github.com/ruby/setup-ruby/tree/d2b39ad0b52eca07d23f3aa14fdf2a3fcc1f411c#windows
+ bundler: 2.2.33
+
+ - uses: actions/download-artifact@v4
+ id: download
+ if: always()
+ with:
+ # Note: Not specifying a name will download all artifacts from the previous workflow jobs
+ path: raw-data
+
+ - name: allure generate
+ if: always()
+ run: |
+ export VERSION=2.22.1
+
+ curl -o allure-$VERSION.tgz -Ls https://github.com/allure-framework/allure2/releases/download/$VERSION/allure-$VERSION.tgz
+ tar -zxvf allure-$VERSION.tgz -C .
+
+ ls -la ${{steps.download.outputs.download-path}}
+ ./allure-$VERSION/bin/allure generate ${{steps.download.outputs.download-path}}/* -o ./allure-report
+
+ find ${{steps.download.outputs.download-path}}
+ bundle exec ruby tools/dev/report_generation/support_matrix/generate.rb --allure-data ${{steps.download.outputs.download-path}} > ./allure-report/support_matrix.html
+
+ - name: archive results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: final-report-${{ github.run_id }}
+ path: |
+ ./allure-report

.github/workflows/meterpreter_acceptance.yml

@@ -248,7 +248,7 @@ jobs:
 
  - name: Build Windows payloads via Visual Studio 2022 Build (Windows)
  shell: cmd
- if: ${{ (runner.os == 'Windows') && (matrix.os == 'windows-2022') && (contains(github.event.issue.labels.*.name, 'payload-testing-branch'))}}
+ if: ${{ (runner.os == 'Windows') && (matrix.os == 'windows-2022') && (contains(github.event.issue.labels.*.name, 'payload-testing-branch')) }}
  run: |
  cd c/meterpreter
  git submodule init && git submodule update

data/cmd_exec/README.md

@@ -1,25 +1,29 @@
 ## Setup
 
-This contains setup steps used for acceptance testing of the `cmd_exec` API. We will make use of the gcc docker image to 
-build out the C binaries to then be uploaded to the host machine, so they can be used as part of the `cmd_exec` 
+This contains setup steps used for acceptance testing of the `cmd_exec` API. We will make use of the gcc docker image to
+build out the C binaries to then be uploaded to the host machine, so they can be used as part of the `cmd_exec`
 create process API.
 
 This directory contains:
 - C executable `show_args.c`
-This file is used as part of the `cmd_exec` testing as it requires a file to take args, then loop over them and output 
-those args back to the user.
+ This file is used as part of the `cmd_exec` testing as it requires a file to take args, then loop over them and output
+ those args back to the user.
 
 - Makefile to build the binaries `makefile.mk`
-This file is used to create the binaries for both Windows and Linux that the docker command below will make use of.
+ This file is used to create the binaries for both Windows and Linux that the docker command below will make use of.
+ This will output the following binaries:
 
-- Precompiled binaries for Windows
- - `show_args.exe`
+ - Precompiled binary for Windows
+  - `show_args.exe`
 
-- Precompiled binaries for Linux and Mettle
- - `show_args`
+ - Precompiled binary for Linux and Mettle
+ - `show_args`
+
+### Note
+
+You will need to compile the OSX payload separately on an OSX machine, Docker is not supported. The test assume the file
+will be named as `show_args_macos`.
 
-- Precompiled binaries for macOS
- - `show_args_macos`
 
 ## Compile binaries locally
 
@@ -29,5 +33,3 @@ We make use of gcc for this: https://hub.docker.com/_/gcc
 ```shell
 docker run --rm -v "$PWD":/usr/src/myapp -w /usr/src/myapp gcc:11.4.0 /bin/bash -c "apt update && apt install -y gcc-mingw-w64 && make all -f makefile.mk"
 ```
-
-You will need to compile the OSX payload separately on an OSX machine, Docker is not supported.

lib/msf/core/session/provider/single_command_shell.rb

@@ -93,6 +93,14 @@ def shell_command_token(cmd, timeout=10)
  output
  end
 
+ def to_cmd(cmd, args)
+ if platform == 'windows'
+ result = Msf::Sessions::CommandShellWindows.to_cmd(cmd, args)
+ else
+ result = Msf::Sessions::CommandShellUnix.to_cmd(cmd, args)
+ end
+ end
+
  # We don't know initially whether the shell we have is one that
  # echos input back to the output stream. If it is, we need to
  # take this into account when using tokens to extract the data corresponding