From 27e7d72a8098a098d17c91ae7c93df50fa19668d Mon Sep 17 00:00:00 2001 From: James Murdza Date: Thu, 24 Oct 2024 05:21:55 -0600 Subject: [PATCH] Add CI tests with Markdown output --- .github/workflows/update-tests-md.yml | 56 +++++++++++++++++++++++++++ tests/examples.test.ts | 19 ++++----- tests/updateTestsMd.js | 39 +++++++++++++++++++ tests/utils.ts | 15 +++++-- 4 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/update-tests-md.yml create mode 100644 tests/updateTestsMd.js diff --git a/.github/workflows/update-tests-md.yml b/.github/workflows/update-tests-md.yml new file mode 100644 index 00000000..3e004a5b --- /dev/null +++ b/.github/workflows/update-tests-md.yml @@ -0,0 +1,56 @@ +name: Update Tests Results + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + run_tests_and_update_markdown: + runs-on: ubuntu-latest + + env: + E2B_API_KEY: ${{ secrets.E2B_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + FIRECRAWL_API_KEY: ${{ secrets.FIRECRAWL_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: '20' + + - name: Install dependencies + run: npm install + + - name: Run Jest tests and save results + run: | + npm test -- --json --outputFile=./results.json + continue-on-error: true # Ensures the job continues even if tests fail + + - name: Upload test results as an artifact + uses: actions/upload-artifact@v3 + with: + name: test-results + path: ./results.json + + - name: Update Tests.md with test results + run: node ./tests/updateTestsMd.js # Your custom script to update Tests.md with test results + + - name: Upload markdown report as an artifact + uses: actions/upload-artifact@v3 + with: + name: test-results-md + path: ./Tests.md \ No newline at end of file diff --git a/tests/examples.test.ts b/tests/examples.test.ts index 22ae5be2..828b9048 100644 --- a/tests/examples.test.ts +++ b/tests/examples.test.ts @@ -3,7 +3,7 @@ import path from 'path'; import dotenv from 'dotenv'; import Sandbox from 'e2b'; -import { uploadPathToPath, readEnvFile } from "./utils" +import { uploadPathToPath, getApiKeys } from "./utils" // Read the E2B API key dotenv.config({ path: path.resolve(process.cwd(), ".env") }); @@ -95,7 +95,7 @@ describe('Integration test for multiple scripts in e2b sandbox', () => { }); scripts.forEach(({ name, interpreter, file : examplePath }) => { - it.concurrent(`should upload and execute ${name} successfully in e2b sandbox`, async () => { + it.concurrent(name, async () => { let attempts = 0; const maxAttempts = 3; @@ -104,9 +104,10 @@ describe('Integration test for multiple scripts in e2b sandbox', () => { while (attempts < maxAttempts && !success) { attempts++; + // Create a new E2B sandbox + const sandbox = await Sandbox.create({ timeoutMs: SANDBOX_TIMEOUT }); + try { - // Create a new E2B sandbox - const sandbox = await Sandbox.create({ timeoutMs: SANDBOX_TIMEOUT }); // Upload the example directory to the sandbox. await uploadPathToPath(examplePath, SANDBOX_TEST_DIRECTORY, sandbox); @@ -132,13 +133,10 @@ describe('Integration test for multiple scripts in e2b sandbox', () => { stdoutData += output; await fs.appendFile(logFilePath, output); }, - envs: readEnvFile(), + envs: getApiKeys(), timeoutMs: COMMAND_TIMEOUT, }); - // Kill the sandbox - await sandbox.kill(); - // Check the exit code to see if the test passed if (result.exitCode !== 0) { await fs.appendFile(logFilePath, `Attempt ${attempts}: Test for ${name} failed with exit code ${result.exitCode}\n`); @@ -156,7 +154,10 @@ describe('Integration test for multiple scripts in e2b sandbox', () => { await fs.appendFile(logFilePath, `Test for ${name} completed successfully on attempt ${attempts}.\n`); } } catch (error) { - console.log(`Attempt ${attempts}/${maxAttempts}: An error occurred while running the test for ${name}`); + console.log(`Attempt ${attempts}/${maxAttempts}: An error occurred while running the test for ${name}`, error); + } finally { + // Kill the sandbox + await sandbox.kill(); } if (!success && attempts === maxAttempts) { diff --git a/tests/updateTestsMd.js b/tests/updateTestsMd.js new file mode 100644 index 00000000..185b387e --- /dev/null +++ b/tests/updateTestsMd.js @@ -0,0 +1,39 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +// Get the current directory name +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Define paths for the results and markdown files +const resultsPath = path.join(__dirname, 'results.json'); +const markdownPath = path.join(__dirname, 'Tests.md'); + +// Read the Jest results +fs.readFile(resultsPath, 'utf8') + .then((data) => { + const results = JSON.parse(data); + + // Start constructing the Markdown table + let markdownContent = `# Test Results\n\n`; + markdownContent += `| Test Name | Status |\n`; + markdownContent += `|-----------|--------|\n`; + + // Iterate through each test result and populate the Markdown table + results.testResults.forEach((test) => { + const testName = test.assertionResults[0].fullName; + const status = test.status === 'passed' ? '✅ Passed' : '❌ Failed'; + markdownContent += `| ${testName} | ${status} |\n`; + }); + + // Write the Markdown content to Tests.md + console.log(markdownContent) + return fs.writeFile(markdownPath, markdownContent); + }) + .then(() => { + console.log('Tests.md updated successfully!'); + }) + .catch((err) => { + console.error('Error processing test results:', err); + }); diff --git a/tests/utils.ts b/tests/utils.ts index 3f19c06d..44cf5302 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -3,10 +3,17 @@ import path from 'path'; import dotenv from 'dotenv'; import ignore, { Ignore } from 'ignore'; -// Read and parse a .env file -export function readEnvFile(filePath: string = '.env'): Record { - const envPath = path.resolve(process.cwd(), filePath); - return dotenv.parse(readFileSync(envPath, 'utf-8')); +export function getApiKeys(): Record { + return { + E2B_API_KEY: process.env.E2B_API_KEY || '', + OPENAI_API_KEY: process.env.OPENAI_API_KEY || '', + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || '', + MISTRAL_API_KEY: process.env.MISTRAL_API_KEY || '', + FIREWORKS_API_KEY: process.env.FIREWORKS_API_KEY || '', + FIRECRAWL_API_KEY: process.env.FIRECRAWL_API_KEY || '', + GROQ_API_KEY: process.env.GROQ_API_KEY || '', + TOGETHER_API_KEY: process.env.TOGETHER_API_KEY || '', + }; } // Read and parse a .gitignore file