Add CI tests with Markdown output (#50)

.github/workflows/update-tests-md.yml

@@ -0,0 +1,70 @@
+name: Update Tests Results
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 1'  # Runs every Monday at 00:00 UTC
+
+jobs:
+  run_tests_and_update_markdown:
+    runs-on: ubuntu-latest
+
+    env:
+      E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
+      FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
+      FIRECRAWL_API_KEY: ${{ secrets.FIRECRAWL_API_KEY }}
+      GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v2
+      with:
+        node-version: '20'
+
+    - name: Install dependencies
+      run: npm install
+
+    - name: Run Jest tests and save results
+      run: |
+        npm test -- --json --outputFile=./tests/results.json
+      continue-on-error: true  # Ensures the job continues even if tests fail
+
+    - name: Upload test results as an artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-results
+        path: ./tests/results.json
+
+    - name: Update Tests.md with test results
+      run: node ./tests/updateTestsMd.js  # Your custom script to update Tests.md with test results
+
+    - name: Upload markdown report as an artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-results-md
+        path: ./tests/Tests.txt
+
+    - name: Read Tests.txt for Slack message
+      id: read_tests
+      run: |
+        TESTS_CONTENT=$(cat ./tests/Tests.txt)
+        echo "TESTS_CONTENT<<EOF" >> $GITHUB_ENV
+        echo "$TESTS_CONTENT" >> $GITHUB_ENV
+        echo "EOF" >> $GITHUB_ENV
+
+    - name: Notify Slack with test results
+      uses: rtCamp/action-slack-notify@v2
+      env:
+        SLACK_COLOR: "#36a64f"
+        SLACK_MESSAGE: |
+          *Test Results:*
+          ${{ env.TESTS_CONTENT }}
+        SLACK_TITLE: Test Results Update
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

package.json

@@ -3,7 +3,7 @@
     "type": "module",
     "version": "1.0.0",
     "scripts": {
-        "test": "jest --maxConcurrency=20"
+        "test": "jest --maxConcurrency=5"
     },
     "jest": {
         "preset": "ts-jest",

tests/examples.test.ts

@@ -3,7 +3,7 @@ import path from 'path';
 import dotenv from 'dotenv';
 import Sandbox from 'e2b';
 
-import { uploadPathToPath, readEnvFile } from "./utils"
+import { uploadPathToPath, getApiKeys } from "./utils"
 
 // Read the E2B API key
 dotenv.config({ path: path.resolve(process.cwd(), ".env") });
@@ -95,7 +95,7 @@ describe('Integration test for multiple scripts in e2b sandbox', () => {
   });
 
   scripts.forEach(({ name, interpreter, file : examplePath }) => {
-    it.concurrent(`should upload and execute ${name} successfully in e2b sandbox`, async () => {
+    it.concurrent(name, async () => {
 
       let attempts = 0;
       const maxAttempts = 3;
@@ -104,9 +104,10 @@ describe('Integration test for multiple scripts in e2b sandbox', () => {
       while (attempts < maxAttempts && !success) {
         attempts++;
 
+        // Create a new E2B sandbox
+        const sandbox = await Sandbox.create({ timeoutMs: SANDBOX_TIMEOUT });
+
         try {
-          // Create a new E2B sandbox
-          const sandbox = await Sandbox.create({ timeoutMs: SANDBOX_TIMEOUT });
 
           // Upload the example directory to the sandbox.
           await uploadPathToPath(examplePath, SANDBOX_TEST_DIRECTORY, sandbox);
@@ -132,13 +133,10 @@ describe('Integration test for multiple scripts in e2b sandbox', () => {
               stdoutData += output;
               await fs.appendFile(logFilePath, output);
             },
-            envs: readEnvFile(),
+            envs: getApiKeys(),
             timeoutMs: COMMAND_TIMEOUT,
           });
 
-          // Kill the sandbox
-          await sandbox.kill();
-
           // Check the exit code to see if the test passed
           if (result.exitCode !== 0) {
             await fs.appendFile(logFilePath, `Attempt ${attempts}: Test for ${name} failed with exit code ${result.exitCode}\n`);
@@ -156,7 +154,10 @@ describe('Integration test for multiple scripts in e2b sandbox', () => {
             await fs.appendFile(logFilePath, `Test for ${name} completed successfully on attempt ${attempts}.\n`);
           }
         } catch (error) {
-          console.log(`Attempt ${attempts}/${maxAttempts}: An error occurred while running the test for ${name}`);
+          console.log(`Attempt ${attempts}/${maxAttempts}: An error occurred while running the test for ${name}`, error);
+        } finally {
+          // Kill the sandbox
+          await sandbox.kill();
         }
 
         if (!success && attempts === maxAttempts) {

tests/updateTestsMd.js

@@ -0,0 +1,37 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+// Get the current directory name
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Define paths for the results and markdown files
+const resultsPath = path.join(__dirname, 'results.json');
+const markdownPath = path.join(__dirname, 'Tests.txt');
+
+// Read the Jest results
+fs.readFile(resultsPath, 'utf8')
+    .then((data) => {
+        var markdownContent = ""
+        const results = JSON.parse(data);
+
+        // Iterate through each test result and populate the table
+        results.testResults.forEach((test) => {
+            test.assertionResults.forEach((assertion) => {
+                const testName = assertion.title;
+                const status = assertion.status === 'passed' ? '✅ Passed' : '❌ Failed';
+                markdownContent += `| ${testName} | ${status} |\n`;
+            });
+        });        
+
+        // Write the Markdown content to Tests.md
+        console.log(markdownContent)
+        return fs.writeFile(markdownPath, markdownContent);
+    })
+    .then(() => {
+        console.log('Tests.md updated successfully!');
+    })
+    .catch((err) => {
+        console.error('Error processing test results:', err);
+    });

tests/utils.ts

@@ -3,10 +3,17 @@ import path from 'path';
 import dotenv from 'dotenv';
 import ignore, { Ignore } from 'ignore';
 
-// Read and parse a .env file
-export function readEnvFile(filePath: string = '.env'): Record<string, string> {
-  const envPath = path.resolve(process.cwd(), filePath);
-  return dotenv.parse(readFileSync(envPath, 'utf-8'));
+export function getApiKeys(): Record<string, string> {
+  return {
+    E2B_API_KEY: process.env.E2B_API_KEY || '',
+    OPENAI_API_KEY: process.env.OPENAI_API_KEY || '',
+    ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || '',
+    MISTRAL_API_KEY: process.env.MISTRAL_API_KEY || '',
+    FIREWORKS_API_KEY: process.env.FIREWORKS_API_KEY || '',
+    FIRECRAWL_API_KEY: process.env.FIRECRAWL_API_KEY || '',
+    GROQ_API_KEY: process.env.GROQ_API_KEY || '',
+    TOGETHER_API_KEY: process.env.TOGETHER_API_KEY || '',
+  };
 }
 
 // Read and parse a .gitignore file