chore: migrate hero out of platform

ulixee · Sep 18, 2024 · 64d87a9 · 64d87a9
1 parent 9fc427e
commit 64d87a9
Show file tree

Hide file tree

Showing 15 changed files with 498 additions and 343 deletions.
diff --git a/.github/workflows/js-branch.yml b/.github/workflows/js-branch.yml
@@ -25,28 +25,8 @@ jobs:
           node-version: 18
           cache: yarn
 
-      - name: Setup SSH for Git Operations
-        run: |
-          if [[ "$RUNNER_OS" == "Windows" ]]; then
-            SSH_PATH="$HOME/.ssh"
-            mkdir "$SSH_PATH"
-            echo "$SSH_PRIVATE_KEY" > "$SSH_PATH/id_rsa"
-            chmod 600 "$SSH_PATH/id_rsa"
-            ssh-keyscan github.com >> "$SSH_PATH/known_hosts"
-          else
-            SSH_PATH=~/.ssh
-            mkdir -p "$SSH_PATH"
-            echo "$SSH_PRIVATE_KEY" > "$SSH_PATH/id_rsa"
-            chmod 600 "$SSH_PATH/id_rsa"
-            ssh-keyscan github.com >> "$SSH_PATH/known_hosts"
-          fi
-        env:
-          SSH_PRIVATE_KEY: ${{ secrets.MAINCHAIN_SSH_KEY }}
-
       - name: Clone Mainchain
-        run: |
-          export GIT_SSH_COMMAND="ssh -i $SSH_PATH/id_rsa -o UserKnownHostsFile=$SSH_PATH/known_hosts"
-          git clone -b napi-bindings [email protected]:argonprotocol/mainchain.git
+        run: git clone -b napi-bindings https://github.com/argonprotocol/mainchain.git
         working-directory: ..
 
       - name: Install Mainchain
@@ -57,15 +37,22 @@ jobs:
         run: yarn tsc
         working-directory: ../mainchain/client/nodejs
 
+      - name: Clone Hero
+        run: |
+          git clone -n https://github.com/ulixee/hero.git
+          cd hero
+          (git fetch --depth 1 origin $GITHUB_REF_NAME && git checkout $GITHUB_REF_NAME) || (git fetch --depth 1 origin main && git checkout main)
+        working-directory: ..
+
       - name: Install Hero
         run: yarn build:simple
-        working-directory: ./hero
+        working-directory: ../hero
 
       - name: Set timeout
         run: yarn config set network-timeout 1000000 -g
 
       - name: Build modules
-        run: yarn && yarn build:dist
+        run: yarn install --frozen-lockfile --ignore-engines && yarn build:dist
 
       - name: Publish branch
         run: |

diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml
@@ -57,25 +57,14 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.23'
 
       - name: Install pcap
         run: sudo apt-get install -y libpcap-dev
 
-      - name: Setup SSH for Git Operations
-        shell: bash
-        run: |
-          mkdir -p "$SSH_PATH"
-          echo "$SSH_PRIVATE_KEY" > "$SSH_PATH/id_rsa"
-          chmod 600 "$SSH_PATH/id_rsa"
-          ssh-keyscan github.com >> "$SSH_PATH/known_hosts"
-        env:
-          SSH_PRIVATE_KEY: ${{ secrets.MAINCHAIN_SSH_KEY }}
-
       - name: Clone Mainchain
         run: |
-          export GIT_SSH_COMMAND="ssh -i $SSH_PATH/id_rsa -o UserKnownHostsFile=$SSH_PATH/known_hosts"
-          git clone --depth 1 -b napi-bindings [email protected]:argonprotocol/mainchain.git
+          git clone --depth 1 -b napi-bindings https://github.com/argonprotocol/mainchain.git
           cp mainchain/localchain/npm/**/*.node mainchain/localchain
         working-directory: ..
 
@@ -87,12 +76,23 @@ jobs:
         run: yarn tsc
         working-directory: ../mainchain/client/nodejs
 
+      - name: Clone Hero
+        run: |
+          git clone -n --recurse-submodules https://github.com/ulixee/hero.git
+          cd hero
+          (git fetch --depth 1 origin $GITHUB_REF_NAME && git checkout $GITHUB_REF_NAME) || (git fetch --depth 1 origin main && git checkout main)
+          git submodule update --init --recursive --depth 1
+        working-directory: ..
+
       - name: Install Hero
-        run: yarn build:simple
-        working-directory: ./hero
+        run: yarn build
+        working-directory: ../hero
+
+      - name: Set yarn config
+        run: yarn config set ignore-engines true && yarn config set frozen-lockfile true
 
       - name: Yarn build
-        run: yarn --ignore-engines && yarn build --ignore-engines
+        run: yarn build
         env:
           ELECTRON_SKIP_BINARY_DOWNLOAD: 1
 
@@ -150,7 +150,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.23'
 
       - name: Chocolatey Install Action
         if: ${{ matrix.os == 'windows-latest' }}
@@ -162,31 +162,10 @@ jobs:
         if: ${{ matrix.os == 'windows-latest' }}
         uses: microsoft/setup-msbuild@v2
 
-      - name: Setup SSH for Git Operations
-        shell: bash
-        run: |
-          if [[ "$RUNNER_OS" == "Windows" ]]; then
-            SSH_PATH="$HOME/.ssh"
-            mkdir "$SSH_PATH"
-            echo "$SSH_PRIVATE_KEY" > "$SSH_PATH/id_rsa"
-            chmod 600 "$SSH_PATH/id_rsa"
-            ssh-keyscan github.com >> "$SSH_PATH/known_hosts"
-            echo "GIT_SSH_COMMAND=ssh -i $SSH_PATH/id_rsa -o UserKnownHostsFile=$SSH_PATH/known_hosts" >> $GITHUB_ENV
-          else
-            SSH_PATH=~/.ssh
-            mkdir -p "$SSH_PATH"
-            echo "$SSH_PRIVATE_KEY" > "$SSH_PATH/id_rsa"
-            chmod 600 "$SSH_PATH/id_rsa"
-            ssh-keyscan github.com >> "$SSH_PATH/known_hosts"
-            echo "GIT_SSH_COMMAND=ssh -i $SSH_PATH/id_rsa -o UserKnownHostsFile=$SSH_PATH/known_hosts" >> $GITHUB_ENV
-          fi
-        env:
-          SSH_PRIVATE_KEY: ${{ secrets.MAINCHAIN_SSH_KEY }}
-
       - name: Clone Mainchain
         shell: bash
         run: |
-          git clone -b napi-bindings git@github.com:argonprotocol/mainchain.git
+          git clone -b napi-bindings https://github.com/argonprotocol/mainchain.git
           cp mainchain/localchain/npm/**/*.node mainchain/localchain
         working-directory: ..
 
@@ -198,12 +177,23 @@ jobs:
         run: yarn tsc
         working-directory: ../mainchain/client/nodejs
 
+      - name: Clone Hero
+        run: |
+          git clone -n --recurse-submodules https://github.com/ulixee/hero.git
+          cd hero
+          (git fetch --depth 1 origin $GITHUB_REF_NAME && git checkout $GITHUB_REF_NAME) || (git fetch --depth 1 origin main && git checkout main)
+          git submodule update --init --recursive --depth 1
+        working-directory: ..
+
       - name: Install Hero
-        run: yarn build:simple
-        working-directory: ./hero
+        run: yarn build
+        working-directory: ../hero
+
+      - name: Set yarn config
+        run: yarn config set ignore-engines true && yarn config set frozen-lockfile true
 
       - name: Yarn build
-        run: yarn --ignore-engines && yarn build --ignore-engines
+        run: yarn build
         env:
           ELECTRON_SKIP_BINARY_DOWNLOAD: 1
 

diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml
@@ -42,10 +42,6 @@ jobs:
         uses: microsoft/setup-msbuild@v2
         if: ${{ matrix.os == 'windows-latest' }}
 
-      - name: Install Hero
-        run: yarn && yarn build
-        working-directory: ./hero
-
       - name: Yarn build
         run: yarn --ignore-engines && yarn build --ignore-engines
         env:

diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +0,0 @@
-[submodule "hero"]
-	path = hero
-	url = [email protected]:ulixee/hero.git
-	branch = main

diff --git a/datastore/testing/package.json b/datastore/testing/package.json
@@ -5,8 +5,8 @@
   "main": "index.js",
   "private": true,
   "dependencies": {
-    "@koa/multer": "^3.0.0",
-    "@koa/router": "^8.0.8",
+    "@koa/multer": "^3.0.2",
+    "@koa/router": "^13.0.0",
     "@ulixee/cloud": "2.0.0-alpha.29",
     "@ulixee/commons": "2.0.0-alpha.29",
     "@ulixee/datastore": "2.0.0-alpha.29",
@@ -19,14 +19,14 @@
     "@ulixee/net": "2.0.0-alpha.29",
     "@ulixee/unblocked-agent-mitm-socket": "2.0.0-alpha.29",
     "@ulixee/unblocked-specification": "2.0.0-alpha.29",
-    "http-proxy-agent": "^4.0.1",
-    "https-proxy-agent": "^5.0.0",
-    "koa": "^2.11.0",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.5",
+    "koa": "^2.15.3",
     "multer": "^1.4.5-lts.1"
   },
   "devDependencies": {
-    "@types/koa": "^2.11.3",
-    "@types/koa__multer": "^2.0.2",
-    "@types/koa__router": "^8.0.2"
+    "@types/koa": "^2.15.0",
+    "@types/koa__multer": "^2.0.7",
+    "@types/koa__router": "^12.0.3"
   }
 }
diff --git a/hero b/hero
diff --git a/package.json b/package.json
@@ -5,19 +5,15 @@
   "scripts": {
     "prepare": "husky",
     "build": "yarn && yarn tsc && yarn build:docpage && cd ./build && yarn",
-    "build:hero": "cd hero && yarn build",
     "build:desktop": "yarn workspace @ulixee/desktop build:ui",
     "build:docpage": "yarn workspace @ulixee/datastore-docpage build",
     "build:broker": "yarn workspace @ulixee/databroker-admin build",
-    "build:all": "yarn build:hero && yarn build:desktop && yarn build:broker yarn build",
+    "build:all": "yarn build:desktop && yarn build:broker yarn build",
     "build:dist": "shx rm -rf build-dist && tsc -b tsconfig.dist.json && yarn copy:dist && yarn workspace @ulixee/datastore-docpage build:dist && yarn workspace @ulixee/databroker-admin build:dist",
     "copy:build": "ulx-repo-after-build && ulx-repo-adjust-paths",
     "copy:dist": "ulx-repo-after-build --dist",
     "tsc": "tsc -b -i tsconfig.json && yarn copy:build",
-    "tsc:hero": "cd hero && yarn tsc",
-    "tsc:all": "yarn tsc:hero && yarn tsc",
     "watch": "yarn tsc && tsc -b -w tsconfig.json",
-    "watch:all": "concurrently -n hero,devenv \"cd hero && yarn watch\" \"yarn watch\"",
     "start": "yarn workspace @ulixee/cloud start",
     "test": "yarn copy:build && yarn test:build",
     "test:build": "cd ./build && cross-env ULX_DATA_DIR=.data-test ULX_DATASTORE_DIR=.data-test ULX_QUERY_HERO_SESSIONS_DIR=.data-test/hero-sessions NODE_ENV=test jest",
@@ -87,19 +83,19 @@
       "schema",
       "specification",
       "end-to-end",
-      "hero/build/agent/*",
-      "hero/build/client",
-      "hero/build/commons",
-      "hero/build/core",
-      "hero/build/interfaces",
-      "hero/build/js-path",
-      "hero/build/net",
-      "hero/build/plugin-utils",
-      "hero/build/plugins/*",
-      "hero/build/real-user-agents",
-      "hero/build/specification",
-      "hero/build/testing",
-      "hero/build/timetravel"
+      "../hero/build/agent/*",
+      "../hero/build/client",
+      "../hero/build/commons",
+      "../hero/build/core",
+      "../hero/build/interfaces",
+      "../hero/build/js-path",
+      "../hero/build/net",
+      "../hero/build/plugin-utils",
+      "../hero/build/plugins/*",
+      "../hero/build/real-user-agents",
+      "../hero/build/specification",
+      "../hero/build/testing",
+      "../hero/build/timetravel"
     ],
     "nohoist": [
       "**/babel-loader",
@@ -110,10 +106,6 @@
       "**electron**",
       "**/@argonprotocol/mainchain",
       "**/desktop/**/@argonprotocol/localchain-*",
-      "**/desktop/main/app/**",
-      "**/desktop/**/@ulixee/datastore-docpage",
-      "**/desktop/**/@ulixee/databroker-admin",
-      "**/desktop/**/better-sqlite3",
       "**/electron-*",
       "**/electron-log",
       "**/electron-builder",

diff --git a/release.sh b/release.sh
@@ -3,7 +3,7 @@
 CURRENT_DIR=$( cd "$(dirname "${BASH_SOURCE[0]}")/.." ; pwd -P );
 trap "exit" INT;
 
-declare -a dirs=("shared" "unblocked" "platform/hero" "platform");
+declare -a dirs=("platform/hero" "platform");
 
 # loop through dirs
 for dir in "${dirs[@]}"; do
@@ -44,8 +44,8 @@ for dir in "${dirs[@]}"; do
 
   wait 5;
 
-  if [[ "$dir" == "unblocked" ]]; then
-    echo "You need to approve the DRAFT release at https://github.com/ulixee/unblocked/releases/tag/$TAG";
+  if [[ "$dir" == "hero" ]]; then
+    echo "You need to approve the DRAFT release at https://github.com/ulixee/hero/releases/tag/$TAG";
     read -p "Press enter only once you've done that (npm needs the assets built)...";
   fi
 

diff --git a/website/data-generator/generateDoc.mjs b/website/data-generator/generateDoc.mjs
@@ -7,7 +7,10 @@ const toolKey = matches[1];
 const absoluteFilePath = Path.resolve(process.cwd(), process.argv[2]);
 
 const __dirname = Path.dirname(Url.fileURLToPath(import.meta.url));
-const rootDir = Path.resolve(__dirname, '../../');
+let rootDir = Path.resolve(__dirname, '../../');
+if (toolKey === 'hero') {
+  rootDir = Path.resolve(__dirname, '../../../');
+}
 const mdDocsRootPath = Path.resolve(rootDir, toolKey, 'docs');
 
 await saveToWebsite(mdDocsRootPath, absoluteFilePath, toolKey);
diff --git a/website/data-generator/generateDocs.mjs b/website/data-generator/generateDocs.mjs
@@ -17,7 +17,7 @@ const rootDir = Path.resolve(__dirname, '../../');
 }
 
 {
-  const mdDocsRootPath = `${rootDir}/hero/docs`;
+  const mdDocsRootPath = `${rootDir}/../hero/docs`;
   walkDirectory(mdDocsRootPath, async filePath => {
     await saveToWebsite(mdDocsRootPath, filePath, 'hero');
   });

diff --git a/website/data-generator/generateRoadmaps.mjs b/website/data-generator/generateRoadmaps.mjs
@@ -16,13 +16,10 @@ const milestones = JSON.parse(rawMilestones);
 const roadmapPaths = [
   'cloud/ROADMAP-Cloud.md',
   'stream/ROADMAP-Stream.md',
-  'hero/ROADMAP-Hero.md',
+  '../hero/ROADMAP-Hero.md',
   'datastore/ROADMAP-Datastore.md',
-  '../payments/ROADMAP-Argon.md',
-  '../payments/mainchain/ROADMAP-Mainchain.md',
-  '../payments/sidechain/ROADMAP-Sidechain.md',
-  '../unblocked/ROADMAP-DoubleAgent.md',
-  '../unblocked/ROADMAP-ScraperReport.md',
+  '../hero/ROADMAP-DoubleAgent.md',
+  '../hero/ROADMAP-ScraperReport.md',
   'coming-soon/ROADMAP-NFTs.md',
   'desktop/ROADMAP-Desktop.md',
 ];

diff --git a/website/data-generator/generateToc.mjs b/website/data-generator/generateToc.mjs
@@ -17,7 +17,7 @@ Fs.mkdirSync(tocDataDir, { recursive: true });
 }
 
 {
-  const yamlLinks = Fs.readFileSync(`${rootDir}/hero/docs/links.yaml`, 'utf-8');
+  const yamlLinks = Fs.readFileSync(`${rootDir}/../hero/docs/links.yaml`, 'utf-8');
   const links = JsYaml.load(yamlLinks);
   const saveToFilePath = Path.join(tocDataDir, `hero.json`);
   Fs.writeFileSync(saveToFilePath, JSON.stringify(links, null, 2));

diff --git a/website/package.json b/website/package.json
@@ -8,9 +8,10 @@
     "deploy": "firebase deploy --only hosting",
     "build": "yarn generate && vue-cli-service build --mode production",
     "lint": "vue-cli-service lint",
+    "watch:server": "yarn start",
     "watch:stream": "npx onchange '../stream/docs/**/*.md' -- node ./data-generator/generateDoc.mjs {{file}}",
     "watch:datastore": "npx onchange '../datastore/docs/**/*.md' -- node ./data-generator/generateDoc.mjs {{file}}",
-    "watch:hero": "npx onchange '../hero/docs/**/*.md' -- node ./data-generator/generateDoc.mjs {{file}}",
+    "watch:hero": "npx onchange '../../hero/docs/**/*.md' -- node ./data-generator/generateDoc.mjs {{file}}",
     "watch:cloud": "npx onchange '../cloud/docs/**/*.md' -- node ./data-generator/generateDoc.mjs {{file}}",
     "watch:sql": "npx onchange '../sql/docs/**/*.md' -- node ./data-generator/generateDoc.mjs {{file}}",
     "watch:client": "npx onchange '../client/docs/**/*.md' -- node ./data-generator/generateDoc.mjs {{file}}",
@@ -20,7 +21,7 @@
     "@headlessui/vue": "^1.5.0",
     "@heroicons/vue": "^2.0.16",
     "@tailwindcss/forms": "^0.5.0",
-    "axios": "^0.26.1",
+    "axios": "^0.28.1",
     "babel-plugin-prismjs": "^2.1.0",
     "concurrently": "^6.2.2",
     "core-js": "^3.21.1",