diff --git a/.github/workflows/pyflask-build-and-dist-tests.yml b/.github/workflows/pyflask-build-and-dist-tests.yml
new file mode 100644
index 000000000..dbc149371
--- /dev/null
+++ b/.github/workflows/pyflask-build-and-dist-tests.yml
@@ -0,0 +1,97 @@
+name: PyFlask build and distributable tests
+on:
+  schedule:
+    - cron: "0 16 * * *"  # Daily at noon EST
+  pull_request:
+  workflow_dispatch:
+
+# Cancel previous workflows on the same pull request
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CACHE_NUMBER: 1  # increase to reset cache manually
+
+jobs:
+  testing:
+    name: PyFlask build and distributable tests on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+          - python-version: "3.9"
+            os: ubuntu-latest
+            label: tools/anaconda-env/environment-Linux.yml
+            prefix: /usr/share/miniconda3/envs/nwb-guide
+            shorthand: unix
+
+          - python-version: "3.10"
+            os: macos-latest
+            label: tools/anaconda-env/environment-Mac.yml
+            prefix: /Users/runner/miniconda3/envs/nwb-guide
+            shorthand: unix
+
+          - python-version: "3.9"
+            os: windows-latest
+            label: tools/anaconda-env/environment-Windows.yml
+            prefix: C:\Miniconda3\envs\nwb-guide
+            shorthand: win
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: git fetch --prune --unshallow --tags
+
+      # see https://github.com/conda-incubator/setup-miniconda#caching-environments
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          activate-environment: nwb-guide
+          use-mamba: true
+
+      - name: Set cache date
+        id: get-date
+        run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Cache Mamba env
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.CONDA }}/envs
+          key:
+            conda-${{ runner.os }}-${{ runner.arch }}-${{steps.get-date.outputs.today }}-${{ hashFiles(matrix.label) }}-${{ env.CACHE_NUMBER }}
+        env:
+          CACHE_NUMBER: ${{ env.CACHE_NUMBER }}
+        id: cache
+
+      - if: steps.cache.outputs.cache-hit != 'true'
+        name: Update environment
+        run: mamba env update -f ${{ matrix.label }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16"
+
+      - name: Build PyFlask distribution
+        run: npm run build:flask:${{ matrix.shorthand }}
+
+      # encountering parsing issue for second arg
+      #- if: matrix.os == 'windows-latest'
+      #  name: Run test on build executable
+      #  run: node tests/testPyinstallerExecutable.js ./build/nwb-guide/nwb-guide.exe
+
+      #- if: matrix.os != 'windows-latest'
+      #  name: Run test on build executable
+      #  run: node tests/testPyinstallerExecutable.js ./build/nwb-guide/nwb-guide
+
+      - name: Run test on distributed executable
+        run: node tests/testPyinstallerExecutable.js
diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js
index fd571429e..3ee379952 100644
--- a/tests/testPyinstallerExecutable.js
+++ b/tests/testPyinstallerExecutable.js
@@ -12,30 +12,39 @@ console.log("Found file", fs.existsSync(script));
 const proc2 = child_process.spawn(`${script}`, [port + 1]);
 handleProcess(proc2, "spawn");
 
-let now = Date.now(),
-    started;
+let now = Date.now();
+
+let outputCollection = "";
+
+const regex = /.+Error: .+/;
 
 function handleProcess(proc, id = "process") {
     if (proc != null) {
         // Listen for errors from Python process
         proc.stderr.on("data", function (data) {
-            console.error(`[${id}]: ${data}`);
+            const message = data.toString();
+            console.error(`[${id}] Error: ${data}`);
+            outputCollection += message;
+            if (regex.test(message)) throw new Error(outputCollection);
         });
 
         proc.stdout.on("data", function (data) {
-            if (!started) {
-                started = Date.now();
-                console.log(`Time to Start: ${(started - now).toFixed(2)}ms`);
-            }
-            console.error(`[${id}]: ${data}`);
+            console.log(`Time to Start: ${(Date.now() - now).toFixed(2)}ms`);
+            process.exit();
         });
 
+        const error = () => () => {
+            throw new Error("The distributable pyflask failed to run!");
+        };
+
         proc.on("error", (error) => {
-            console.error(`[${id}] error: ${error.message}`);
+            console.error(`[${id}] Error: ${error.message}`);
+            error();
         });
 
         proc.on("close", (code) => {
-            console.error(`[${id}] exit: ${code}`);
+            console.error(`[${id}] Exit: ${code}`);
+            error();
         });
     } else console.error("child process failed to start on port" + port);
 }