mendableai · mogery · Nov 7, 2024 · Sep 28, 2024 · Sep 28, 2024 · Sep 28, 2024
diff --git a/.github/archive/js-sdk.yml b/.github/archive/js-sdk.yml
@@ -8,7 +8,6 @@ env:
   FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
   HOST: ${{ secrets.HOST }}
   LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
-  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
   POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
   POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
   NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
@@ -21,7 +20,6 @@ env:
   SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
   SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
   TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
-  HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
   HDX_NODE_BETA_MODE: 1
 
 jobs:  

diff --git a/.github/archive/python-sdk.yml b/.github/archive/python-sdk.yml
@@ -8,7 +8,6 @@ env:
   FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
   HOST: ${{ secrets.HOST }}
   LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
-  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
   POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
   POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
   NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
@@ -21,7 +20,6 @@ env:
   SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
   SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
   TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
-  HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
   HDX_NODE_BETA_MODE: 1
 
 jobs:  

diff --git a/.github/archive/rust-sdk.yml b/.github/archive/rust-sdk.yml
@@ -8,7 +8,6 @@ env:
   FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
   HOST: ${{ secrets.HOST }}
   LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
-  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
   POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
   POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
   NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
@@ -21,7 +20,6 @@ env:
   SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
   SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
   TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
-  HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
   HDX_NODE_BETA_MODE: 1
 
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,7 +12,6 @@ env:
   FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
   HOST: ${{ secrets.HOST }}
   LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
-  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
   POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
   POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
   NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
@@ -25,7 +24,6 @@ env:
   SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
   SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
   TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
-  HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
   HDX_NODE_BETA_MODE: 1
   FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}
   USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }}

diff --git a/.github/workflows/deploy-image-staging.yml b/.github/workflows/deploy-image-staging.yml
@@ -0,0 +1,32 @@
+name: STAGING Deploy Images to GHCR
+
+env:
+  DOTNET_VERSION: '6.0.x'
+
+on:
+  push:
+    branches:
+      - mog/webscraper-refactor
+  workflow_dispatch:
+
+jobs:
+  push-app-image:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: './apps/api'
+    steps:
+      - name: 'Checkout GitHub Action'
+        uses: actions/checkout@main
+
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: 'Build Inventory Image'
+        run: |
+          docker build . --tag ghcr.io/mendableai/firecrawl-staging:latest
+          docker push ghcr.io/mendableai/firecrawl-staging:latest
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -41,7 +41,6 @@ TEST_API_KEY= # use if you've set up authentication and want to test with a real
 SCRAPING_BEE_API_KEY= #Set if you'd like to use scraping Be to handle JS blocking
 OPENAI_API_KEY= # add for LLM dependednt features (image alt generation, etc.)
 BULL_AUTH_KEY= @
-LOGTAIL_KEY= # Use if you're configuring basic logging with logtail
 PLAYWRIGHT_MICROSERVICE_URL=  # set if you'd like to run a playwright fallback
 LLAMAPARSE_API_KEY= #Set if you have a llamaparse key you'd like to use to parse pdfs
 SLACK_WEBHOOK_URL= # set if you'd like to send slack server health status messages

diff --git a/README.md b/README.md
@@ -56,7 +56,7 @@ We provide an easy to use API with our hosted version. You can find the playgrou
 Check out the following resources to get started:
 - [x] **API**: [Documentation](https://docs.firecrawl.dev/api-reference/introduction)
 - [x] **SDKs**: [Python](https://docs.firecrawl.dev/sdks/python), [Node](https://docs.firecrawl.dev/sdks/node), [Go](https://docs.firecrawl.dev/sdks/go), [Rust](https://docs.firecrawl.dev/sdks/rust)
-- [x] **LLM Frameworks**: [Langchain (python)](https://python.langchain.com/docs/integrations/document_loaders/firecrawl/), [Langchain (js)](https://js.langchain.com/docs/integrations/document_loaders/web_loaders/firecrawl), [Llama Index](https://docs.llamaindex.ai/en/latest/examples/data_connectors/WebPageDemo/#using-firecrawl-reader), [Crew.ai](https://docs.crewai.com/), [Composio](https://composio.dev/tools/firecrawl/all), [PraisonAI](https://docs.praison.ai/firecrawl/)
+- [x] **LLM Frameworks**: [Langchain (python)](https://python.langchain.com/docs/integrations/document_loaders/firecrawl/), [Langchain (js)](https://js.langchain.com/docs/integrations/document_loaders/web_loaders/firecrawl), [Llama Index](https://docs.llamaindex.ai/en/latest/examples/data_connectors/WebPageDemo/#using-firecrawl-reader), [Crew.ai](https://docs.crewai.com/), [Composio](https://composio.dev/tools/firecrawl/all), [PraisonAI](https://docs.praison.ai/firecrawl/), [Superinterface](https://superinterface.ai/docs/assistants/functions/firecrawl), [Vectorize](https://docs.vectorize.io/integrations/source-connectors/firecrawl)
 - [x] **Low-code Frameworks**: [Dify](https://dify.ai/blog/dify-ai-blog-integrated-with-firecrawl), [Langflow](https://docs.langflow.org/), [Flowise AI](https://docs.flowiseai.com/integrations/langchain/document-loaders/firecrawl), [Cargo](https://docs.getcargo.io/integration/firecrawl), [Pipedream](https://pipedream.com/apps/firecrawl/)
 - [x] **Others**: [Zapier](https://zapier.com/apps/firecrawl/integrations), [Pabbly Connect](https://www.pabbly.com/connect/integrations/firecrawl/)
 - [ ] Want an SDK or Integration? Let us know by opening an issue.

diff --git a/SELF_HOST.md b/SELF_HOST.md
@@ -62,7 +62,6 @@ TEST_API_KEY= # use if you've set up authentication and want to test with a real
 SCRAPING_BEE_API_KEY= # use if you'd like to use as a fallback scraper
 OPENAI_API_KEY= # add for LLM-dependent features (e.g., image alt generation)
 BULL_AUTH_KEY= @
-LOGTAIL_KEY= # Use if you're configuring basic logging with logtail
 PLAYWRIGHT_MICROSERVICE_URL=  # set if you'd like to run a playwright fallback
 LLAMAPARSE_API_KEY= #Set if you have a llamaparse key you'd like to use to parse pdfs
 SLACK_WEBHOOK_URL= # set if you'd like to send slack server health status messages

diff --git a/apps/api/.env.example b/apps/api/.env.example
@@ -33,8 +33,6 @@ SCRAPING_BEE_API_KEY=
 # add for LLM dependednt features (image alt generation, etc.)
 OPENAI_API_KEY=
 BULL_AUTH_KEY=@
-# use if you're configuring basic logging with logtail
-LOGTAIL_KEY=
 # set if you have a llamaparse key you'd like to use to parse pdfs
 LLAMAPARSE_API_KEY=
 # set if you'd like to send slack server health status messages
@@ -54,9 +52,6 @@ STRIPE_PRICE_ID_STANDARD_NEW_YEARLY=
 STRIPE_PRICE_ID_GROWTH=
 STRIPE_PRICE_ID_GROWTH_YEARLY=
 
-HYPERDX_API_KEY=
-HDX_NODE_BETA_MODE=1
-
 # set if you'd like to use the fire engine closed beta
 FIRE_ENGINE_BETA_URL=
 

diff --git a/apps/api/jest.setup.js b/apps/api/jest.setup.js
@@ -1 +1 @@
-global.fetch = require('jest-fetch-mock');
+// global.fetch = require('jest-fetch-mock');
diff --git a/apps/api/package.json b/apps/api/package.json
@@ -32,9 +32,11 @@
     "@tsconfig/recommended": "^1.0.3",
     "@types/body-parser": "^1.19.2",
     "@types/cors": "^2.8.13",
+    "@types/escape-html": "^1.0.4",
     "@types/express": "^4.17.17",
     "@types/jest": "^29.5.12",
     "@types/node": "^20.14.1",
+    "@types/pdf-parse": "^1.1.4",
     "body-parser": "^1.20.1",
     "express": "^4.18.2",
     "jest": "^29.6.3",
@@ -53,9 +55,7 @@
     "@bull-board/api": "^5.20.5",
     "@bull-board/express": "^5.20.5",
     "@devil7softwares/pos": "^1.0.2",
-    "@dqbd/tiktoken": "^1.0.13",
-    "@hyperdx/node-opentelemetry": "^0.8.1",
-    "@logtail/node": "^0.4.12",
+    "@dqbd/tiktoken": "^1.0.16",
     "@nangohq/node": "^0.40.8",
     "@sentry/cli": "^2.33.1",
     "@sentry/node": "^8.26.0",
@@ -78,6 +78,7 @@
     "date-fns": "^3.6.0",
     "dotenv": "^16.3.1",
     "dotenv-cli": "^7.4.2",
+    "escape-html": "^1.0.3",
     "express-rate-limit": "^7.3.1",
     "express-ws": "^5.0.2",
     "form-data": "^4.0.0",
@@ -92,6 +93,7 @@
     "languagedetect": "^2.0.0",
     "logsnag": "^1.0.0",
     "luxon": "^3.4.3",
+    "marked": "^14.1.2",
     "md5": "^2.3.0",
     "moment": "^2.29.4",
     "mongoose": "^8.4.4",
@@ -114,6 +116,8 @@
     "typesense": "^1.5.4",
     "unstructured-client": "^0.11.3",
     "uuid": "^10.0.0",
+    "winston": "^3.14.2",
+    "winston-transport": "^4.8.0",
     "wordpos": "^2.1.0",
     "ws": "^8.18.0",
     "xml2js": "^0.6.2",