From 10c66b06671dbb5ea4900c613fcb8c0d7075d2eb Mon Sep 17 00:00:00 2001 From: Mayo Date: Mon, 27 Mar 2023 22:44:28 +0100 Subject: [PATCH] upgrade langchain, add customPDFLoader --- README.md | 22 +- components/layout.tsx | 2 +- config/pinecone.ts | 8 +- declarations/pdf-parse.d.ts | 5 + package.json | 5 +- pages/api/chat.ts | 8 +- pages/index.tsx | 9 +- pnpm-lock.yaml | 1027 ++--------------------------------- scripts/ingest-data.ts | 16 +- utils/customPDFLoader.ts | 61 +++ utils/pinecone-client.ts | 1 - 11 files changed, 162 insertions(+), 1002 deletions(-) create mode 100644 declarations/pdf-parse.d.ts create mode 100644 utils/customPDFLoader.ts diff --git a/README.md b/README.md index e4d796977..5f961024d 100644 --- a/README.md +++ b/README.md @@ -37,14 +37,16 @@ OPENAI_API_KEY= PINECONE_API_KEY= PINECONE_ENVIRONMENT= +PINECONE_INDEX_NAME= + ``` - Visit [openai](https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key) to retrieve API keys and insert into your `.env` file. -- Visit [pinecone](https://pinecone.io/) to create and retrieve your API keys. +- Visit [pinecone](https://pinecone.io/) to create and retrieve your API keys, and also retrieve your environment and index name from the dashboard. -4. In the `config` folder, replace the `PINECONE_INDEX_NAME` and `PINECONE_NAME_SPACE` with your own details from your pinecone dashboard. +4. In the `config` folder, replace the `PINECONE_NAME_SPACE` with a `namespace` where you'd like to store your embeddings on Pinecone when you run `pnpm run ingest`. This namespace will later be used for queries and retrieval. -5. In `utils/makechain.ts` chain change the `QA_PROMPT` for your own usecase. Change `modelName` in `new OpenAIChat` to a different api model if you don't have access to `gpt-4`. See [the OpenAI docs](https://platform.openai.com/docs/models/model-endpoint-compatibility) for a list of supported `modelName`s. For example you could use `gpt-3.5-turbo` if you do not have access to `gpt-4`, yet. +5. In `utils/makechain.ts` chain change the `QA_PROMPT` for your own usecase. Change `modelName` in `new OpenAIChat` to `gpt-3.5-turbo`, if you don't have access to `gpt-4`. Please verify outside this repo that you have access to `gpt-4`, otherwise the application will not work with it. ## Convert your PDF to embeddings @@ -52,13 +54,13 @@ PINECONE_ENVIRONMENT= 2. In `scripts/ingest-data.ts` replace `filePath` with `docs/{yourdocname}.pdf` -3. Run the script `npm run ingest` to 'ingest' and embed your docs +3. Run the script `npm run ingest` to 'ingest' and embed your docs. If you run into errors troubleshoot below. 4. Check Pinecone dashboard to verify your namespace and vectors have been added. ## Run the app -Once you've verified that the embeddings and content have been successfully added to your Pinecone, you can run the app `npm run dev` to launch the local dev environment and then type a question in the chat interface. +Once you've verified that the embeddings and content have been successfully added to your Pinecone, you can run the app `pnpm run dev` to launch the local dev environment, and then type a question in the chat interface. ## Troubleshooting @@ -68,15 +70,17 @@ In general, keep an eye out in the `issues` and `discussions` section of this re - Make sure you're running the latest Node version. Run `node -v` - Make sure you're using the same versions of LangChain and Pinecone as this repo. -- Check that you've created an `.env` file that contains your valid (and working) API keys. +- Check that you've created an `.env` file that contains your valid (and working) API keys, environment and index name. - If you change `modelName` in `OpenAIChat` note that the correct name of the alternative model is `gpt-3.5-turbo` -- Pinecone indexes of users on the Starter(free) plan are deleted after 7 days of inactivity. To prevent this, send an API request to Pinecone to reset the counter. +- Make sure you have access to `gpt-4` if you decide to use. Test your openAI keys outside the repo and make sure it works and that you have enough API credits. **Pinecone errors** -- Make sure your pinecone dashboard `environment` and `index` matches the one in your `config` folder. +- Make sure your pinecone dashboard `environment` and `index` matches the one in the `pinecone.ts` and `.env` files. - Check that you've set the vector dimensions to `1536`. -- Switch your Environment in pinecone to `us-east1-gcp` if the other environment is causing issues. +- Make sure your pinecone namespace is in lowercase. +- Pinecone indexes of users on the Starter(free) plan are deleted after 7 days of inactivity. To prevent this, send an API request to Pinecone to reset the counter. +- Retry with a new Pinecone index. If you're stuck after trying all these steps, delete `node_modules`, restart your computer, then `pnpm install` again. diff --git a/components/layout.tsx b/components/layout.tsx index 4481b4dcb..5e3d20700 100644 --- a/components/layout.tsx +++ b/components/layout.tsx @@ -14,7 +14,7 @@ export default function Layout({ children }: LayoutProps) { -
+
{children}
diff --git a/config/pinecone.ts b/config/pinecone.ts index f1851c8da..ce2dadaad 100644 --- a/config/pinecone.ts +++ b/config/pinecone.ts @@ -1,8 +1,12 @@ /** - * Change the index and namespace to your own + * Change the namespace to the namespace on Pinecone you'd like to store your embeddings. */ -const PINECONE_INDEX_NAME = 'langchainjsfundamentals'; +if (!process.env.PINECONE_INDEX_NAME) { + throw new Error('Missing Pinecone index name in .env file'); +} + +const PINECONE_INDEX_NAME = process.env.PINECONE_INDEX_NAME ?? ''; const PINECONE_NAME_SPACE = 'pdf-test'; //namespace is optional for your vectors diff --git a/declarations/pdf-parse.d.ts b/declarations/pdf-parse.d.ts new file mode 100644 index 000000000..5b2ab5020 --- /dev/null +++ b/declarations/pdf-parse.d.ts @@ -0,0 +1,5 @@ +declare module 'pdf-parse/lib/pdf-parse.js' { + import pdf from 'pdf-parse'; + + export default pdf; +} diff --git a/package.json b/package.json index 8d10f96c9..914013743 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "@radix-ui/react-accordion": "^1.1.1", "clsx": "^1.2.1", "dotenv": "^16.0.3", - "langchain": "0.0.33", + "langchain": "0.0.41", "lucide-react": "^0.125.0", "next": "13.2.3", "pdf-parse": "1.1.1", @@ -43,6 +43,9 @@ "tsx": "^3.12.3", "typescript": "^4.9.5" }, + "engines": { + "node": ">=18" + }, "keywords": [ "starter", "gpt4", diff --git a/pages/api/chat.ts b/pages/api/chat.ts index a96464411..82f97bb4f 100644 --- a/pages/api/chat.ts +++ b/pages/api/chat.ts @@ -21,10 +21,12 @@ export default async function handler( /* create vectorstore*/ const vectorStore = await PineconeStore.fromExistingIndex( - index, new OpenAIEmbeddings({}), - 'text', - PINECONE_NAME_SPACE, //optional + { + pineconeIndex: index, + textKey: 'text', + namespace: PINECONE_NAME_SPACE, + }, ); res.writeHead(200, { diff --git a/pages/index.tsx b/pages/index.tsx index 03a182a8e..90bb65f78 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -36,8 +36,6 @@ export default function Home() { const { messages, pending, history, pendingSourceDocs } = messageState; - console.log('messageState', messageState); - const messageListRef = useRef(null); const textAreaRef = useRef(null); @@ -201,7 +199,10 @@ export default function Home() {
{message.sourceDocs && ( -
+
{sourceDocs.map((doc, index) => ( -
+

Source {index + 1}

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 24cdbcfa6..deb6b2a79 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,7 +13,7 @@ specifiers: dotenv: ^16.0.3 eslint: 8.35.0 eslint-config-next: 13.2.3 - langchain: 0.0.33 + langchain: 0.0.41 lucide-react: ^0.125.0 next: 13.2.3 pdf-parse: 1.1.1 @@ -33,7 +33,7 @@ dependencies: '@radix-ui/react-accordion': 1.1.1_biqbaboplfbrettd7655fr4n2y clsx: 1.2.1 dotenv: 16.0.3 - langchain: 0.0.33_ppsbqni3ixqkis2lkrnfyb5ism + langchain: 0.0.41_gmj7kmepz6e4hbef53kmpddf3a lucide-react: 0.125.0_react@18.2.0 next: 13.2.3_biqbaboplfbrettd7655fr4n2y pdf-parse: 1.1.1 @@ -305,11 +305,6 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dev: true - /@gar/promisify/1.1.3: - resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} - dev: false - optional: true - /@humanwhocodes/config-array/0.11.8: resolution: {integrity: sha512-UybHIJzJnR5Qc/MsD9Kr+RpO2h+/P1GhOwdiLPXK5TWk5sgTdu88bTD9UP+CKbPPh5Rni1u0GjAdYQLemG8g+g==} engines: {node: '>=10.10.0'} @@ -330,24 +325,6 @@ packages: resolution: {integrity: sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==} dev: true - /@mapbox/node-pre-gyp/1.0.10: - resolution: {integrity: sha512-4ySo4CjzStuprMwk35H5pPbkymjv1SF3jGLj6rAHp/xT/RF7TL7bd9CTm1xDY49K2qF7jmR/g7k+SkLETP6opA==} - hasBin: true - dependencies: - detect-libc: 2.0.1 - https-proxy-agent: 5.0.1 - make-dir: 3.1.0 - node-fetch: 2.6.7 - nopt: 5.0.0 - npmlog: 5.0.1 - rimraf: 3.0.2 - semver: 7.3.8 - tar: 6.1.13 - transitivePeerDependencies: - - encoding - - supports-color - dev: false - /@microsoft/fetch-event-source/2.0.1: resolution: {integrity: sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==} dev: false @@ -500,24 +477,6 @@ packages: fastq: 1.15.0 dev: true - /@npmcli/fs/1.1.1: - resolution: {integrity: sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ==} - dependencies: - '@gar/promisify': 1.1.3 - semver: 7.3.8 - dev: false - optional: true - - /@npmcli/move-file/1.1.2: - resolution: {integrity: sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==} - engines: {node: '>=10'} - deprecated: This functionality has been moved to @npmcli/fs - dependencies: - mkdirp: 1.0.4 - rimraf: 3.0.2 - dev: false - optional: true - /@pinecone-database/pinecone/0.0.10: resolution: {integrity: sha512-SwfobqOmSQvEkEtw44uc6ZZ9R/aU5bDYkBAX0cGOcSk6wWlm5yiJA+XeZkyMeUsFb1EQaGhbafIy97TDqcnnsw==} engines: {node: '>=14.0.0'} @@ -703,22 +662,12 @@ packages: resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==} dev: true - /@sqltools/formatter/1.2.5: - resolution: {integrity: sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw==} - dev: false - /@swc/helpers/0.4.14: resolution: {integrity: sha512-4C7nX/dvpzB7za4Ql9K81xK3HPxCpHMgwTZVyf+9JQ6VUbn9jjZVN7/Nkdz/Ugzs2CSjqnL/UPXroiVBVHUWUw==} dependencies: tslib: 2.5.0 dev: false - /@tootallnate/once/1.1.2: - resolution: {integrity: sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==} - engines: {node: '>= 6'} - dev: false - optional: true - /@types/debug/4.1.7: resolution: {integrity: sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==} dependencies: @@ -834,10 +783,6 @@ packages: eslint-visitor-keys: 3.3.0 dev: true - /abbrev/1.1.1: - resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} - dev: false - /acorn-jsx/5.3.2_acorn@8.8.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -871,36 +816,6 @@ packages: hasBin: true dev: true - /agent-base/6.0.2: - resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} - engines: {node: '>= 6.0.0'} - dependencies: - debug: 4.3.4 - transitivePeerDependencies: - - supports-color - dev: false - - /agentkeepalive/4.3.0: - resolution: {integrity: sha512-7Epl1Blf4Sy37j4v9f9FjICCh4+KAQOyXgHEwlyBiAQLbhKdq/i2QQU3amQalS/wPhdPzDXPL5DMR5bkn+YeWg==} - engines: {node: '>= 8.0.0'} - dependencies: - debug: 4.3.4 - depd: 2.0.0 - humanize-ms: 1.2.1 - transitivePeerDependencies: - - supports-color - dev: false - optional: true - - /aggregate-error/3.1.0: - resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} - engines: {node: '>=8'} - dependencies: - clean-stack: 2.2.0 - indent-string: 4.0.0 - dev: false - optional: true - /ajv/6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} dependencies: @@ -913,16 +828,14 @@ packages: /ansi-regex/5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} + dev: true /ansi-styles/4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} dependencies: color-convert: 2.0.1 - - /any-promise/1.3.0: - resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} - dev: false + dev: true /anymatch/3.1.3: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} @@ -932,38 +845,13 @@ packages: picomatch: 2.3.1 dev: true - /app-root-path/3.1.0: - resolution: {integrity: sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==} - engines: {node: '>= 6.0.0'} - dev: false - - /aproba/2.0.0: - resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==} - dev: false - - /are-we-there-yet/2.0.0: - resolution: {integrity: sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==} - engines: {node: '>=10'} - dependencies: - delegates: 1.0.0 - readable-stream: 3.6.2 - dev: false - - /are-we-there-yet/3.0.1: - resolution: {integrity: sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - delegates: 1.0.0 - readable-stream: 3.6.2 - dev: false - optional: true - /arg/5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} dev: true /argparse/2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + dev: true /aria-query/5.1.3: resolution: {integrity: sha512-R5iJ5lkuHybztUfuOAznmboyjWq8O6sqNqtK7CLOqdydi54VNbORp49mb14KbWgG1QD3JFO9hJdZ+y4KutfdOQ==} @@ -1075,27 +963,18 @@ packages: /balanced-match/1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - - /base64-js/1.5.1: - resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - dev: false + dev: true /binary-extensions/2.2.0: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} - dev: true /brace-expansion/1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: balanced-match: 1.0.2 concat-map: 0.0.1 - - /brace-expansion/2.0.1: - resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==} - dependencies: - balanced-match: 1.0.2 - dev: false + dev: true /braces/3.0.2: resolution: {integrity: sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==} @@ -1123,40 +1002,6 @@ packages: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} dev: true - /buffer/6.0.3: - resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} - dependencies: - base64-js: 1.5.1 - ieee754: 1.2.1 - dev: false - - /cacache/15.3.0: - resolution: {integrity: sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==} - engines: {node: '>= 10'} - dependencies: - '@npmcli/fs': 1.1.1 - '@npmcli/move-file': 1.1.2 - chownr: 2.0.0 - fs-minipass: 2.1.0 - glob: 7.2.3 - infer-owner: 1.0.4 - lru-cache: 6.0.0 - minipass: 3.3.6 - minipass-collect: 1.0.2 - minipass-flush: 1.0.5 - minipass-pipeline: 1.2.4 - mkdirp: 1.0.4 - p-map: 4.0.0 - promise-inflight: 1.0.1 - rimraf: 3.0.2 - ssri: 8.0.1 - tar: 6.1.13 - unique-filename: 1.1.1 - transitivePeerDependencies: - - bluebird - dev: false - optional: true - /call-bind/1.0.2: resolution: {integrity: sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==} dependencies: @@ -1183,6 +1028,7 @@ packages: dependencies: ansi-styles: 4.3.0 supports-color: 7.2.0 + dev: true /character-entities/2.0.2: resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} @@ -1203,51 +1049,10 @@ packages: fsevents: 2.3.2 dev: true - /chownr/2.0.0: - resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} - engines: {node: '>=10'} - dev: false - - /clean-stack/2.2.0: - resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} - engines: {node: '>=6'} - dev: false - optional: true - - /cli-highlight/2.1.11: - resolution: {integrity: sha512-9KDcoEVwyUXrjcJNvHD0NFc/hiwe/WPVYIleQh2O1N2Zro5gWJZ/K+3DGn8w8P/F6FxOgzyC5bxDyHIgCSPhGg==} - engines: {node: '>=8.0.0', npm: '>=5.0.0'} - hasBin: true - dependencies: - chalk: 4.1.2 - highlight.js: 10.7.3 - mz: 2.7.0 - parse5: 5.1.1 - parse5-htmlparser2-tree-adapter: 6.0.1 - yargs: 16.2.0 - dev: false - /client-only/0.0.1: resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} dev: false - /cliui/7.0.4: - resolution: {integrity: sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==} - dependencies: - string-width: 4.2.3 - strip-ansi: 6.0.1 - wrap-ansi: 7.0.0 - dev: false - - /cliui/8.0.1: - resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} - engines: {node: '>=12'} - dependencies: - string-width: 4.2.3 - strip-ansi: 6.0.1 - wrap-ansi: 7.0.0 - dev: false - /clsx/1.2.1: resolution: {integrity: sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg==} engines: {node: '>=6'} @@ -1258,14 +1063,11 @@ packages: engines: {node: '>=7.0.0'} dependencies: color-name: 1.1.4 + dev: true /color-name/1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - - /color-support/1.1.3: - resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} - hasBin: true - dev: false + dev: true /combined-stream/1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} @@ -1278,17 +1080,9 @@ packages: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} dev: false - /commander/7.2.0: - resolution: {integrity: sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==} - engines: {node: '>= 10'} - dev: false - /concat-map/0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} - - /console-control-strings/1.1.0: - resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} - dev: false + dev: true /cross-fetch/3.1.5: resolution: {integrity: sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==} @@ -1316,25 +1110,10 @@ packages: /csstype/3.1.1: resolution: {integrity: sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==} - /d3-dsv/3.0.1: - resolution: {integrity: sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==} - engines: {node: '>=12'} - hasBin: true - dependencies: - commander: 7.2.0 - iconv-lite: 0.6.3 - rw: 1.3.3 - dev: false - /damerau-levenshtein/1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} dev: true - /date-fns/2.29.3: - resolution: {integrity: sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA==} - engines: {node: '>=0.11'} - dev: false - /debug/3.2.7: resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -1410,26 +1189,11 @@ packages: engines: {node: '>=0.4.0'} dev: false - /delegates/1.0.0: - resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} - dev: false - - /depd/2.0.0: - resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} - engines: {node: '>= 0.8'} - dev: false - optional: true - /dequal/2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} dev: false - /detect-libc/2.0.1: - resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==} - engines: {node: '>=8'} - dev: false - /detective/5.2.1: resolution: {integrity: sha512-v9XE1zRnz1wRtgurGu0Bs8uHKFSTdteYZNbIPFVhUZ39L/S79ppMpdmVOZAnoz1jfEFodc48n6MX483Xo3t1yw==} engines: {node: '>=0.8.0'} @@ -1483,22 +1247,10 @@ packages: resolution: {integrity: sha512-h70iRscrNluMZPVICXYl5SSB+rBKo22XfuIS1ER0OQxQZpKTnFpuS6coj7wY9M/3trv7OR88rRMOlKmRvDty7Q==} dev: true - /emoji-regex/8.0.0: - resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} - dev: false - /emoji-regex/9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} dev: true - /encoding/0.1.13: - resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==} - requiresBuild: true - dependencies: - iconv-lite: 0.6.3 - dev: false - optional: true - /enhanced-resolve/5.12.0: resolution: {integrity: sha512-QHTXI/sZQmko1cbDoNAa3mJ5qhWUUNAq3vR0/YiD379fWQrcfuoX1+HW2S0MTt7XmoPLapdaDKUtelUSPic7hQ==} engines: {node: '>=10.13.0'} @@ -1507,17 +1259,6 @@ packages: tapable: 2.2.1 dev: true - /env-paths/2.2.1: - resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} - engines: {node: '>=6'} - dev: false - optional: true - - /err-code/2.0.3: - resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} - dev: false - optional: true - /es-abstract/1.21.1: resolution: {integrity: sha512-QudMsPOz86xYz/1dG1OuGBKOELjCh99IIWHLzy5znUB6j8xG2yMA7bfTV86VSqKF+Y/H08vQPR+9jyXpuC6hfg==} engines: {node: '>= 0.4'} @@ -1628,6 +1369,7 @@ packages: /escalade/3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} + dev: true /escape-string-regexp/4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} @@ -2033,15 +1775,9 @@ packages: resolution: {integrity: sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==} dev: true - /fs-minipass/2.1.0: - resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} - engines: {node: '>= 8'} - dependencies: - minipass: 3.3.6 - dev: false - /fs.realpath/1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + dev: true /fsevents/2.3.2: resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} @@ -2069,41 +1805,6 @@ packages: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} dev: true - /gauge/3.0.2: - resolution: {integrity: sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==} - engines: {node: '>=10'} - dependencies: - aproba: 2.0.0 - color-support: 1.1.3 - console-control-strings: 1.1.0 - has-unicode: 2.0.1 - object-assign: 4.1.1 - signal-exit: 3.0.7 - string-width: 4.2.3 - strip-ansi: 6.0.1 - wide-align: 1.1.5 - dev: false - - /gauge/4.0.4: - resolution: {integrity: sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - aproba: 2.0.0 - color-support: 1.1.3 - console-control-strings: 1.1.0 - has-unicode: 2.0.1 - signal-exit: 3.0.7 - string-width: 4.2.3 - strip-ansi: 6.0.1 - wide-align: 1.1.5 - dev: false - optional: true - - /get-caller-file/2.0.5: - resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} - engines: {node: 6.* || 8.* || >= 10.*} - dev: false - /get-intrinsic/1.2.0: resolution: {integrity: sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==} dependencies: @@ -2158,17 +1859,7 @@ packages: minimatch: 3.1.2 once: 1.4.0 path-is-absolute: 1.0.1 - - /glob/8.1.0: - resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} - engines: {node: '>=12'} - dependencies: - fs.realpath: 1.0.0 - inflight: 1.0.6 - inherits: 2.0.4 - minimatch: 5.1.6 - once: 1.4.0 - dev: false + dev: true /globals/13.20.0: resolution: {integrity: sha512-Qg5QtVkCy/kv3FUSlu4ukeZDVf9ee0iXLAUYX13gbR17bnejFTzr4iS9bY7kwCf1NztRNm1t91fjOiyx4CSwPQ==} @@ -2230,6 +1921,7 @@ packages: /graceful-fs/4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} + dev: true /grapheme-splitter/1.0.4: resolution: {integrity: sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==} @@ -2242,6 +1934,7 @@ packages: /has-flag/4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + dev: true /has-property-descriptors/1.0.0: resolution: {integrity: sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==} @@ -2266,10 +1959,6 @@ packages: has-symbols: 1.0.3 dev: true - /has-unicode/2.0.1: - resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==} - dev: false - /has/1.0.3: resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} engines: {node: '>= 0.4.0'} @@ -2281,55 +1970,6 @@ packages: resolution: {integrity: sha512-nAxA0v8+vXSBDt3AnRUNjyRIQ0rD+ntpbAp4LnPkumc5M9yUbSMa4XDU9Q6etY4f1Wp4bNgvc1yjiZtsTTrSng==} dev: false - /highlight.js/10.7.3: - resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} - dev: false - - /http-cache-semantics/4.1.1: - resolution: {integrity: sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==} - dev: false - optional: true - - /http-proxy-agent/4.0.1: - resolution: {integrity: sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==} - engines: {node: '>= 6'} - dependencies: - '@tootallnate/once': 1.1.2 - agent-base: 6.0.2 - debug: 4.3.4 - transitivePeerDependencies: - - supports-color - dev: false - optional: true - - /https-proxy-agent/5.0.1: - resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} - engines: {node: '>= 6'} - dependencies: - agent-base: 6.0.2 - debug: 4.3.4 - transitivePeerDependencies: - - supports-color - dev: false - - /humanize-ms/1.2.1: - resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} - dependencies: - ms: 2.1.3 - dev: false - optional: true - - /iconv-lite/0.6.3: - resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} - engines: {node: '>=0.10.0'} - dependencies: - safer-buffer: 2.1.2 - dev: false - - /ieee754/1.2.1: - resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - dev: false - /ignore/5.2.4: resolution: {integrity: sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==} engines: {node: '>= 4'} @@ -2346,26 +1986,18 @@ packages: /imurmurhash/0.1.4: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} - - /indent-string/4.0.0: - resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} - engines: {node: '>=8'} - dev: false - optional: true - - /infer-owner/1.0.4: - resolution: {integrity: sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==} - dev: false - optional: true + dev: true /inflight/1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} dependencies: once: 1.4.0 wrappy: 1.0.2 + dev: true /inherits/2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: true /inline-style-parser/0.1.1: resolution: {integrity: sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==} @@ -2380,11 +2012,6 @@ packages: side-channel: 1.0.4 dev: true - /ip/2.0.0: - resolution: {integrity: sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==} - dev: false - optional: true - /is-arguments/1.1.1: resolution: {integrity: sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==} engines: {node: '>= 0.4'} @@ -2412,7 +2039,6 @@ packages: engines: {node: '>=8'} dependencies: binary-extensions: 2.2.0 - dev: true /is-boolean-object/1.1.2: resolution: {integrity: sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==} @@ -2456,11 +2082,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /is-fullwidth-code-point/3.0.0: - resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} - engines: {node: '>=8'} - dev: false - /is-glob/4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} @@ -2468,11 +2089,6 @@ packages: is-extglob: 2.1.1 dev: true - /is-lambda/1.0.1: - resolution: {integrity: sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==} - dev: false - optional: true - /is-map/2.0.2: resolution: {integrity: sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==} dev: true @@ -2577,6 +2193,7 @@ packages: /isexe/2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + dev: true /js-sdsl/4.3.0: resolution: {integrity: sha512-mifzlm2+5nZ+lEcLJMoBK0/IH/bDg8XnJfd/Wq6IP+xoCjLZsTOnV2QpxlVbX9bMnkl5PdEjNtBJ9Cj1NjifhQ==} @@ -2590,6 +2207,7 @@ packages: hasBin: true dependencies: argparse: 2.0.1 + dev: true /json-schema-traverse/0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} @@ -2624,21 +2242,24 @@ packages: engines: {node: '>=6'} dev: false - /langchain/0.0.33_ppsbqni3ixqkis2lkrnfyb5ism: - resolution: {integrity: sha512-XSquMWGTXDprGYlae079igijWnjP31nMLLXuG46M/F5uFBszZx1YeJh5DMLGmmqU30JT04T570tFKcIfRSg9KA==} + /langchain/0.0.41_gmj7kmepz6e4hbef53kmpddf3a: + resolution: {integrity: sha512-X7JaexSicy1xfXpJNaQtUSTAjuNmpx0m56ygpMhGZ1ZKhXCXUTIXN4odUMOyP2+h9UnsthjZFrILFNZglghv4g==} peerDependencies: - '@dqbd/tiktoken': ^0.4.0 + '@dqbd/tiktoken': ^1.0.2 '@huggingface/inference': ^1.5.1 '@pinecone-database/pinecone': ^0.0.10 '@supabase/supabase-js': ^2.10.0 cheerio: ^1.0.0-rc.12 chromadb: ^1.3.0 cohere-ai: ^5.0.2 + d3-dsv: ^3.0.1 hnswlib-node: ^1.4.2 + pdf-parse: ^1.1.1 puppeteer: ^19.7.2 redis: ^4.6.4 serpapi: ^1.1.1 srt-parser-2: ^1.2.2 + typeorm: ^0.3.12 peerDependenciesMeta: '@dqbd/tiktoken': optional: true @@ -2654,8 +2275,12 @@ packages: optional: true cohere-ai: optional: true + d3-dsv: + optional: true hnswlib-node: optional: true + pdf-parse: + optional: true puppeteer: optional: true redis: @@ -2664,43 +2289,26 @@ packages: optional: true srt-parser-2: optional: true + typeorm: + optional: true dependencies: '@pinecone-database/pinecone': 0.0.10 browser-or-node: 2.1.1 - d3-dsv: 3.0.1 eventsource-parser: 0.1.0 exponential-backoff: 3.1.1 expr-eval: 2.0.2 gpt3-tokenizer: 1.1.5 + is-binary-path: 2.1.0 jsonpointer: 5.0.1 + object-hash: 3.0.0 openai: 3.2.1 p-queue: 7.3.4 pdf-parse: 1.1.1 - sqlite3: 5.1.6 - typeorm: 0.3.12_sqlite3@5.1.6 uuid: 9.0.0 yaml: 2.2.1 zod: 3.21.4 transitivePeerDependencies: - - '@google-cloud/spanner' - - '@sap/hana-client' - - better-sqlite3 - - bluebird - debug - - encoding - - hdb-pool - - ioredis - - mongodb - - mssql - - mysql2 - - oracledb - - pg - - pg-native - - pg-query-stream - - sql.js - - supports-color - - ts-node - - typeorm-aurora-data-api-driver dev: false /language-subtag-registry/0.3.22: @@ -2748,6 +2356,7 @@ packages: engines: {node: '>=10'} dependencies: yallist: 4.0.0 + dev: true /lucide-react/0.125.0_react@18.2.0: resolution: {integrity: sha512-tadphtB6TPytEitR9vX75hqu9PQT/uz5RcvXMq976nC190eukAM9+cHMgBxfvfEGDXwIhIT9aFxTUGdAjxw9uQ==} @@ -2757,49 +2366,16 @@ packages: react: 18.2.0 dev: false - /make-dir/3.1.0: - resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} - engines: {node: '>=8'} + /mdast-util-definitions/5.1.2: + resolution: {integrity: sha512-8SVPMuHqlPME/z3gqVwWY4zVXn8lqKv/pAhC57FuJ40ImXyBpmO5ukh98zB2v7Blql2FiHjHv9LVztSIqjY+MA==} dependencies: - semver: 6.3.0 + '@types/mdast': 3.0.10 + '@types/unist': 2.0.6 + unist-util-visit: 4.1.2 dev: false - /make-fetch-happen/9.1.0: - resolution: {integrity: sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==} - engines: {node: '>= 10'} - dependencies: - agentkeepalive: 4.3.0 - cacache: 15.3.0 - http-cache-semantics: 4.1.1 - http-proxy-agent: 4.0.1 - https-proxy-agent: 5.0.1 - is-lambda: 1.0.1 - lru-cache: 6.0.0 - minipass: 3.3.6 - minipass-collect: 1.0.2 - minipass-fetch: 1.4.1 - minipass-flush: 1.0.5 - minipass-pipeline: 1.2.4 - negotiator: 0.6.3 - promise-retry: 2.0.1 - socks-proxy-agent: 6.2.1 - ssri: 8.0.1 - transitivePeerDependencies: - - bluebird - - supports-color - dev: false - optional: true - - /mdast-util-definitions/5.1.2: - resolution: {integrity: sha512-8SVPMuHqlPME/z3gqVwWY4zVXn8lqKv/pAhC57FuJ40ImXyBpmO5ukh98zB2v7Blql2FiHjHv9LVztSIqjY+MA==} - dependencies: - '@types/mdast': 3.0.10 - '@types/unist': 2.0.6 - unist-util-visit: 4.1.2 - dev: false - - /mdast-util-from-markdown/1.3.0: - resolution: {integrity: sha512-HN3W1gRIuN/ZW295c7zi7g9lVBllMgZE40RxCX37wrTPWXCWtpvOZdfnuK+1WNpvZje6XuJeI3Wnb4TJEUem+g==} + /mdast-util-from-markdown/1.3.0: + resolution: {integrity: sha512-HN3W1gRIuN/ZW295c7zi7g9lVBllMgZE40RxCX37wrTPWXCWtpvOZdfnuK+1WNpvZje6XuJeI3Wnb4TJEUem+g==} dependencies: '@types/mdast': 3.0.10 '@types/unist': 2.0.6 @@ -3041,94 +2617,12 @@ packages: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} dependencies: brace-expansion: 1.1.11 - - /minimatch/5.1.6: - resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} - engines: {node: '>=10'} - dependencies: - brace-expansion: 2.0.1 - dev: false + dev: true /minimist/1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} dev: true - /minipass-collect/1.0.2: - resolution: {integrity: sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==} - engines: {node: '>= 8'} - dependencies: - minipass: 3.3.6 - dev: false - optional: true - - /minipass-fetch/1.4.1: - resolution: {integrity: sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==} - engines: {node: '>=8'} - dependencies: - minipass: 3.3.6 - minipass-sized: 1.0.3 - minizlib: 2.1.2 - optionalDependencies: - encoding: 0.1.13 - dev: false - optional: true - - /minipass-flush/1.0.5: - resolution: {integrity: sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==} - engines: {node: '>= 8'} - dependencies: - minipass: 3.3.6 - dev: false - optional: true - - /minipass-pipeline/1.2.4: - resolution: {integrity: sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==} - engines: {node: '>=8'} - dependencies: - minipass: 3.3.6 - dev: false - optional: true - - /minipass-sized/1.0.3: - resolution: {integrity: sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==} - engines: {node: '>=8'} - dependencies: - minipass: 3.3.6 - dev: false - optional: true - - /minipass/3.3.6: - resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} - engines: {node: '>=8'} - dependencies: - yallist: 4.0.0 - dev: false - - /minipass/4.2.5: - resolution: {integrity: sha512-+yQl7SX3bIT83Lhb4BVorMAHVuqsskxRdlmO9kTpyukp8vsm2Sn/fUOV9xlnG8/a5JsypJzap21lz/y3FBMJ8Q==} - engines: {node: '>=8'} - dev: false - - /minizlib/2.1.2: - resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} - engines: {node: '>= 8'} - dependencies: - minipass: 3.3.6 - yallist: 4.0.0 - dev: false - - /mkdirp/1.0.4: - resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} - engines: {node: '>=10'} - hasBin: true - dev: false - - /mkdirp/2.1.5: - resolution: {integrity: sha512-jbjfql+shJtAPrFoKxHOXip4xS+kul9W3OzfzzrqueWK2QMGon2bFH2opl6W9EagBThjEz+iysyi/swOoVfB/w==} - engines: {node: '>=10'} - hasBin: true - dev: false - /mri/1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} @@ -3140,14 +2634,6 @@ packages: /ms/2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - /mz/2.7.0: - resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} - dependencies: - any-promise: 1.3.0 - object-assign: 4.1.1 - thenify-all: 1.6.0 - dev: false - /nanoid/3.3.4: resolution: {integrity: sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -3157,12 +2643,6 @@ packages: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true - /negotiator/0.6.3: - resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} - engines: {node: '>= 0.6'} - dev: false - optional: true - /next/13.2.3_biqbaboplfbrettd7655fr4n2y: resolution: {integrity: sha512-nKFJC6upCPN7DWRx4+0S/1PIOT7vNlCT157w9AzbXEgKy6zkiPKEt5YyRUsRZkmpEqBVrGgOqNfwecTociyg+w==} engines: {node: '>=14.6.0'} @@ -3210,10 +2690,6 @@ packages: - babel-plugin-macros dev: false - /node-addon-api/4.3.0: - resolution: {integrity: sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==} - dev: false - /node-ensure/0.0.0: resolution: {integrity: sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==} dev: false @@ -3230,40 +2706,10 @@ packages: whatwg-url: 5.0.0 dev: false - /node-gyp/8.4.1: - resolution: {integrity: sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==} - engines: {node: '>= 10.12.0'} - hasBin: true - requiresBuild: true - dependencies: - env-paths: 2.2.1 - glob: 7.2.3 - graceful-fs: 4.2.10 - make-fetch-happen: 9.1.0 - nopt: 5.0.0 - npmlog: 6.0.2 - rimraf: 3.0.2 - semver: 7.3.8 - tar: 6.1.13 - which: 2.0.2 - transitivePeerDependencies: - - bluebird - - supports-color - dev: false - optional: true - /node-releases/2.0.10: resolution: {integrity: sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==} dev: true - /nopt/5.0.0: - resolution: {integrity: sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==} - engines: {node: '>=6'} - hasBin: true - dependencies: - abbrev: 1.1.1 - dev: false - /normalize-path/3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} @@ -3274,26 +2720,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /npmlog/5.0.1: - resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==} - dependencies: - are-we-there-yet: 2.0.0 - console-control-strings: 1.1.0 - gauge: 3.0.2 - set-blocking: 2.0.0 - dev: false - - /npmlog/6.0.2: - resolution: {integrity: sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - are-we-there-yet: 3.0.1 - console-control-strings: 1.1.0 - gauge: 4.0.4 - set-blocking: 2.0.0 - dev: false - optional: true - /object-assign/4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -3301,7 +2727,6 @@ packages: /object-hash/3.0.0: resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} engines: {node: '>= 6'} - dev: true /object-inspect/1.12.3: resolution: {integrity: sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==} @@ -3368,6 +2793,7 @@ packages: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} dependencies: wrappy: 1.0.2 + dev: true /open/8.4.2: resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} @@ -3413,14 +2839,6 @@ packages: p-limit: 3.1.0 dev: true - /p-map/4.0.0: - resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==} - engines: {node: '>=10'} - dependencies: - aggregate-error: 3.1.0 - dev: false - optional: true - /p-queue/7.3.4: resolution: {integrity: sha512-esox8CWt0j9EZECFvkFl2WNPat8LN4t7WWeXq73D9ha0V96qPRufApZi4ZhPwXAln1uVVal429HVVKPa2X0yQg==} engines: {node: '>=12'} @@ -3441,20 +2859,6 @@ packages: callsites: 3.1.0 dev: true - /parse5-htmlparser2-tree-adapter/6.0.1: - resolution: {integrity: sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==} - dependencies: - parse5: 6.0.1 - dev: false - - /parse5/5.1.1: - resolution: {integrity: sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==} - dev: false - - /parse5/6.0.1: - resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} - dev: false - /path-exists/4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -3463,6 +2867,7 @@ packages: /path-is-absolute/1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} + dev: true /path-key/3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} @@ -3591,25 +2996,6 @@ packages: hasBin: true dev: true - /promise-inflight/1.0.1: - resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} - peerDependencies: - bluebird: '*' - peerDependenciesMeta: - bluebird: - optional: true - dev: false - optional: true - - /promise-retry/2.0.1: - resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} - engines: {node: '>=10'} - dependencies: - err-code: 2.0.3 - retry: 0.12.0 - dev: false - optional: true - /prop-types/15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} dependencies: @@ -3692,15 +3078,6 @@ packages: pify: 2.3.0 dev: true - /readable-stream/3.6.2: - resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} - engines: {node: '>= 6'} - dependencies: - inherits: 2.0.4 - string_decoder: 1.3.0 - util-deprecate: 1.0.2 - dev: false - /readdirp/3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} @@ -3708,10 +3085,6 @@ packages: picomatch: 2.3.1 dev: true - /reflect-metadata/0.1.13: - resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==} - dev: false - /regenerator-runtime/0.13.11: resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==} @@ -3748,11 +3121,6 @@ packages: unified: 10.1.2 dev: false - /require-directory/2.1.1: - resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} - engines: {node: '>=0.10.0'} - dev: false - /resolve-from/4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -3776,12 +3144,6 @@ packages: supports-preserve-symlinks-flag: 1.0.0 dev: true - /retry/0.12.0: - resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} - engines: {node: '>= 4'} - dev: false - optional: true - /reusify/1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -3792,6 +3154,7 @@ packages: hasBin: true dependencies: glob: 7.2.3 + dev: true /run-parallel/1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -3799,10 +3162,6 @@ packages: queue-microtask: 1.2.3 dev: true - /rw/1.3.3: - resolution: {integrity: sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==} - dev: false - /sade/1.8.1: resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} engines: {node: '>=6'} @@ -3810,10 +3169,6 @@ packages: mri: 1.2.0 dev: false - /safe-buffer/5.2.1: - resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} - dev: false - /safe-regex-test/1.0.0: resolution: {integrity: sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==} dependencies: @@ -3822,14 +3177,6 @@ packages: is-regex: 1.1.4 dev: true - /safer-buffer/2.1.2: - resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} - dev: false - - /sax/1.2.4: - resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==} - dev: false - /scheduler/0.23.0: resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==} dependencies: @@ -3839,6 +3186,7 @@ packages: /semver/6.3.0: resolution: {integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==} hasBin: true + dev: true /semver/7.3.8: resolution: {integrity: sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==} @@ -3846,18 +3194,7 @@ packages: hasBin: true dependencies: lru-cache: 6.0.0 - - /set-blocking/2.0.0: - resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} - dev: false - - /sha.js/2.4.11: - resolution: {integrity: sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==} - hasBin: true - dependencies: - inherits: 2.0.4 - safe-buffer: 5.2.1 - dev: false + dev: true /shebang-command/2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} @@ -3879,10 +3216,6 @@ packages: object-inspect: 1.12.3 dev: true - /signal-exit/3.0.7: - resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} - dev: false - /slash/3.0.0: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} @@ -3893,33 +3226,6 @@ packages: engines: {node: '>=12'} dev: true - /smart-buffer/4.2.0: - resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} - engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} - dev: false - optional: true - - /socks-proxy-agent/6.2.1: - resolution: {integrity: sha512-a6KW9G+6B3nWZ1yB8G7pJwL3ggLy1uTzKAgCb7ttblwqdz9fMGJUuTy3uFzEP48FAs9FLILlmzDlE2JJhVQaXQ==} - engines: {node: '>= 10'} - dependencies: - agent-base: 6.0.2 - debug: 4.3.4 - socks: 2.7.1 - transitivePeerDependencies: - - supports-color - dev: false - optional: true - - /socks/2.7.1: - resolution: {integrity: sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==} - engines: {node: '>= 10.13.0', npm: '>= 3.0.0'} - dependencies: - ip: 2.0.0 - smart-buffer: 4.2.0 - dev: false - optional: true - /source-map-js/1.0.2: resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==} engines: {node: '>=0.10.0'} @@ -3940,32 +3246,6 @@ packages: resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} dev: false - /sqlite3/5.1.6: - resolution: {integrity: sha512-olYkWoKFVNSSSQNvxVUfjiVbz3YtBwTJj+mfV5zpHmqW3sELx2Cf4QCdirMelhM5Zh+KDVaKgQHqCxrqiWHybw==} - requiresBuild: true - peerDependenciesMeta: - node-gyp: - optional: true - dependencies: - '@mapbox/node-pre-gyp': 1.0.10 - node-addon-api: 4.3.0 - tar: 6.1.13 - optionalDependencies: - node-gyp: 8.4.1 - transitivePeerDependencies: - - bluebird - - encoding - - supports-color - dev: false - - /ssri/8.0.1: - resolution: {integrity: sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==} - engines: {node: '>= 8'} - dependencies: - minipass: 3.3.6 - dev: false - optional: true - /stop-iteration-iterator/1.0.0: resolution: {integrity: sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==} engines: {node: '>= 0.4'} @@ -3973,15 +3253,6 @@ packages: internal-slot: 1.0.5 dev: true - /string-width/4.2.3: - resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} - engines: {node: '>=8'} - dependencies: - emoji-regex: 8.0.0 - is-fullwidth-code-point: 3.0.0 - strip-ansi: 6.0.1 - dev: false - /string.prototype.matchall/4.0.8: resolution: {integrity: sha512-6zOCOcJ+RJAQshcTvXPHoxoQGONa3e/Lqx90wUA+wEzX78sg5Bo+1tQo4N0pohS0erG9qtCqJDjNCQBjeWVxyg==} dependencies: @@ -4011,17 +3282,12 @@ packages: es-abstract: 1.21.1 dev: true - /string_decoder/1.3.0: - resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} - dependencies: - safe-buffer: 5.2.1 - dev: false - /strip-ansi/6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} dependencies: ansi-regex: 5.0.1 + dev: true /strip-bom/3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} @@ -4061,6 +3327,7 @@ packages: engines: {node: '>=8'} dependencies: has-flag: 4.0.0 + dev: true /supports-preserve-symlinks-flag/1.0.0: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} @@ -4118,35 +3385,10 @@ packages: engines: {node: '>=6'} dev: true - /tar/6.1.13: - resolution: {integrity: sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==} - engines: {node: '>=10'} - dependencies: - chownr: 2.0.0 - fs-minipass: 2.1.0 - minipass: 4.2.5 - minizlib: 2.1.2 - mkdirp: 1.0.4 - yallist: 4.0.0 - dev: false - /text-table/0.2.0: resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} dev: true - /thenify-all/1.6.0: - resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} - engines: {node: '>=0.8'} - dependencies: - thenify: 3.3.1 - dev: false - - /thenify/3.3.1: - resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} - dependencies: - any-promise: 1.3.0 - dev: false - /tiny-glob/0.2.9: resolution: {integrity: sha512-g/55ssRPUjShh+xkfx9UPDXqhckHEsHr4Vd9zX55oSdGZc/MD0m3sferOkwWtp98bv+kcVfEHtRJgBVJzelrzg==} dependencies: @@ -4230,86 +3472,6 @@ packages: is-typed-array: 1.1.10 dev: true - /typeorm/0.3.12_sqlite3@5.1.6: - resolution: {integrity: sha512-sYSxBmCf1nJLLTcYtwqZ+lQIRtLPyUoO93rHTOKk9vJCyT4UfRtU7oRsJvfvKP3nnZTD1hzz2SEy2zwPEN6OyA==} - engines: {node: '>= 12.9.0'} - hasBin: true - peerDependencies: - '@google-cloud/spanner': ^5.18.0 - '@sap/hana-client': ^2.12.25 - better-sqlite3: ^7.1.2 || ^8.0.0 - hdb-pool: ^0.1.6 - ioredis: ^5.0.4 - mongodb: ^3.6.0 - mssql: ^7.3.0 - mysql2: ^2.2.5 || ^3.0.1 - oracledb: ^5.1.0 - pg: ^8.5.1 - pg-native: ^3.0.0 - pg-query-stream: ^4.0.0 - redis: ^3.1.1 || ^4.0.0 - sql.js: ^1.4.0 - sqlite3: ^5.0.3 - ts-node: ^10.7.0 - typeorm-aurora-data-api-driver: ^2.0.0 - peerDependenciesMeta: - '@google-cloud/spanner': - optional: true - '@sap/hana-client': - optional: true - better-sqlite3: - optional: true - hdb-pool: - optional: true - ioredis: - optional: true - mongodb: - optional: true - mssql: - optional: true - mysql2: - optional: true - oracledb: - optional: true - pg: - optional: true - pg-native: - optional: true - pg-query-stream: - optional: true - redis: - optional: true - sql.js: - optional: true - sqlite3: - optional: true - ts-node: - optional: true - typeorm-aurora-data-api-driver: - optional: true - dependencies: - '@sqltools/formatter': 1.2.5 - app-root-path: 3.1.0 - buffer: 6.0.3 - chalk: 4.1.2 - cli-highlight: 2.1.11 - date-fns: 2.29.3 - debug: 4.3.4 - dotenv: 16.0.3 - glob: 8.1.0 - js-yaml: 4.1.0 - mkdirp: 2.1.5 - reflect-metadata: 0.1.13 - sha.js: 2.4.11 - sqlite3: 5.1.6 - tslib: 2.5.0 - uuid: 9.0.0 - xml2js: 0.4.23 - yargs: 17.7.1 - transitivePeerDependencies: - - supports-color - dev: false - /typescript/4.9.5: resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} engines: {node: '>=4.2.0'} @@ -4337,20 +3499,6 @@ packages: vfile: 5.3.7 dev: false - /unique-filename/1.1.1: - resolution: {integrity: sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==} - dependencies: - unique-slug: 2.0.2 - dev: false - optional: true - - /unique-slug/2.0.2: - resolution: {integrity: sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==} - dependencies: - imurmurhash: 0.1.4 - dev: false - optional: true - /unist-util-generated/2.0.1: resolution: {integrity: sha512-qF72kLmPxAw0oN2fwpWIqbXAVyEqUzDHMsbtPvOudIlUzXYFIeQIuxXQCRCFh22B7cixvU0MG7m3MW8FTq/S+A==} dev: false @@ -4407,6 +3555,7 @@ packages: /util-deprecate/1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + dev: true /uuid/9.0.0: resolution: {integrity: sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==} @@ -4488,55 +3637,25 @@ packages: hasBin: true dependencies: isexe: 2.0.0 - - /wide-align/1.1.5: - resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==} - dependencies: - string-width: 4.2.3 - dev: false + dev: true /word-wrap/1.2.3: resolution: {integrity: sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==} engines: {node: '>=0.10.0'} dev: true - /wrap-ansi/7.0.0: - resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} - engines: {node: '>=10'} - dependencies: - ansi-styles: 4.3.0 - string-width: 4.2.3 - strip-ansi: 6.0.1 - dev: false - /wrappy/1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - - /xml2js/0.4.23: - resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==} - engines: {node: '>=4.0.0'} - dependencies: - sax: 1.2.4 - xmlbuilder: 11.0.1 - dev: false - - /xmlbuilder/11.0.1: - resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==} - engines: {node: '>=4.0'} - dev: false + dev: true /xtend/4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} dev: true - /y18n/5.0.8: - resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} - engines: {node: '>=10'} - dev: false - /yallist/4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: true /yaml/1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} @@ -4548,42 +3667,6 @@ packages: engines: {node: '>= 14'} dev: false - /yargs-parser/20.2.9: - resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} - engines: {node: '>=10'} - dev: false - - /yargs-parser/21.1.1: - resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} - engines: {node: '>=12'} - dev: false - - /yargs/16.2.0: - resolution: {integrity: sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==} - engines: {node: '>=10'} - dependencies: - cliui: 7.0.4 - escalade: 3.1.1 - get-caller-file: 2.0.5 - require-directory: 2.1.1 - string-width: 4.2.3 - y18n: 5.0.8 - yargs-parser: 20.2.9 - dev: false - - /yargs/17.7.1: - resolution: {integrity: sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw==} - engines: {node: '>=12'} - dependencies: - cliui: 8.0.1 - escalade: 3.1.1 - get-caller-file: 2.0.5 - require-directory: 2.1.1 - string-width: 4.2.3 - y18n: 5.0.8 - yargs-parser: 21.1.1 - dev: false - /yocto-queue/0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} diff --git a/scripts/ingest-data.ts b/scripts/ingest-data.ts index 392cfd5a9..c17a651cc 100644 --- a/scripts/ingest-data.ts +++ b/scripts/ingest-data.ts @@ -2,7 +2,7 @@ import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; import { OpenAIEmbeddings } from 'langchain/embeddings'; import { PineconeStore } from 'langchain/vectorstores'; import { pinecone } from '@/utils/pinecone-client'; -import { PDFLoader } from 'langchain/document_loaders'; +import { CustomPDFLoader } from '@/utils/customPDFLoader'; import { PINECONE_INDEX_NAME, PINECONE_NAME_SPACE } from '@/config/pinecone'; /* Name of directory to retrieve files from. You can change this as required */ @@ -11,7 +11,7 @@ const filePath = 'docs/MorseVsFrederick.pdf'; export const run = async () => { try { /*load raw docs from the pdf file in the directory */ - const loader = new PDFLoader(filePath); + const loader = new CustomPDFLoader(filePath); // const loader = new PDFLoader(filePath); const rawDocs = await loader.load(); @@ -38,13 +38,11 @@ export const run = async () => { for (let i = 0; i < docs.length; i += chunkSize) { const chunk = docs.slice(i, i + chunkSize); console.log('chunk', i, chunk); - await PineconeStore.fromDocuments( - index, - chunk, - embeddings, - 'text', - PINECONE_NAME_SPACE, - ); + await PineconeStore.fromDocuments(chunk, embeddings, { + pineconeIndex: index, + namespace: PINECONE_NAME_SPACE, + textKey: 'text', + }); } } catch (error) { console.log('error', error); diff --git a/utils/customPDFLoader.ts b/utils/customPDFLoader.ts new file mode 100644 index 000000000..dba711485 --- /dev/null +++ b/utils/customPDFLoader.ts @@ -0,0 +1,61 @@ +import { Document } from 'langchain/document'; +import { readFile } from 'fs/promises'; +import { BaseDocumentLoader } from 'langchain/document_loaders'; + +export abstract class BufferLoader extends BaseDocumentLoader { + constructor(public filePathOrBlob: string | Blob) { + super(); + } + + protected abstract parse( + raw: Buffer, + metadata: Document['metadata'], + ): Promise; + + public async load(): Promise { + let buffer: Buffer; + let metadata: Record; + if (typeof this.filePathOrBlob === 'string') { + buffer = await readFile(this.filePathOrBlob); + metadata = { source: this.filePathOrBlob }; + } else { + buffer = await this.filePathOrBlob + .arrayBuffer() + .then((ab) => Buffer.from(ab)); + metadata = { source: 'blob', blobType: this.filePathOrBlob.type }; + } + return this.parse(buffer, metadata); + } +} + +export class CustomPDFLoader extends BufferLoader { + public async parse( + raw: Buffer, + metadata: Document['metadata'], + ): Promise { + const { pdf } = await PDFLoaderImports(); + const parsed = await pdf(raw); + return [ + new Document({ + pageContent: parsed.text, + metadata: { + ...metadata, + pdf_numpages: parsed.numpages, + }, + }), + ]; + } +} + +async function PDFLoaderImports() { + try { + // the main entrypoint has some debug code that we don't want to import + const { default: pdf } = await import('pdf-parse/lib/pdf-parse.js'); + return { pdf }; + } catch (e) { + console.error(e); + throw new Error( + 'Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`.', + ); + } +} diff --git a/utils/pinecone-client.ts b/utils/pinecone-client.ts index b0da2c655..c84990034 100644 --- a/utils/pinecone-client.ts +++ b/utils/pinecone-client.ts @@ -1,6 +1,5 @@ import { PineconeClient } from '@pinecone-database/pinecone'; -console.log(process.env.PINECONE_ENVIRONMENT); if (!process.env.PINECONE_ENVIRONMENT || !process.env.PINECONE_API_KEY) { throw new Error('Pinecone environment or api key vars missing'); }