From e1894f6bf39d9145d55542cfec0be0074d44dc69 Mon Sep 17 00:00:00 2001 From: Sai Kranthi Date: Mon, 6 May 2024 12:21:22 +0530 Subject: [PATCH 1/4] feat: add arfs to create public repos and read them --- package.json | 1 + src/helpers/constants.ts | 2 +- src/helpers/getArrayBufSize.ts | 12 ++- src/lib/arfs/arfsSingleton.ts | 51 +++++++++++ src/lib/arfs/getArFS.ts | 7 ++ src/lib/arfs/getBifrost.ts | 7 ++ src/lib/git/helpers/fsWithName.ts | 12 ++- src/lib/git/repo.ts | 98 +++++++++++----------- src/pages/home/components/NewRepoModal.tsx | 29 +++---- src/stores/repository-core/actions/git.ts | 41 +++++---- tsconfig.json | 2 +- yarn.lock | 29 ++++++- 12 files changed, 203 insertions(+), 88 deletions(-) create mode 100644 src/lib/arfs/arfsSingleton.ts create mode 100644 src/lib/arfs/getArFS.ts create mode 100644 src/lib/arfs/getBifrost.ts diff --git a/package.json b/package.json index ac4d0136..72e04d78 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "@uiw/react-codemirror": "^4.21.11", "@uiw/react-md-editor": "^3.23.5", "ardb": "^1.1.10", + "arfs-js": "^1.2.6", "arweave": "^1.14.4", "clsx": "^2.0.0", "date-fns": "^2.30.0", diff --git a/src/helpers/constants.ts b/src/helpers/constants.ts index fcbe9fc2..d923e638 100644 --- a/src/helpers/constants.ts +++ b/src/helpers/constants.ts @@ -1,4 +1,4 @@ -export const CONTRACT_TX_ID = 'w5ZU15Y2cLzZlu3jewauIlnzbKw-OAxbN9G5TbuuiDQ' +export const CONTRACT_TX_ID = 'NLIpqKz21gA68AQUL7zDorx7MIYHTfrrSbXDn2vhm-I' export const VITE_GA_TRACKING_ID = 'G-L433HSR0D0' export const AMPLITUDE_TRACKING_ID = '92a463755ed8c8b96f0f2353a37b7b2' export const PL_REPO_ID = '6ace6247-d267-463d-b5bd-7e50d98c3693' diff --git a/src/helpers/getArrayBufSize.ts b/src/helpers/getArrayBufSize.ts index 5c7d7f4e..5695e52f 100644 --- a/src/helpers/getArrayBufSize.ts +++ b/src/helpers/getArrayBufSize.ts @@ -1,5 +1,13 @@ -export function getArrayBufSize(arrayBuffer: ArrayBuffer): GetArrayBufSizeReturnType { - const byteSize = arrayBuffer.byteLength +export function getRepoSize(arrayBufferOrSize: ArrayBuffer | number): GetArrayBufSizeReturnType { + let byteSize = 0 + + if (arrayBufferOrSize instanceof ArrayBuffer) { + byteSize = arrayBufferOrSize.byteLength + } + + if (typeof arrayBufferOrSize === 'number') { + byteSize = arrayBufferOrSize + } if (byteSize >= 1073741824) { return { diff --git a/src/lib/arfs/arfsSingleton.ts b/src/lib/arfs/arfsSingleton.ts new file mode 100644 index 00000000..1100e911 --- /dev/null +++ b/src/lib/arfs/arfsSingleton.ts @@ -0,0 +1,51 @@ +import { ArFS, BiFrost, Drive } from 'arfs-js' + +let instance: ArFSSingleton +let driveInstance: Drive | null = null +let bifrostInstance: BiFrost | null = null +let arfsInstance: ArFS | null = null + +class ArFSSingleton { + driveInstance: Drive | null = null + bifrostInstance: BiFrost | null = null + arfsInstance: ArFS | null = null + + constructor() { + if (instance) { + throw new Error('You can only create one instance!') + } + // eslint-disable-next-line @typescript-eslint/no-this-alias + instance = this + } + + getInstance() { + return this + } + + getBifrostInstance() { + return bifrostInstance + } + + getArfsInstance() { + return arfsInstance + } + + getDriveInstance() { + return driveInstance + } + + setDrive(drive: Drive) { + driveInstance = drive + } + + setBifrost(bifrost: BiFrost) { + bifrostInstance = bifrost + } + + setArFS(arfs: ArFS) { + arfsInstance = arfs + } +} + +const singletonArfs = Object.freeze(new ArFSSingleton()) +export default singletonArfs diff --git a/src/lib/arfs/getArFS.ts b/src/lib/arfs/getArFS.ts new file mode 100644 index 00000000..b073863d --- /dev/null +++ b/src/lib/arfs/getArFS.ts @@ -0,0 +1,7 @@ +import { ArFS } from 'arfs-js' + +export function getArFS() { + const arfs = new ArFS({ wallet: 'use_wallet' }) + + return arfs +} diff --git a/src/lib/arfs/getBifrost.ts b/src/lib/arfs/getBifrost.ts new file mode 100644 index 00000000..aeb9033b --- /dev/null +++ b/src/lib/arfs/getBifrost.ts @@ -0,0 +1,7 @@ +import { ArFS, BiFrost, Drive } from 'arfs-js' + +export function getBifrost(drive: Drive, arfs: ArFS) { + const bifrost = new BiFrost(drive, arfs) + + return bifrost +} diff --git a/src/lib/git/helpers/fsWithName.ts b/src/lib/git/helpers/fsWithName.ts index 81401fd0..fc27b79b 100644 --- a/src/lib/git/helpers/fsWithName.ts +++ b/src/lib/git/helpers/fsWithName.ts @@ -1,7 +1,13 @@ -import LightningFS from '@isomorphic-git/lightning-fs' +// import LightningFS from '@isomorphic-git/lightning-fs' + +import singletonArfs from '@/lib/arfs/arfsSingleton' export function fsWithName(name: string) { - return new LightningFS(name) + const bifrost = singletonArfs.getBifrostInstance() + + if (!bifrost) throw new Error('Bifrost uninitialized.') + console.log({ name }) + return bifrost.fs } -export type FSType = ReturnType \ No newline at end of file +export type FSType = ReturnType diff --git a/src/lib/git/repo.ts b/src/lib/git/repo.ts index e366ffce..3f89bcf4 100644 --- a/src/lib/git/repo.ts +++ b/src/lib/git/repo.ts @@ -33,64 +33,64 @@ const arweave = new Arweave({ protocol: 'https' }) -export async function postNewRepo({ id, title, description, file, owner, visibility }: any) { - const publicKey = await getActivePublicKey() +export async function postNewRepo({ id, dataTxId, title, description, visibility }: any) { + // const publicKey = await getActivePublicKey() const userSigner = await getSigner() - let data = (await toArrayBuffer(file)) as ArrayBuffer + // let data = (await toArrayBuffer(file)) as ArrayBuffer - const inputTags = [ - { name: 'App-Name', value: 'Protocol.Land' }, - { name: 'Content-Type', value: file.type }, - { name: 'Creator', value: owner }, - { name: 'Title', value: title }, - { name: 'Description', value: description }, - { name: 'Repo-Id', value: id }, - { name: 'Type', value: 'repo-create' }, - { name: 'Visibility', value: visibility } - ] as Tag[] + // const inputTags = [ + // { name: 'App-Name', value: 'Protocol.Land' }, + // { name: 'Content-Type', value: file.type }, + // { name: 'Creator', value: owner }, + // { name: 'Title', value: title }, + // { name: 'Description', value: description }, + // { name: 'Repo-Id', value: id }, + // { name: 'Type', value: 'repo-create' }, + // { name: 'Visibility', value: visibility } + // ] as Tag[] - let privateStateTxId = '' - if (visibility === 'private') { - const pubKeyArray = [strToJwkPubKey(publicKey)] - // Encrypt - const { aesKey, encryptedFile, iv } = await encryptFileWithAesGcm(data) - const encryptedAesKeysArray = await encryptAesKeyWithPublicKeys(aesKey, pubKeyArray) - // // Store 'encrypted', 'iv', and 'encryptedKeyArray' securely - - const privateState = { - version: '0.1', - iv, - encKeys: encryptedAesKeysArray, - pubKeys: [publicKey] - } + const privateStateTxId = '' + // if (visibility === 'private') { + // const pubKeyArray = [strToJwkPubKey(publicKey)] + // // Encrypt + // const { aesKey, encryptedFile, iv } = await encryptFileWithAesGcm(data) + // const encryptedAesKeysArray = await encryptAesKeyWithPublicKeys(aesKey, pubKeyArray) + // // // Store 'encrypted', 'iv', and 'encryptedKeyArray' securely - const privateInputTags = [ - { name: 'App-Name', value: 'Protocol.Land' }, - { name: 'Content-Type', value: 'application/json' }, - { name: 'Type', value: 'private-state' }, - { name: 'ID', value: id } - ] as Tag[] + // const privateState = { + // version: '0.1', + // iv, + // encKeys: encryptedAesKeysArray, + // pubKeys: [publicKey] + // } - const privateStateTxResponse = await signAndSendTx(JSON.stringify(privateState), privateInputTags, userSigner) + // const privateInputTags = [ + // { name: 'App-Name', value: 'Protocol.Land' }, + // { name: 'Content-Type', value: 'application/json' }, + // { name: 'Type', value: 'private-state' }, + // { name: 'ID', value: id } + // ] as Tag[] - if (!privateStateTxResponse) { - throw new Error('Failed to post Private State') - } + // const privateStateTxResponse = await signAndSendTx(JSON.stringify(privateState), privateInputTags, userSigner) - privateStateTxId = privateStateTxResponse + // if (!privateStateTxResponse) { + // throw new Error('Failed to post Private State') + // } - data = encryptedFile - } + // privateStateTxId = privateStateTxResponse - await waitFor(500) + // data = encryptedFile + // } - const dataTxResponse = await signAndSendTx(data, inputTags, userSigner, true) + // await waitFor(500) - if (!dataTxResponse) { - throw new Error('Failed to post Git repository') - } + // const dataTxResponse = await signAndSendTx(data, inputTags, userSigner, true) + + // if (!dataTxResponse) { + // throw new Error('Failed to post Git repository') + // } const contract = await getWarpContract(CONTRACT_TX_ID, userSigner) @@ -100,13 +100,13 @@ export async function postNewRepo({ id, title, description, file, owner, visibil id, name: title, description, - dataTxId: dataTxResponse, + dataTxId, visibility, privateStateTxId } }) - return { txResponse: dataTxResponse } + return { txResponse: id } } export async function updateGithubSync({ id, currentGithubSync, githubSync }: any) { @@ -398,9 +398,9 @@ export async function createNewRepo(title: string, fs: FSType, owner: string, id await waitFor(1000) - const repoBlob = await packGitRepo({ fs, dir }) + // const repoBlob = await packGitRepo({ fs, dir }) - return { repoBlob, commit: sha } + return { commit: sha } } catch (error) { console.error('failed to create repo') } diff --git a/src/pages/home/components/NewRepoModal.tsx b/src/pages/home/components/NewRepoModal.tsx index 4ddb6ca7..6bc323f5 100644 --- a/src/pages/home/components/NewRepoModal.tsx +++ b/src/pages/home/components/NewRepoModal.tsx @@ -6,7 +6,7 @@ import { useForm } from 'react-hook-form' import toast from 'react-hot-toast' import SVG from 'react-inlinesvg' import { useNavigate } from 'react-router-dom' -import { v4 as uuidv4 } from 'uuid' +// import { v4 as uuidv4 } from 'uuid' import * as yup from 'yup' import CloseCrossIcon from '@/assets/icons/close-cross.svg' @@ -14,8 +14,10 @@ import { Button } from '@/components/common/buttons' import CostEstimatesToolTip from '@/components/CostEstimatesToolTip' import { trackGoogleAnalyticsEvent } from '@/helpers/google-analytics' import { withAsync } from '@/helpers/withAsync' +import { getArFS } from '@/lib/arfs/getArFS' +import { getBifrost } from '@/lib/arfs/getBifrost' import { createNewRepo, postNewRepo } from '@/lib/git' -import { fsWithName } from '@/lib/git/helpers/fsWithName' +// import { fsWithName } from '@/lib/git/helpers/fsWithName' import { useGlobalStore } from '@/stores/globalStore' import { isRepositoryNameAvailable } from '@/stores/repository-core/actions/repoMeta' @@ -57,7 +59,8 @@ export default function NewRepoModal({ setIsOpen, isOpen }: NewRepoModalProps) { async function handleCreateBtnClick(data: yup.InferType) { setIsSubmitting(true) - const id = uuidv4() + const arfs = getArFS() + const { title, description } = data const owner = authState.address || 'Protocol.Land user' @@ -70,28 +73,26 @@ export default function NewRepoModal({ setIsOpen, isOpen }: NewRepoModalProps) { } try { - const fs = fsWithName(id) - const createdRepo = await createNewRepo(title, fs, owner, id) - - if (createdRepo && createdRepo.commit && createdRepo.repoBlob) { - const { repoBlob } = createdRepo + const drive = await arfs.drive.create(title) + const bifrost = getBifrost(drive, arfs) + const createdRepo = await createNewRepo(title, bifrost.fs, owner, drive.driveId!) + if (createdRepo && createdRepo.commit) { const result = await postNewRepo({ - id, + id: drive.driveId!, title, description, - file: repoBlob, - owner: authState.address, - visibility + visibility, + dataTxId: drive.id! }) if (result.txResponse) { trackGoogleAnalyticsEvent('Repository', 'Successfully created a repo', 'Create new repo', { - repo_id: id, + repo_id: drive.id!, repo_name: title }) - navigate(`/repository/${id}`) + navigate(`/repository/${drive.driveId!}`) } } } catch (error) { diff --git a/src/stores/repository-core/actions/git.ts b/src/stores/repository-core/actions/git.ts index 1d235b8e..6d675370 100644 --- a/src/stores/repository-core/actions/git.ts +++ b/src/stores/repository-core/actions/git.ts @@ -1,11 +1,14 @@ import Arweave from 'arweave' import toast from 'react-hot-toast' -import { getArrayBufSize } from '@/helpers/getArrayBufSize' +import { getRepoSize } from '@/helpers/getArrayBufSize' import { waitFor } from '@/helpers/waitFor' import { getActivePublicKey } from '@/helpers/wallet/getPublicKey' import { withAsync } from '@/helpers/withAsync' -import { importRepoFromBlob, unmountRepoFromBrowser } from '@/lib/git' +import singletonArfs from '@/lib/arfs/arfsSingleton' +import { getArFS } from '@/lib/arfs/getArFS' +import { getBifrost } from '@/lib/arfs/getBifrost' +import { unmountRepoFromBrowser } from '@/lib/git' import { getAllCommits } from '@/lib/git/commit' import { fsWithName } from '@/lib/git/helpers/fsWithName' import { getOidFromRef, readFileFromOid, readFilesFromOid } from '@/lib/git/helpers/oid' @@ -57,27 +60,33 @@ export async function saveRepository(id: string, name: string) { document.body.removeChild(downloadLink) } -export async function loadRepository(id: string, dataTxId: string, uploadStrategy: string, privateStateTxId?: string) { - await unmountRepository(id) +export async function loadRepository(id: string) { + const arfs = getArFS() - const gatewayUrl = uploadStrategy === 'ARSEEDING' ? 'https://arseed.web3infra.dev' : 'https://arweave.net' - const response = await fetch(`${gatewayUrl}/${dataTxId}`) - let repoArrayBuf = await response.arrayBuffer() + const drive = await arfs.drive.get(id) + const bifrost = getBifrost(drive!, arfs) + await bifrost.buildDriveState() + await waitFor(500) - if (privateStateTxId) { - repoArrayBuf = await decryptRepo(repoArrayBuf, privateStateTxId) - } + await bifrost.syncDrive() - const fs = fsWithName(id) - const dir = `/${id}` + singletonArfs.setArFS(arfs) + singletonArfs.setDrive(drive!) + singletonArfs.setBifrost(bifrost) - const repoSize = getArrayBufSize(repoArrayBuf) + let repoSize = 0 - const success = await importRepoFromBlob(fs, dir, new Blob([repoArrayBuf])) + if (bifrost.driveState) { + for (const entry in bifrost.driveState) { + const entity = bifrost.driveState[entry] - await waitFor(1000) + if (entity.entityType === 'folder' || !entity.size) continue + + repoSize += entity.size + } + } - return { success, repoSize } + return { success: true, repoSize: getRepoSize(repoSize) } } export async function renameRepoDir(id: string, currentName: string, newName: string) { diff --git a/tsconfig.json b/tsconfig.json index ef57f447..0b93dde2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,7 +5,7 @@ "lib": ["ES2020", "DOM", "DOM.Iterable"], "module": "ESNext", "skipLibCheck": true, - + "allowSyntheticDefaultImports": true, /* Bundler mode */ "moduleResolution": "Bundler", "allowImportingTsExtensions": true, diff --git a/yarn.lock b/yarn.lock index 570bd1f6..7b3a82e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2121,7 +2121,7 @@ base64-js "^1.5.1" bignumber.js "^9.1.1" -"@isomorphic-git/idb-keyval@3.3.2": +"@isomorphic-git/idb-keyval@3.3.2", "@isomorphic-git/idb-keyval@^3.3.2": version "3.3.2" resolved "https://registry.yarnpkg.com/@isomorphic-git/idb-keyval/-/idb-keyval-3.3.2.tgz#c0509a6c5987d8a62efb3e47f2815bcc5eda2489" integrity sha512-r8/AdpiS0/WJCNR/t/gsgL+M8NMVj/ek7s60uz3LmpCaTF2mEVlZJlB01ZzalgYzRLXwSPC92o+pdzjM7PN/pA== @@ -3195,6 +3195,11 @@ aproba@^1.0.3: resolved "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== +ar-gql@^1.2.9: + version "1.2.9" + resolved "https://registry.yarnpkg.com/ar-gql/-/ar-gql-1.2.9.tgz#423583dbebef7e8c838b7634d9d3ffb6d972aa3c" + integrity sha512-LZu4Mt92oFTA+JJ0PdiowJEFS2t6FhKWeYRBo9LTqx9shrIMGRb3iP5SvPbqLREXSSBriJOyyM3tgQ4wDYKr/w== + arbundles@^0.10.0: version "0.10.1" resolved "https://registry.npmjs.org/arbundles/-/arbundles-0.10.1.tgz#1f542d9edf185a8a272994aef501a8ee12aaaa46" @@ -3302,6 +3307,21 @@ are-we-there-yet@~1.1.2: delegates "^1.0.0" readable-stream "^2.0.6" +arfs-js@^1.2.6: + version "1.2.6" + resolved "https://registry.yarnpkg.com/arfs-js/-/arfs-js-1.2.6.tgz#bae7823da1514b2f2fec24f2188f2dc4ef97a3ed" + integrity sha512-G+9jejcPQKQMdNIq4FyBSwRgD9IGGUTB58f60mGYflqBjbkQ3P/CYz4ZR4CNYZTw3/MTMtpgzh49JNFLHBg81Q== + dependencies: + "@isomorphic-git/idb-keyval" "^3.3.2" + "@isomorphic-git/lightning-fs" "^4.6.0" + ar-gql "^1.2.9" + arweave "^1.14.4" + isomorphic-textencoder "^1.0.1" + just-debounce-it "^3.2.0" + uuid "^9.0.0" + warp-arbundles "^1.0.4" + warp-contracts-plugin-signature "^1.0.20" + arg@^5.0.2: version "5.0.2" resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c" @@ -6032,7 +6052,7 @@ isomorphic-git@^1.24.5: sha.js "^2.4.9" simple-get "^4.0.1" -isomorphic-textencoder@1.0.1: +isomorphic-textencoder@1.0.1, isomorphic-textencoder@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/isomorphic-textencoder/-/isomorphic-textencoder-1.0.1.tgz#38dcd3b4416d29cd33e274f64b99ae567cd15e83" integrity sha512-676hESgHullDdHDsj469hr+7t3i/neBKU9J7q1T4RHaWwLAsaQnywC0D1dIUId0YZ+JtVrShzuBk1soo0+GVcQ== @@ -6140,6 +6160,11 @@ just-debounce-it@1.1.0: resolved "https://registry.yarnpkg.com/just-debounce-it/-/just-debounce-it-1.1.0.tgz#8e92578effc155358a44f458c52ffbee66983bef" integrity sha512-87Nnc0qZKgBZuhFZjYVjSraic0x7zwjhaTMrCKlj0QYKH6lh0KbFzVnfu6LHan03NO7J8ygjeBeD0epejn5Zcg== +just-debounce-it@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/just-debounce-it/-/just-debounce-it-3.2.0.tgz#4352265f4af44188624ce9fdbc6bff4d49c63a80" + integrity sha512-WXzwLL0745uNuedrCsCs3rpmfD6DBaf7uuVwaq98/8dafURfgQaBsSpjiPp5+CW6Vjltwy9cOGI6qE71b3T8iQ== + just-once@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/just-once/-/just-once-1.1.0.tgz#fe81a185ebaeeb0947a7e705bf01cb6808db0ad8" From 6fff13173b2e01d9895e224b98baf0d5b001b735 Mon Sep 17 00:00:00 2001 From: Sai Kranthi Date: Tue, 7 May 2024 18:55:55 +0530 Subject: [PATCH 2/4] fix: types issue in PR store --- src/stores/pull-request/index.ts | 6 ++---- src/stores/repository-core/actions/git.ts | 1 + src/stores/repository-core/index.ts | 25 ++++------------------- 3 files changed, 7 insertions(+), 25 deletions(-) diff --git a/src/stores/pull-request/index.ts b/src/stores/pull-request/index.ts index 35c9e576..ba6e86f1 100644 --- a/src/stores/pull-request/index.ts +++ b/src/stores/pull-request/index.ts @@ -52,10 +52,8 @@ const createPullRequestSlice: StateCreator { - const { error, response } = await withAsync(() => - loadRepository(id, dataTxId, uploadStrategy, privateStateTxId) - ) + const loadRepoWithStatus = async ({ id }: Repo) => { + const { error, response } = await withAsync(() => loadRepository(id)) return !error && response && response.success ? { success: true } : { success: false } } diff --git a/src/stores/repository-core/actions/git.ts b/src/stores/repository-core/actions/git.ts index 6d675370..7e20e209 100644 --- a/src/stores/repository-core/actions/git.ts +++ b/src/stores/repository-core/actions/git.ts @@ -64,6 +64,7 @@ export async function loadRepository(id: string) { const arfs = getArFS() const drive = await arfs.drive.get(id) + const bifrost = getBifrost(drive!, arfs) await bifrost.buildDriveState() await waitFor(500) diff --git a/src/stores/repository-core/index.ts b/src/stores/repository-core/index.ts index 05936bbf..e48fb058 100644 --- a/src/stores/repository-core/index.ts +++ b/src/stores/repository-core/index.ts @@ -594,15 +594,7 @@ const createRepoCoreSlice: StateCreator - loadRepository(repoId, dataTxId, uploadStrategy, privateStateTxId) - ) + const { error: repoFetchError, response: repoFetchResponse } = await withAsync(() => loadRepository(repoId)) if (fork && parentRepoId && repoId !== parentRepoId) { const renamed = await renameRepoDir(repoId, parentRepoId, repoId) @@ -695,9 +685,7 @@ const createRepoCoreSlice: StateCreator - loadRepository(repo.id, repo.dataTxId, repo.uploadStrategy, repo.privateStateTxId) - ) + const { error: repoFetchError, response: repoFetchResponse } = await withAsync(() => loadRepository(repo.id)) if (repoFetchError) { set((state) => { @@ -729,12 +717,7 @@ const createRepoCoreSlice: StateCreator - loadRepository( - metaResponse.result.id, - metaResponse.result.dataTxId, - metaResponse.result.uploadStrategy, - metaResponse.result.privateStateTxId - ) + loadRepository(metaResponse.result.id) ) if (repoFetchError) { From 5d89aa2e57e3b474f6e98851f6b7676fe661696f Mon Sep 17 00:00:00 2001 From: Sai Kranthi Date: Thu, 9 May 2024 19:23:37 +0530 Subject: [PATCH 3/4] chore: bump arfs --- package.json | 2 +- src/lib/arfs/getArFS.ts | 2 +- yarn.lock | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 72e04d78..6993c4c5 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@uiw/react-codemirror": "^4.21.11", "@uiw/react-md-editor": "^3.23.5", "ardb": "^1.1.10", - "arfs-js": "^1.2.6", + "arfs-js": "^1.2.7", "arweave": "^1.14.4", "clsx": "^2.0.0", "date-fns": "^2.30.0", diff --git a/src/lib/arfs/getArFS.ts b/src/lib/arfs/getArFS.ts index b073863d..eb22ac0b 100644 --- a/src/lib/arfs/getArFS.ts +++ b/src/lib/arfs/getArFS.ts @@ -1,7 +1,7 @@ import { ArFS } from 'arfs-js' export function getArFS() { - const arfs = new ArFS({ wallet: 'use_wallet' }) + const arfs = new ArFS({ wallet: 'use_wallet', appName: 'Protocol.Land' }) return arfs } diff --git a/yarn.lock b/yarn.lock index 7b3a82e8..9a0adf5b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3307,10 +3307,10 @@ are-we-there-yet@~1.1.2: delegates "^1.0.0" readable-stream "^2.0.6" -arfs-js@^1.2.6: - version "1.2.6" - resolved "https://registry.yarnpkg.com/arfs-js/-/arfs-js-1.2.6.tgz#bae7823da1514b2f2fec24f2188f2dc4ef97a3ed" - integrity sha512-G+9jejcPQKQMdNIq4FyBSwRgD9IGGUTB58f60mGYflqBjbkQ3P/CYz4ZR4CNYZTw3/MTMtpgzh49JNFLHBg81Q== +arfs-js@^1.2.7: + version "1.2.7" + resolved "https://registry.yarnpkg.com/arfs-js/-/arfs-js-1.2.7.tgz#3dce0e900b27bcc64b5cc4a96d4f00283a48c0c1" + integrity sha512-xmQbaclTulyJYN8C3CD/mPeAfu4Ja3pyP4xbQHPKYYII9jnAGZQhGj9FBVQn6U4FCGtd6vdsikqb66NfYKq58A== dependencies: "@isomorphic-git/idb-keyval" "^3.3.2" "@isomorphic-git/lightning-fs" "^4.6.0" From 7a61601fd23f27e029d8e545bd45399545b9280d Mon Sep 17 00:00:00 2001 From: Sai Kranthi Date: Thu, 30 May 2024 18:52:23 +0530 Subject: [PATCH 4/4] feat: use queue to collect tx and bundle them --- src/lib/arfs/arfsTxSubmissionOverride.ts | 24 ++++ src/lib/arfs/getArFS.ts | 3 + src/lib/queue/BaseQueue.ts | 52 +++++++++ src/lib/queue/PendingQueue.ts | 14 +++ src/lib/queue/QueueObserver.ts | 3 + src/lib/queue/TaskQueue.ts | 122 +++++++++++++++++++++ src/lib/queue/helpers.ts | 22 ++++ src/pages/home/components/NewRepoModal.tsx | 10 +- src/pages/repository/hooks/useCommit.ts | 13 +-- 9 files changed, 254 insertions(+), 9 deletions(-) create mode 100644 src/lib/arfs/arfsTxSubmissionOverride.ts create mode 100644 src/lib/queue/BaseQueue.ts create mode 100644 src/lib/queue/PendingQueue.ts create mode 100644 src/lib/queue/QueueObserver.ts create mode 100644 src/lib/queue/TaskQueue.ts create mode 100644 src/lib/queue/helpers.ts diff --git a/src/lib/arfs/arfsTxSubmissionOverride.ts b/src/lib/arfs/arfsTxSubmissionOverride.ts new file mode 100644 index 00000000..4aab8f62 --- /dev/null +++ b/src/lib/arfs/arfsTxSubmissionOverride.ts @@ -0,0 +1,24 @@ +import Transaction from 'arweave/web/lib/transaction' +import { v4 as uuidv4 } from 'uuid' + +import { createSignedQueuePayload } from '../queue/helpers' +import taskQueueSingleton from '../queue/TaskQueue' + +export async function arfsTxSubmissionOverride(txList: Transaction[]) { + const queueStatus = taskQueueSingleton.getTaskQueueStatus() + const txIds: string[] = [] + + if (queueStatus === 'Busy') throw new Error('Task Queue is busy. Try again later.') + + for (const tx of txList) { + const dataItem = await createSignedQueuePayload(tx) + + const token = uuidv4() + taskQueueSingleton.sendToPending(token, dataItem) + + const txid = await dataItem.id + txIds.push(txid) + } + + return { successTxIds: txIds, failedTxIndex: [] } +} diff --git a/src/lib/arfs/getArFS.ts b/src/lib/arfs/getArFS.ts index eb22ac0b..81461aac 100644 --- a/src/lib/arfs/getArFS.ts +++ b/src/lib/arfs/getArFS.ts @@ -1,7 +1,10 @@ import { ArFS } from 'arfs-js' +import { arfsTxSubmissionOverride } from './arfsTxSubmissionOverride' + export function getArFS() { const arfs = new ArFS({ wallet: 'use_wallet', appName: 'Protocol.Land' }) + arfs.api.signAndSendAllTransactions = arfsTxSubmissionOverride return arfs } diff --git a/src/lib/queue/BaseQueue.ts b/src/lib/queue/BaseQueue.ts new file mode 100644 index 00000000..823afd0c --- /dev/null +++ b/src/lib/queue/BaseQueue.ts @@ -0,0 +1,52 @@ +import { QueueObserver } from './QueueObserver' + +export class BaseQueue { + public list: Array<{ token: string; payload: T }> = [] + protected addedList: Record = {} + private observers: QueueObserver[] = [] + + constructor() {} + + public enqueue(token: string, payload: T) { + if (!this.isInList(token)) { + this.list.unshift({ token, payload }) + this.addedList[token] = token + } + } + + public dequeue(token?: string) { + // you can pass token for removing specific item + if (token) { + const itemShouldRemove = this.list.find((item) => item.token === token) + this.list = this.list.filter((item) => item.token !== token) + + delete this.addedList[token] + + return itemShouldRemove + } else { + const item = this.list.pop() + + if (item) delete this.addedList[item?.token] + + return item + } + } + + public isInList(token: string) { + return token in this.addedList + } + + public addObserver(observer: QueueObserver) { + this.observers.push(observer) + } + + public notifyObservers(item: { token: string; payload: T }) { + for (const observer of this.observers) { + observer.update(item) + } + } + + public getList() { + return this.list + } +} diff --git a/src/lib/queue/PendingQueue.ts b/src/lib/queue/PendingQueue.ts new file mode 100644 index 00000000..e9238413 --- /dev/null +++ b/src/lib/queue/PendingQueue.ts @@ -0,0 +1,14 @@ +import Transaction from 'arweave/web/lib/transaction' +import { DataItem } from 'warp-arbundles' + +import { BaseQueue } from './BaseQueue' +import { QueueObserver } from './QueueObserver' + +export class PendingQueue extends BaseQueue {} + +export class PendingObserver implements QueueObserver { + update(item: { token: string; payload: Transaction | DataItem }) { + console.log({ item }, "<-- added to queue") + //check for progress queue length and move items from pending to progress + } +} diff --git a/src/lib/queue/QueueObserver.ts b/src/lib/queue/QueueObserver.ts new file mode 100644 index 00000000..5d53d30e --- /dev/null +++ b/src/lib/queue/QueueObserver.ts @@ -0,0 +1,3 @@ +export abstract class QueueObserver { + abstract update(item: { token: string; payload: T }): void +} diff --git a/src/lib/queue/TaskQueue.ts b/src/lib/queue/TaskQueue.ts new file mode 100644 index 00000000..202ec8d3 --- /dev/null +++ b/src/lib/queue/TaskQueue.ts @@ -0,0 +1,122 @@ +import Transaction from 'arweave/web/lib/transaction' +import axios from 'axios' +import { DataItem } from 'warp-arbundles' + +import { useGlobalStore } from '@/stores/globalStore' + +import { bundleAndSignData } from '../subsidize/utils' +import { PendingObserver, PendingQueue } from './PendingQueue' + +export const MAX_LENGTH_PROGRESS_QUEUE = 5 + +let instance: TaskQueueSingleton +let taskQueueStatus: 'Busy' | 'Idle' = 'Idle' +export class TaskQueueSingleton { + pendingQueue = new PendingQueue() + + constructor() { + if (instance) { + throw new Error('You can only create one instance!') + } + + this.pendingQueue.addObserver(new PendingObserver()) + // eslint-disable-next-line @typescript-eslint/no-this-alias + instance = this + } + + getInstance() { + return this + } + + getTaskQueueStatus() { + return taskQueueStatus + } + + setTaskQueueStatus(status: 'Busy' | 'Idle') { + taskQueueStatus = status + } + + sendToPending(token: string, tx: Transaction | DataItem) { + this.pendingQueue.enqueue(token, tx) + this.pendingQueue.notifyObservers({ token, payload: tx }) + } + + getPending() { + return this.pendingQueue.getList() + } + + async execute(driveId: string) { + if (taskQueueStatus === 'Busy') { + //toast message notify that new batch cant be run + return [] + } + + taskQueueStatus = 'Busy' + + const dataItems = [] + const ids = [] + let bundle: Awaited> | null = null + + try { + while (this.pendingQueue.getList().length > 0) { + const item = this.pendingQueue.dequeue() + + if (item) { + const { payload } = item + + dataItems.push(payload as DataItem) + ids.push(await payload.id) + } + } + + bundle = await bundleAndSignData(dataItems, () => console.log('error: unsigned data items.')) + } catch (error) { + taskQueueStatus = 'Idle' + + return [] + } + + const userAddress = useGlobalStore.getState().authState.address + if (!bundle || !userAddress) { + taskQueueStatus = 'Idle' + + return [] + } + + try { + const dataBinary = bundle.getRaw() + const res = ( + await axios.post( + 'https://bundle.saikranthi.dev/api/v1/postrepo', + { + txBundle: dataBinary, + platform: 'UI', + owner: userAddress!, + driveId + }, + { + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json' + } + } + ) + ).data + + if (!res || !res.success) { + throw new Error('Failed to subsidize your transaction. Please try again.') + } + + return ids + } catch (error: any) { + taskQueueStatus = 'Idle' + + return [] + } finally { + taskQueueStatus = 'Idle' + } + } +} + +const taskQueueSingleton = Object.freeze(new TaskQueueSingleton()) +export default taskQueueSingleton diff --git a/src/lib/queue/helpers.ts b/src/lib/queue/helpers.ts new file mode 100644 index 00000000..accc92d4 --- /dev/null +++ b/src/lib/queue/helpers.ts @@ -0,0 +1,22 @@ +import Transaction, { Tag } from 'arweave/web/lib/transaction' +import { DataItem } from 'warp-arbundles' + +import { createAndSignDataItem } from '@/helpers/wallet/createAndSignDataItem' +import { getSigner } from '@/helpers/wallet/getSigner' + +export async function createSignedQueuePayload(tx: Transaction | DataItem) { + const data = tx.data + let tags = tx.tags as Tag[] + + tags = tags.map((tag) => { + const name = tag.get('name', { decode: true, string: true }) + const value = tag.get('value', { decode: true, string: true }) + + return { name, value } + }) as Tag[] + + const signer = await getSigner() + const dataItem = await createAndSignDataItem(data, tags, signer) + + return dataItem +} \ No newline at end of file diff --git a/src/pages/home/components/NewRepoModal.tsx b/src/pages/home/components/NewRepoModal.tsx index 6bc323f5..69c49b3e 100644 --- a/src/pages/home/components/NewRepoModal.tsx +++ b/src/pages/home/components/NewRepoModal.tsx @@ -17,6 +17,7 @@ import { withAsync } from '@/helpers/withAsync' import { getArFS } from '@/lib/arfs/getArFS' import { getBifrost } from '@/lib/arfs/getBifrost' import { createNewRepo, postNewRepo } from '@/lib/git' +import taskQueueSingleton from '@/lib/queue/TaskQueue' // import { fsWithName } from '@/lib/git/helpers/fsWithName' import { useGlobalStore } from '@/stores/globalStore' import { isRepositoryNameAvailable } from '@/stores/repository-core/actions/repoMeta' @@ -76,8 +77,15 @@ export default function NewRepoModal({ setIsOpen, isOpen }: NewRepoModalProps) { const drive = await arfs.drive.create(title) const bifrost = getBifrost(drive, arfs) const createdRepo = await createNewRepo(title, bifrost.fs, owner, drive.driveId!) + const taskQueueItemsLength = taskQueueSingleton.getPending().length + + if (createdRepo && createdRepo.commit && taskQueueItemsLength > 0) { + const uploadedToArFS = await taskQueueSingleton.execute(drive.id!) + + if (uploadedToArFS.length !== taskQueueItemsLength) { + throw new Error('Failed to upload.') + } - if (createdRepo && createdRepo.commit) { const result = await postNewRepo({ id: drive.driveId!, title, diff --git a/src/pages/repository/hooks/useCommit.ts b/src/pages/repository/hooks/useCommit.ts index 255bf754..609f0073 100644 --- a/src/pages/repository/hooks/useCommit.ts +++ b/src/pages/repository/hooks/useCommit.ts @@ -4,7 +4,6 @@ import toast from 'react-hot-toast' import { trackGoogleAnalyticsEvent } from '@/helpers/google-analytics' import { withAsync } from '@/helpers/withAsync' -import { postUpdatedRepo } from '@/lib/git' import { getCurrentBranch } from '@/lib/git/branch' import { addFilesForCommit, @@ -15,6 +14,7 @@ import { stageFilesForCommit } from '@/lib/git/commit' import { fsWithName } from '@/lib/git/helpers/fsWithName' +import taskQueueSingleton from '@/lib/queue/TaskQueue' import { postCommitStatDataTxToArweave } from '@/lib/user' import { useGlobalStore } from '@/stores/globalStore' import { CommitResult } from '@/types/commit' @@ -31,8 +31,7 @@ type AddFilesOptions = { } export default function useCommit() { - const [selectedRepo, repoCommitsG, setRepoCommitsG, triggerGithubSync] = useGlobalStore((state) => [ - state.repoCoreState.selectedRepo, + const [repoCommitsG, setRepoCommitsG, triggerGithubSync] = useGlobalStore((state) => [ state.repoCoreState.git.commits, state.repoCoreActions.git.setCommits, state.repoCoreActions.triggerGithubSync @@ -85,11 +84,9 @@ export default function useCommit() { if (commitError || !commitSHA) throw trackAndThrowError('Failed to commit files', name, id) - const isPrivate = selectedRepo.repo?.private || false - const privateStateTxId = selectedRepo.repo?.privateStateTxId - const { error, response } = await withAsync(() => - postUpdatedRepo({ fs, dir, owner, id, isPrivate, privateStateTxId }) - ) + // const isPrivate = selectedRepo.repo?.private || false + // const privateStateTxId = selectedRepo.repo?.privateStateTxId + const { error, response } = await withAsync(() => taskQueueSingleton.execute(id)) if (error) throw trackAndThrowError('Failed to update repository', name, id)