working on downloading repo

README.md

@@ -1,6 +1,6 @@
 # Checking Tool
 
-This is an implementation of a [React](https://reactjs.org/) + [Vite](https://vitejs.dev/) + [Webview UI Toolkit](https://github.com/microsoft/vscode-webview-ui-toolkit) webview extension.
+This is an implementation of a [React](https://reactjs.org/) + [Vite](https://vitejs.dev/) + [Webview UI Toolkit](https://github.com/microsoft/vscode-webview-ui-toolkit) vs-code webview extension.
 
 This extension will use a custom editor to open files with the `twl_check` or `tn_check` extension and display a UI to do checking for book 
 

package.json

@@ -185,7 +185,7 @@
     "original-fs": "^1.2.0",
     "ospath": "^1.2.2",
     "string-punctuation-tokenizer": "^2.2.0",
-    "tc-source-content-updater": "1.4.30",
+    "tc-source-content-updater": "1.4.31-beta",
     "tsv-groupdata-parser": "^1.1.1",
     "typescript": "^5.4.5",
     "usfm-js": "^3.4.2",

src/test/dcs.test.ts

@@ -20,7 +20,7 @@ import {
 import {
   createRepoBranch,
   createCheckingRepository,
-  downloadRepoFromBranch,
+  downloadPublicRepoFromBranch,
   getRepoName,
   getRepoTree,
   uploadRepoFileFromPath,
@@ -96,7 +96,7 @@ suite.skip('Repo Tests', async ()=> {
   test('Test downloadRepo', async () => {
     const branch = 'master'
     const repo = env.REPO || ''
-    const results = await downloadRepoFromBranch(testRepoPath, server, owner, repo, branch, token)
+    const results = await downloadPublicRepoFromBranch(testRepoPath, server, owner, repo, branch)
     assert.ok(!results.error)
   })
 

src/utilities/network.ts

@@ -3,6 +3,7 @@ import {
   checkDataToTn,
   checkDataToTwl,
   convertJsonToUSFM,
+  fetchFileFromRepo,
   flattenGroupData,
   getBibleBookFolders,
   getMetaData,
@@ -40,14 +41,14 @@ interface RepoResponseItem {
   error: undefined|string
 }
 
-export async function getRepoTree(server: string, owner: string, repo: string, sha: string, token: string): Promise<RepoResponseItem> {
+export async function getRepoTree(server: string, owner: string, repo: string, sha: string, token: string = ''): Promise<RepoResponseItem> {
   const url = `${server}/api/v1/repos/${owner}/${repo}/git/trees/${sha}?recursive=true`;
   try {
-    const response = await axios.get(url, {
-      headers: {
-        'Authorization': `token ${token}`
-      }
-    });
+    const headers = token ? 
+    {
+      "Authorization": `token ${token}`,
+    } : {};
+    const response = await axios.get(url, { headers });
     return response.data;
   } catch (error) {
     // @ts-ignore
@@ -1159,9 +1160,9 @@ async function updateFilesInBranch(localFiles: string[], localRepoPath: string,
   return { changedFiles }
 }
 
-export async function downloadRepoFromBranch(localRepoPath: string, server: string, owner: string, repo: string, branch: string, token: string): Promise<GeneralObject> {
+export async function downloadPublicRepoFromBranch(localRepoPath: string, server: string, owner: string, repo: string, branch: string): Promise<GeneralObject> {
   fs.ensureDirSync(localRepoPath)
-  const treeResults = await getRepoTree(server, owner, repo, 'master', token);
+  const treeResults = await getRepoTree(server, owner, repo, 'master');
   if (treeResults.error) {
     return treeResults
   }
@@ -1174,7 +1175,7 @@ export async function downloadRepoFromBranch(localRepoPath: string, server: stri
   }
 
   const commit = treeResults.sha || '';
-  const results = await downloadFilesInBranch(localRepoPath, dcsFiles, server, owner, repo, commit, token)
+  const results = await downloadFilesInBranch(localRepoPath, dcsFiles, server, owner, repo, branch)
 
   const newStatus = {
     files: dcsFiles,
@@ -1187,15 +1188,15 @@ export async function downloadRepoFromBranch(localRepoPath: string, server: stri
   return results
 }
 
-async function downloadFilesInBranch(localRepoPath: string, dcsFiles: NestedObject, server: string, owner: string, repo: string, branch: string, token: string): Promise<GeneralObject> {
+async function downloadFilesInBranch(localRepoPath: string, dcsFiles: NestedObject, server: string, owner: string, repo: string, branch: string): Promise<GeneralObject> {
   let changedFiles = 0;
   const files = Object.keys(dcsFiles)
   for (const fileName of files) {
     if ((fileName.includes(dcsStatusFile)) || (fileName === ".DS_Store")) { // skip over DCS data file, and system files
       continue;
     }
-
     const fullFilePath = path.join(localRepoPath, fileName);
+
     const dcsFileData = dcsFiles[fileName];
     let localChecksum: string = "";
 
@@ -1204,14 +1205,14 @@ async function downloadFilesInBranch(localRepoPath: string, dcsFiles: NestedObje
 
     if (fileType === "file" || fileType === "blob") {
       console.log(`downloadFilesInBranch - downloading file ${fileName}`);
-      results = await getFileFromBranch(server, owner, repo, branch, fileName, token);
+      const results = await fetchFileFromRepo(server, owner, repo, branch, localRepoPath, fileName);
+
+      // using API - probably slowing using base46 encoding
+      // results = await getFileFromBranch(server, owner, repo, branch, fileName, token);
 
       changedFiles++
 
       if (!results?.error) {
-        // @ts-ignore
-        const fileData = results.content;
-        fs.outputFileSync(fullFilePath, fileData, 'UTF-8');
         localChecksum = await getChecksum(fullFilePath);
         dcsFileData.checksum = localChecksum
         // @ts-ignore

src/utilities/resourceUtils.ts

@@ -34,6 +34,7 @@ import {
 import * as tsvparser from "uw-tsv-parser";
 // @ts-ignore
 import * as tsvGroupdataParser from "tsv-groupdata-parser"
+import { request } from "node:http";
 
 // helpers
 const {
@@ -68,6 +69,93 @@ const RESOURCE_ID_MAP = {
 const checkingHelpsResources = [
     { id:'ta' }, { id:'tw' }, { id:'twl', bookRes: true }, { id: 'tn', bookRes: true }]
 
+/**
+ * does http request and returns the response data parsed from JSON
+ * @param {string} url
+ * @param {number} retries
+ * @return {Promise<{Object}>}
+ */
+export async function makeJsonRequestDetailed(url:string, retries= 5): Promise<any> {
+    let result_;
+    for (let i = 1; i <= retries; i++) {
+        result_ = null;
+        try {
+            result_ = await new Promise((resolve, reject) => {
+                // @ts-ignore
+                request(url, function(error: any, response: any, body: string) {
+                    if (error)
+                        reject(error);
+                    else if (response.statusCode === 200) {
+                        let result = body;
+                        try {
+                            result = JSON.parse(body);
+                        } catch (e) {
+                            reject(e);
+                        }
+                        resolve({result, response, body});
+                    } else {
+                        reject(`fetch error ${response.statusCode}`);
+                    }
+                });
+            });
+        } catch (e) {
+            if (i >= retries) {
+                console.warn(`makeJsonRequestDetailed(${url}) - error getting data`, e);
+                throw e;
+            }
+            result_ = null;
+            console.log(`makeJsonRequestDetailed(${url}) - retry ${i+1} getting data, last error`, e);
+            await delay(500);
+        }
+
+        if (result_) {
+            break;
+        }
+    }
+    return result_;
+}
+
+/**
+ * does specific page query and returns the response data parsed from JSON
+ * @param {string} url
+ * @param {number} page
+ * @param {number} retries
+ * @return {Promise<{Object}>}
+ */
+export async function doMultipartQueryPage(url:string, page = 1, retries = 5):Promise<any> {
+    const url_ = `${url}&page=${page}`;
+    const {result, response} = await makeJsonRequestDetailed(url_, retries);
+    const pos = response && response.rawHeaders && response.rawHeaders.indexOf('X-Total-Count');
+    const totalCount = (pos >= 0) ? parseInt(response.rawHeaders[pos + 1]) : 0;
+    const items = result && result.data || null;
+    return {items, totalCount};
+}
+
+/**
+ * does multipart query and returns the response data parsed from JSON. Continues to read pages until all results are returned.
+ * @param {string} url
+ * @param {number} retries
+ * @return {Promise<{Object}>}
+ */
+export async function doMultipartQuery(url:string, retries = 5) {
+    let page = 1;
+    let data:any[] = [];
+    const {items, totalCount} = await doMultipartQueryPage(url, page, retries = 5);
+    let lastItems = items;
+    let totalCount_ = totalCount;
+    data = data.concat(items);
+    while (lastItems && data.length < totalCount_) {
+        const {items, totalCount} = await doMultipartQueryPage(url, ++page, retries = 5);
+        lastItems = items;
+        totalCount_ = totalCount;
+        if (items && items.length) {
+            data = data.concat(items);
+        }
+    }
+
+    return data;
+}
+
 /**
  * merge helps folder into single json file
  * @param {object} resource - current resource
@@ -401,8 +489,41 @@ function getDestFolderForRepoFile(resourcesPath: string, languageId: string, res
     return destFolder;
 }
 
-async function fetchRepoFile(masterBranchUrl: string, repoFilePath: string, resourcesPath: string, languageId: string, resourceId: string, bookId: string, version: string, owner: string) {
-    let downloadUrl = masterBranchUrl + repoFilePath;
+export async function fetchFileFromRepo(server: string, owner: string, repo: string, branch: string, localRepoPath: string, fileName: string) {
+    const baseUrl = `${server}/${owner}/${repo}/raw/branch/${branch}`;
+    const results = await fetchFileFromUrl(baseUrl, localRepoPath, fileName);
+    return results;
+}
+
+export async function fetchFileFromUrl(baseUrl: string, repoFilePath: string, filePath: string) {
+    const _filePath = filePath?.replace('./', '')
+    const downloadUrl = `${baseUrl}/${_filePath}`
+    const destFilePath = path.join(repoFilePath, _filePath)
+    const destFolder = path.dirname(destFilePath)
+    fs.ensureDirSync(destFolder)
+    let error:string = ''
+
+    try {
+        const results = await downloadHelpers.download(downloadUrl, destFilePath);
+        if (results.status === 200) {
+            return { 
+                success: true
+            }
+        }
+        error = `fetchFileFromUrl(${downloadUrl}) - returned status ${results.status}`
+        console.warn(`fetchFileFromUrl(${downloadUrl}) - returned status ${results.status}`)
+    } catch (e) {
+        error = `fetchFileFromUrl(${downloadUrl}) - failed`
+        console.warn(`fetchFileFromUrl(${downloadUrl}) - failed`)
+    }
+    return { 
+        success: false,
+        error
+    }
+}
+
+async function fetchRepoFile(baseBranchUrl: string, repoFilePath: string, resourcesPath: string, languageId: string, resourceId: string, bookId: string, version: string, owner: string) {
+    let downloadUrl = baseBranchUrl + repoFilePath;
     const destFolder = getDestFolderForRepoFile(resourcesPath, languageId, resourceId, bookId, version, owner);
     let destFilePath: string | null = path.join(destFolder, repoFilePath);
     let contents = ''
@@ -485,6 +606,14 @@ async function fetchBibleResourceBook(catalog:any[], languageId:string, owner:st
                     contents: bookContents,
                     filePath: bookFilePath
                 } = await fetchRepoFile(rawUrl, bookPath, resourcesPath, languageId, resourceId, bookId, version, owner);
+
+                try {
+                    const bookPath = 'LICENSE.md'
+                    await fetchRepoFile(rawUrl, bookPath, resourcesPath, languageId, resourceId, bookId, version, owner);
+                } catch (e) {
+                    console.warn(`fetchBibleResourceBook - could not download license file from ${path.join(rawUrl)}`)
+                }
+
                 if (bookContents && bookFilePath) {
                     return destFolder;
                 }
@@ -1531,6 +1660,12 @@ export async function initProject(repoPath:string, targetLanguageId:string, targ
                                 // copy manifest
                                 const manifest = getResourceManifest(targetFoundPath)
                                 fs.outputJsonSync(path.join(targetPath, 'manifest.json'), manifest, { spaces: 2 });
+                                const fileName = 'LICENSE.md';
+                                const sourcePath = path.join(targetFoundPath, fileName);
+                                if (fs.existsSync(sourcePath)) {
+                                    fs.copySync(sourcePath, path.join(targetPath, fileName))
+                                    fs.copySync(sourcePath, path.join(repoPath, fileName))
+                                }
                             }
                         }