Merge branch 'main' of github.com:andrewhertog/codex-editor into feat/ci

src/activationHelpers/contextAware/miniIndex/indexes/translationPairsIndex.ts

@@ -5,6 +5,8 @@ import { StatusBarHandler } from '../statusBarHandler';
 import * as fs from 'fs';
 import * as path from 'path';
 import { getWorkSpaceFolder } from '../../../../utils';
+import { getCodexCells, cleanCodexCell } from '../../../../utils/verseRefUtils/verseCleanup';
+
 export interface minisearchDoc {
     id: string;
     vref: string;
@@ -41,10 +43,10 @@ export async function createTranslationPairsIndex(context: vscode.ExtensionConte
         const completeDrafts: string[] = [];
 
         for (const file of targetBibleFiles) {
-            const document = await vscode.workspace.openNotebookDocument(file);
-            const cells = document.getCells();
+            const cells = await getCodexCells(file.fsPath);
             for (const cell of cells) {
-                const lines = cell.document.getText().split('\n');
+                const cleanedCell = cleanCodexCell(cell);
+                const lines = cleanedCell.split('\n');
                 for (const line of lines) {
                     const match = line.match(verseRefRegex);
                     if (match) {

src/activationHelpers/contextAware/miniIndex/indexes/wordsIndex.ts

@@ -1,7 +1,7 @@
 import * as vscode from 'vscode';
 import * as path from 'path';
-import { cleanWord } from '../../../../utils/spellingUtils';
 import { FileHandler } from '../../../../providers/dictionaryTable/utilities/FileHandler';
+import { cleanWord } from '../../../../utils/cleaningUtils';
 
 interface WordFrequency {
     word: string;

src/extension.ts

@@ -21,8 +21,6 @@ import {
 import { createIndexWithContext } from "./activationHelpers/contextAware/miniIndex/indexes/index";
 import { initializeWebviews } from "./activationHelpers/contextAware/webviewInitializers";
 import { syncUtils } from "./activationHelpers/contextAware/syncUtils";
-import { initializeStateStore } from "./stateStore";
-import { projectFileExists } from "./utils/fileUtils";
 import { registerCompletionsCodeLensProviders } from "./activationHelpers/contextAware/completionsCodeLensProviders";
 import { CodexChunkEditorProvider } from "./providers/codexChunkEditorProvider/CodexChunkEditorProvider";
 import * as path from "path";

src/tsServer/spellCheck.ts

@@ -18,7 +18,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { URI } from 'vscode-uri';
 import { Connection } from 'vscode-languageserver/node';
-import { cleanWord } from '../utils/spellingUtils';
+import { cleanWord } from '../utils/cleaningUtils';
 
 export class SpellChecker {
     private dictionary: Dictionary | null = null;

src/utils/spellingUtils.ts → src/utils/cleaningUtils.ts

@@ -1,4 +1,4 @@
-export function cleanWord(word: string | undefined | null): string {
+export function cleanWord(word: string | undefined | null): string { // this is for the spellchecker
     if (!word) return '';
     return word
         // Remove non-alphanumeric characters from start and end
@@ -9,4 +9,4 @@ export function cleanWord(word: string | undefined | null): string {
         .replace(/(?<!\S)'|'(?!\S)|[^\p{L}\p{N}'\s]/gu, '')
         // Convert to lowercase
         .toLowerCase();
-}
+}

src/utils/verseRefUtils/verseCleanup.ts

@@ -0,0 +1,17 @@
+import * as fs from 'fs/promises';
+
+export async function getCodexCells(filePath: string): Promise<string[]> {
+    const fileContent = await fs.readFile(filePath, 'utf-8');
+    const cells = JSON.parse(fileContent).cells;
+    return cells.filter((cell: any) => cell.kind === 2 && cell.language === 'scripture')
+                .map((cell: any) => cell.value);
+}
+
+export function getCleanedCell(cellContent: string): string {
+    // Remove HTML tags while preserving newlines
+    return cellContent.replace(/<[^>]*>/g, '');
+}
+
+export function cleanCodexCell(cellContent: string): string {
+    return getCleanedCell(cellContent);
+}