Merge branch 'wordplaydev:main' into 474-chinese-simplified

CHANGELOG.md

@@ -3,7 +3,14 @@
 We'll note all notable changes in this file, including bug fixes, enhancements, and all closed issues.
 Dates are in `YYYY-MM-DD` format and versions are in [semantic versioning](http://semver.org/) format.
 
-## 0.10.4 2024-7-08
+## 0.10.5 2024-07-13
+
+### Fixed
+
+-   [#514](https://github.com/wordplaydev/wordplay/issues/514) Fixed cursor position on hidden language tags.
+-   Fixed parsing bug that prevented complete parsing of the program.
+
+## 0.10.4 2024-07-08
 
 ### Fixed
 

src/components/editor/CaretView.svelte

@@ -457,10 +457,21 @@
             const editorVerticalStart = editorPadding + 4;
 
             // Find the right side of token just prior to the current one that has this space.
-            const priorToken = caret.source.getNextToken(token, -1);
-            const priorTokenView = priorToken
-                ? getNodeView(priorToken)
-                : undefined;
+            let priorToken: Token | undefined = token;
+            let priorTokenView: Element | null = null;
+            do {
+                priorToken = caret.source.getNextToken(priorToken, -1);
+                priorTokenView = priorToken ? getNodeView(priorToken) : null;
+                // We need to make sure the prior token is visible. If we found a visible one,
+                // then stop and compute based on that position.
+                if (
+                    priorToken === undefined ||
+                    (priorTokenView !== null &&
+                        !priorTokenView.classList.contains('hide'))
+                )
+                    break;
+            } while (true);
+
             const priorTokenViewRect = priorTokenView?.getBoundingClientRect();
             let priorTokenHorizontalEnd =
                 priorTokenViewRect === undefined

src/components/project/RootView.svelte

@@ -67,7 +67,7 @@
     $: {
         const newHidden = new Set<Node>();
 
-        if ($localize) {
+        if ($localize !== 'actual') {
             // Hide any language tagged nodes that 1) the caret isn't in, and 2) either have no language tag or aren't one of the selected tags.
             // Also hide any name separators if the first visible name has one.
             for (const tagged of node

src/components/widgets/Toggle.svelte

@@ -57,6 +57,7 @@
         user-select: none;
         border: none;
         border-radius: var(--wordplay-border-radius);
+        background: var(--wordplay-alternating-color);
         color: currentColor;
         stroke: currentColor;
         fill: var(--wordplay-background);

src/parser/Parser.test.ts

@@ -11,7 +11,7 @@ import NumberLiteral from '@nodes/NumberLiteral';
 import NumberType from '@nodes/NumberType';
 import NameType from '@nodes/NameType';
 import NoneType from '@nodes/NoneType';
-import { toProgram } from './parseProgram';
+import parseProgram, { toProgram } from './parseProgram';
 import Program from '@nodes/Program';
 import StreamType from '@nodes/StreamType';
 import TableType from '@nodes/TableType';
@@ -384,3 +384,13 @@ test('unparsables in docs', () => {
     expect(doc.markup.paragraphs[0].segments[2]).toBeInstanceOf(Token);
     expect(doc.markup.paragraphs[0].segments.length).toBe(3);
 });
+
+test('unparsables in blocks', () => {
+    const program = parseProgram(toTokens('test: Phrase(\\\\)\ntest'));
+    expect(program).toBeInstanceOf(Program);
+    expect(program.expression).toBeInstanceOf(Block);
+    expect(program.expression.statements[0]).toBeInstanceOf(Bind);
+    expect(program.expression.statements[1]).toBeInstanceOf(
+        UnparsableExpression,
+    );
+});

src/parser/Tokens.ts

@@ -45,7 +45,7 @@ export default class Tokens {
     }
 
     peekUnread() {
-        return this.#unread;
+        return this.#unread.slice();
     }
 
     /** Returns true if the token list isn't empty. */

src/parser/parseExpression.ts

@@ -62,6 +62,7 @@ import Spread from '../nodes/Spread';
 import Otherwise from '@nodes/Otherwise';
 import Match from '@nodes/Match';
 import Input from '@nodes/Input';
+import type Token from '@nodes/Token';
 
 export function toExpression(code: string): Expression {
     return parseExpression(toTokens(code));
@@ -128,12 +129,23 @@ export function parseBlock(
             ((root && !doc) ||
                 (!root && !doc && tokens.nextIsnt(Sym.EvalClose)) ||
                 (doc && tokens.nextIsnt(Sym.Code))),
-        () =>
-            statements.push(
-                nextIsBind(tokens, true)
-                    ? parseBind(tokens)
-                    : parseExpression(tokens),
-            ),
+        () => {
+            const next = nextIsBind(tokens, true)
+                ? parseBind(tokens)
+                : parseExpression(tokens);
+            statements.push(next);
+            // Did we get an unparsable expression with no tokens? Read until we get to the block close or the end of the
+            // program. If we don't do this, the we will stop reading statements and will not parse the remainder of the program.
+            if (
+                next instanceof UnparsableExpression &&
+                next.unparsables.length === 0
+            ) {
+                const unparsed: Token[] = [];
+                while (tokens.hasNext() && tokens.nextIsnt(Sym.EvalClose))
+                    unparsed.push(tokens.read());
+                statements.push(new UnparsableExpression(unparsed));
+            }
+        },
     );
 
     const close = root

src/parser/parseProgram.ts

@@ -22,8 +22,7 @@ export default function parseProgram(tokens: Tokens, doc = false): Program {
 
     const block = parseBlock(tokens, BlockKind.Root, doc);
 
-    // If the next token is the end, we're done! Otherwise, read all of the remaining
-    // tokens and bundle them into an unparsable.
+    // If the next token is the end, we're done!
     const end = tokens.nextIsEnd() ? tokens.read(Sym.End) : undefined;
 
     return new Program(docs, borrows, block, end);

src/util/verify-locales.ts

@@ -146,17 +146,10 @@ function getTutorialJSON(log: Log, locale: string): object | undefined {
 async function validateLocale(
     log: Log,
     language: string,
-): Promise<Locale | undefined> {
-    let localeJSON = getLocaleJSON(log, language);
-    if (localeJSON === undefined) {
-        log.bad(
-            2,
-            "Couldn't find locale file. Can't validate it, or it's tutorial.",
-        );
-        return undefined;
-    }
-
-    const valid = LocaleValidator(localeJSON);
+    originalJSON: Locale,
+): Promise<[Locale, boolean] | undefined> {
+    let revisedJSON: Locale = originalJSON as Locale;
+    const valid = LocaleValidator(originalJSON);
     if (!valid && LocaleValidator.errors) {
         log.bad(
             2,
@@ -167,10 +160,19 @@ async function validateLocale(
                 log.bad(3, `${error.instancePath}: ${error.message}`);
         }
 
-        localeJSON = repairLocale(log, DefaultLocale, localeJSON as Locale);
+        revisedJSON = repairLocale(log, DefaultLocale, originalJSON as Locale);
     } else log.good(2, 'Found valid locale');
 
-    return await checkLocale(log, localeJSON as Locale, language !== 'example');
+    revisedJSON = await checkLocale(
+        log,
+        revisedJSON as Locale,
+        language !== 'example',
+    );
+
+    return [
+        revisedJSON,
+        JSON.stringify(revisedJSON) !== JSON.stringify(originalJSON),
+    ];
 }
 
 /** Add missing keys and remove extra ones from a given locale, relative to a source locale. */
@@ -189,15 +191,9 @@ function repairLocale(log: Log, source: Locale, target: Locale): Locale {
 /** Load, validate, and check the tutorial. */
 function validateTutorial(
     log: Log,
-    language: string,
     locale: Locale,
+    tutorialJSON: Tutorial,
 ): Tutorial | undefined {
-    const tutorialJSON = getTutorialJSON(log, language);
-    if (tutorialJSON === undefined) {
-        log.bad(2, "Couldn't find tutorial file.");
-        return undefined;
-    }
-
     const validate = ajv.compile(tutorialSchema);
     const valid = validate(tutorialJSON);
     if (!valid && validate.errors) {
@@ -857,35 +853,69 @@ fs.readdirSync(path.join('static', 'locales'), { withFileTypes: true }).forEach(
             log.flush();
             log.say(1, `Results for ${chalk.blue(language)}`);
 
-            // Validate, repair, and translate the locale file.
-            const revisedLocale = await validateLocale(log, language);
-            if (revisedLocale) {
-                // Write a formatted version of the revised locale file.
-                const localePath = getLocalePath(language);
-                const prettierOptions =
-                    await prettier.resolveConfig(localePath);
-
-                const prettyLocale = await prettier.format(
-                    JSON.stringify(revisedLocale, null, 4),
-                    { ...prettierOptions, parser: 'json' },
+            // Find prettier options
+            const localePath = getLocalePath(language);
+            const prettierOptions = await prettier.resolveConfig(localePath);
+
+            const originalLocale = getLocaleJSON(log, language);
+            if (originalLocale === undefined) {
+                log.bad(
+                    2,
+                    "Couldn't find locale file. Can't validate it, or it's tutorial.",
                 );
+                return undefined;
+            }
+
+            // Validate, repair, and translate the locale file.
+            const localeResults = await validateLocale(
+                log,
+                language,
+                originalLocale as Locale,
+            );
+            if (localeResults) {
+                const [revisedLocale, localeChanged] = localeResults;
+                if (localeChanged) {
+                    // Write a formatted version of the revised locale file.
 
-                fs.writeFileSync(getLocalePath(language), prettyLocale);
+                    const prettyLocale = await prettier.format(
+                        JSON.stringify(revisedLocale, null, 4),
+                        { ...prettierOptions, parser: 'json' },
+                    );
+
+                    console.log('Writing ' + language);
+                    fs.writeFileSync(getLocalePath(language), prettyLocale);
+                }
+
+                const currentTutorial = getTutorialJSON(log, language);
+                if (currentTutorial === undefined) {
+                    log.bad(2, "Couldn't find tutorial file.");
+                    return undefined;
+                }
 
                 // Validate, repair, and translate the tutorial file.
                 const revisedTutorial = validateTutorial(
                     log,
-                    language,
                     revisedLocale,
+                    currentTutorial as Tutorial,
                 );
-                if (revisedTutorial) {
+                if (
+                    revisedTutorial &&
+                    JSON.stringify(currentTutorial) !==
+                        JSON.stringify(revisedTutorial)
+                ) {
                     // Write a formatted version of the revised tutorial file.
                     const prettyTutorial = await prettier.format(
                         JSON.stringify(revisedTutorial, null, 4),
                         { ...prettierOptions, parser: 'json' },
                     );
 
-                    fs.writeFileSync(getTutorialPath(language), prettyTutorial);
+                    if (JSON.stringify(revisedTutorial) !== prettyTutorial) {
+                        console.log('Writing ' + language + ' tutorial');
+                        fs.writeFileSync(
+                            getTutorialPath(language),
+                            prettyTutorial,
+                        );
+                    }
                 }
             }