Table.input for pasting tabular data (#11695)

Closes #11350 - Copy/pasting tabular data now creates `Table.input` nodes. - Column names are always copied when you work inside Enso (excluding cases when you paste some cells into an existing table) - When working with external apps, column names are copied only if `Copy with headers` is selected. https://github.com/user-attachments/assets/a1233483-ee4a-47e4-84a1-64dd0b1505ef Roundtrip with Google Spreadsheets (shows non-trivial TSV data that includes quotes and newlines): https://github.com/user-attachments/assets/4ac662a2-809f-423a-9e47-628f46f92835
enso-org · Dec 3, 2024 · 4d2e44c · 4d2e44c
1 parent 78c2068
commit 4d2e44c
Show file tree

Hide file tree

Showing 16 changed files with 325 additions and 69 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -42,6 +42,7 @@
 - [New design for vector-editing widget][11620].
 - [Default values on widgets are displayed in italic][11666].
 - [Fixed bug causing Table Visualization to show wrong data][11684].
+- [Pasting tabular data now creates Table.input expressions][11695].
 - [No halo is displayed around components when hovering][11715].
 - [The hover area of the component output port extended twice its size][11715].
 
@@ -70,6 +71,7 @@
 [11666]: https://github.com/enso-org/enso/pull/11666
 [11690]: https://github.com/enso-org/enso/pull/11690
 [11684]: https://github.com/enso-org/enso/pull/11684
+[11695]: https://github.com/enso-org/enso/pull/11695
 [11715]: https://github.com/enso-org/enso/pull/11715
 
 #### Enso Standard Library

diff --git a/app/common/src/utilities/data/__tests__/array.test.ts b/app/common/src/utilities/data/__tests__/array.test.ts
@@ -0,0 +1,31 @@
+import { expect, test } from 'vitest'
+import * as array from '../array'
+
+interface TransposeCase {
+  matrix: number[][]
+  expected: number[][]
+}
+
+const transposeCases: TransposeCase[] = [
+  { matrix: [], expected: [] },
+  { matrix: [[]], expected: [[]] },
+  { matrix: [[1]], expected: [[1]] },
+  { matrix: [[1, 2]], expected: [[1], [2]] },
+  { matrix: [[1], [2]], expected: [[1, 2]] },
+  {
+    matrix: [
+      [1, 2, 3],
+      [4, 5, 6],
+    ],
+    expected: [
+      [1, 4],
+      [2, 5],
+      [3, 6],
+    ],
+  },
+]
+
+test.each(transposeCases)('transpose: case %#', ({ matrix, expected }) => {
+  const transposed = array.transpose(matrix)
+  expect(transposed).toStrictEqual(expected)
+})
diff --git a/app/common/src/utilities/data/array.ts b/app/common/src/utilities/data/array.ts
@@ -77,3 +77,10 @@ export function spliceAfter<T>(array: T[], items: T[], predicate: (value: T) =>
 export function splicedAfter<T>(array: T[], items: T[], predicate: (value: T) => boolean) {
   return spliceAfter(Array.from(array), items, predicate)
 }
+
+/** Transpose the matrix. */
+export function transpose<T>(matrix: T[][]): T[][] {
+  if (matrix.length === 0) return []
+  if (matrix[0] && matrix[0].length === 0) return [[]]
+  return matrix[0]!.map((_, colIndex) => matrix.map(row => row[colIndex]!))
+}
diff --git a/app/common/src/utilities/data/iter.ts b/app/common/src/utilities/data/iter.ts
@@ -218,3 +218,11 @@ export function last<T>(iter: Iterable<T>): T | undefined {
   for (const el of iter) last = el
   return last
 }
+
+/** Yields items of the iterable with their index. */
+export function* enumerate<T>(items: Iterable<T>): Generator<[T, number]> {
+  let index = 0
+  for (const item of items) {
+    yield [item, index++]
+  }
+}
diff --git a/app/gui/integration-test/project-view/tableVisualisation.spec.ts b/app/gui/integration-test/project-view/tableVisualisation.spec.ts
@@ -1,7 +1,7 @@
-import { test, type Page } from '@playwright/test'
+import { test, type Locator, type Page } from '@playwright/test'
 import * as actions from './actions'
 import { expect } from './customExpect'
-import { mockExpressionUpdate } from './expressionUpdates'
+import { mockExpressionUpdate, mockMethodCallInfo } from './expressionUpdates'
 import { CONTROL_KEY } from './keyboard'
 import * as locate from './locate'
 import { graphNodeByBinding } from './locate'
@@ -33,7 +33,7 @@ test('Load Table Visualisation', async ({ page }) => {
   await expect(tableVisualization).toContainText('3,0')
 })
 
-test('Copy from Table Visualization', async ({ page, context }) => {
+test('Copy/paste from Table Visualization', async ({ page, context }) => {
   await context.grantPermissions(['clipboard-read', 'clipboard-write'])
   await actions.goToGraph(page)
 
@@ -44,16 +44,28 @@ test('Copy from Table Visualization', async ({ page, context }) => {
   await page.mouse.down()
   await tableVisualization.getByText('2,1').hover()
   await page.mouse.up()
+
+  // Copy from table visualization
   await page.keyboard.press(`${CONTROL_KEY}+C`)
+  let clipboardContent = await page.evaluate(() => window.navigator.clipboard.readText())
+  expect(clipboardContent).toMatch(/^0,0\t0,1\r\n1,0\t1,1\r\n2,0\t2,1$/)
 
   // Paste to Node.
   await actions.clickAtBackground(page)
   const nodesCount = await locate.graphNode(page).count()
   await page.keyboard.press(`${CONTROL_KEY}+V`)
   await expect(locate.graphNode(page)).toHaveCount(nodesCount + 1)
-  await expect(locate.graphNode(page).last().locator('input')).toHaveValue(
-    '0,0\t0,11,0\t1,12,0\t2,1',
-  )
+  // Node binding would be `node1` for pasted node.
+  const nodeBinding = 'node1'
+  await mockMethodCallInfo(page, nodeBinding, {
+    methodPointer: {
+      module: 'Standard.Table.Table',
+      definedOnType: 'Standard.Table.Table.Table',
+      name: 'input',
+    },
+    notAppliedArguments: [],
+  })
+  await expectTableInputContent(page, locate.graphNode(page).last())
 
   // Paste to Table Widget.
   const node = await actions.createTableNode(page)
@@ -62,6 +74,32 @@ test('Copy from Table Visualization', async ({ page, context }) => {
   await widget.getByRole('button', { name: 'Add new column' }).click()
   await widget.locator('.valueCell').first().click()
   await page.keyboard.press(`${CONTROL_KEY}+V`)
+  await expectTableInputContent(page, node)
+
+  // Copy from table input widget
+  await node.getByText('0,0').hover()
+  await page.mouse.down()
+  await node.getByText('2,1').hover()
+  await page.mouse.up()
+  await page.keyboard.press(`${CONTROL_KEY}+C`)
+  clipboardContent = await page.evaluate(() => window.navigator.clipboard.readText())
+  expect(clipboardContent).toMatch(/^0,0\t0,1\r\n1,0\t1,1\r\n2,0\t2,1$/)
+
+  // Copy from table input widget with headers
+  await node.getByText('0,0').hover()
+  await page.mouse.down()
+  await node.getByText('2,1').hover()
+  await page.mouse.up()
+  await page.mouse.down({ button: 'right' })
+  await page.mouse.up({ button: 'right' })
+  await page.getByText('Copy with Headers').click()
+  clipboardContent = await page.evaluate(() => window.navigator.clipboard.readText())
+  expect(clipboardContent).toMatch(/^Column #1\tColumn #2\r\n0,0\t0,1\r\n1,0\t1,1\r\n2,0\t2,1$/)
+})
+
+async function expectTableInputContent(page: Page, node: Locator) {
+  const widget = node.locator('.WidgetTableEditor')
+  await expect(widget).toBeVisible({ timeout: 5000 })
   await expect(widget.locator('.valueCell')).toHaveText([
     '0,0',
     '0,1',
@@ -72,4 +110,4 @@ test('Copy from Table Visualization', async ({ page, context }) => {
     '',
     '',
   ])
-})
+}
diff --git a/app/gui/package.json b/app/gui/package.json
@@ -55,6 +55,7 @@
     "ajv": "^8.12.0",
     "amazon-cognito-identity-js": "6.3.6",
     "clsx": "^2.1.1",
+    "papaparse": "^5.4.1",
     "enso-common": "workspace:*",
     "framer-motion": "11.3.0",
     "idb-keyval": "^6.2.1",
@@ -144,6 +145,7 @@
     "@storybook/vue3-vite": "^8.4.2",
     "@tanstack/react-query-devtools": "5.45.1",
     "@types/node": "^22.9.0",
+    "@types/papaparse": "^5.3.15",
     "@types/react": "^18.0.27",
     "@types/react-dom": "^18.0.10",
     "@types/validator": "^13.11.7",

diff --git a/app/gui/src/project-view/components/GraphEditor.vue b/app/gui/src/project-view/components/GraphEditor.vue
@@ -277,8 +277,10 @@ const { scheduleCreateNode, createNodes, placeNode } = provideNodeCreation(
   toRef(graphNavigator, 'sceneMousePos'),
   (nodes) => {
     clearFocus()
-    nodeSelection.setSelection(nodes)
-    panToSelected()
+    if (nodes.size > 0) {
+      nodeSelection.setSelection(nodes)
+      panToSelected()
+    }
   },
 )
 

diff --git a/app/gui/src/project-view/components/GraphEditor/__tests__/clipboard.test.ts b/app/gui/src/project-view/components/GraphEditor/__tests__/clipboard.test.ts
@@ -3,7 +3,6 @@ import {
   isSpreadsheetTsv,
   nodesFromClipboardContent,
   nodesToClipboardData,
-  tsvTableToEnsoExpression,
 } from '@/components/GraphEditor/clipboard'
 import { type Node } from '@/stores/graph'
 import { Ast } from '@/util/ast'
@@ -13,36 +12,6 @@ import { expect, test } from 'vitest'
 import { assertDefined } from 'ydoc-shared/util/assert'
 import { type VisualizationMetadata } from 'ydoc-shared/yjsModel'
 
-test.each([
-  {
-    description: 'Unpaired surrogate',
-    tableData: '𝌆\t\uDAAA',
-    expectedEnsoExpression: "'𝌆\\t\\u{daaa}'.to Table",
-  },
-  {
-    description: 'Multiple rows, empty cells',
-    tableData: [
-      '\t36\t52',
-      '11\t\t4.727272727',
-      '12\t\t4.333333333',
-      '13\t2.769230769\t4',
-      '14\t2.571428571\t3.714285714',
-      '15\t2.4\t3.466666667',
-      '16\t2.25\t3.25',
-      '17\t2.117647059\t3.058823529',
-      '19\t1.894736842\t2.736842105',
-      '21\t1.714285714\t2.476190476',
-      '24\t1.5\t2.166666667',
-      '27\t1.333333333\t1.925925926',
-      '30\t1.2\t',
-    ].join('\n'),
-    expectedEnsoExpression:
-      "'\\t36\\t52\\n11\\t\\t4.727272727\\n12\\t\\t4.333333333\\n13\\t2.769230769\\t4\\n14\\t2.571428571\\t3.714285714\\n15\\t2.4\\t3.466666667\\n16\\t2.25\\t3.25\\n17\\t2.117647059\\t3.058823529\\n19\\t1.894736842\\t2.736842105\\n21\\t1.714285714\\t2.476190476\\n24\\t1.5\\t2.166666667\\n27\\t1.333333333\\t1.925925926\\n30\\t1.2\\t'.to Table",
-  },
-])('Enso expression from Excel data: $description', ({ tableData, expectedEnsoExpression }) => {
-  expect(tsvTableToEnsoExpression(tableData)).toEqual(expectedEnsoExpression)
-})
-
 class MockClipboardItem {
   readonly types: ReadonlyArray<string>
 

diff --git a/app/gui/src/project-view/components/GraphEditor/__tests__/clipboardTestCases.json b/app/gui/src/project-view/components/GraphEditor/__tests__/clipboardTestCases.json
@@ -7,7 +7,7 @@
         "firefox-127.0": "<google-sheets-html-origin><style type=\"text/css\"><!--td {border: 1px solid #cccccc;}br {mso-data-placement:same-cell;}--></style><table xmlns=\"http://www.w3.org/1999/xhtml\" cellspacing=\"0\" cellpadding=\"0\" dir=\"ltr\" border=\"1\" style=\"table-layout:fixed;font-size:10pt;font-family:Arial;width:0px;border-collapse:collapse;border:none\" data-sheets-root=\"1\"><colgroup><col width=\"155\"/><col width=\"71\"/></colgroup><tbody><tr style=\"height:21px;\"><td style=\"overflow:hidden;padding:2px 3px 2px 3px;vertical-align:bottom;\" data-sheets-value=\"{&quot;1&quot;:2,&quot;2&quot;:&quot;f/1.4 R LM WR&quot;}\">f/1.4 R LM WR</td><td style=\"overflow:hidden;padding:2px 3px 2px 3px;vertical-align:bottom;text-align:right;\" data-sheets-value=\"{&quot;1&quot;:3,&quot;3&quot;:18}\">18</td></tr><tr style=\"height:21px;\"><td style=\"overflow:hidden;padding:2px 3px 2px 3px;vertical-align:bottom;background-color:#ffffff;\" data-sheets-value=\"{&quot;1&quot;:2,&quot;2&quot;:&quot;f/1.4 R LM WR&quot;}\">f/1.4 R LM WR</td><td style=\"overflow:hidden;padding:2px 3px 2px 3px;vertical-align:bottom;text-align:right;\" data-sheets-value=\"{&quot;1&quot;:3,&quot;3&quot;:23}\">23</td></tr></tbody></table>"
       },
       "plainText": "SEL-20F18G\t20\nSEL-24F28G\t24",
-      "ensoCode": "'SEL-20F18G\\t20\\nSEL-24F28G\\t24'.to Table"
+      "ensoCode": "Table.input [['Column #1', ['SEL-20F18G', 'SEL-24F28G']], ['Column #2', ['20', '24']]]"
     },
     {
       "spreadsheet": "Excel",

diff --git a/app/gui/src/project-view/components/GraphEditor/clipboard.ts b/app/gui/src/project-view/components/GraphEditor/clipboard.ts
@@ -1,12 +1,10 @@
 import type { NodeCreationOptions } from '@/composables/nodeCreation'
 import type { Node } from '@/stores/graph'
-import { Ast } from '@/util/ast'
-import { Pattern } from '@/util/ast/match'
 import { nodeDocumentationText } from '@/util/ast/node'
 import { Vec2 } from '@/util/data/vec2'
 import * as iter from 'enso-common/src/utilities/data/iter'
-import { computed } from 'vue'
 import type { NodeMetadataFields } from 'ydoc-shared/ast'
+import { parseTsvData, tableToEnsoExpression } from './widgets/WidgetTableEditor/tableParsing'
 
 // MIME type in *vendor tree*; see https://www.rfc-editor.org/rfc/rfc6838#section-3.2
 // The `web ` prefix is required by Chromium:
@@ -133,19 +131,15 @@ const spreadsheetDecoder: ClipboardDecoder<CopiedNode[]> = {
     if (!item.types.includes('text/plain')) return
     if (isSpreadsheetTsv(htmlContent)) {
       const textData = await item.getType('text/plain').then((blob) => blob.text())
-      return [{ expression: tsvTableToEnsoExpression(textData) }]
+      const rows = parseTsvData(textData)
+      if (rows == null) return
+      const expression = tableToEnsoExpression(rows)
+      if (expression == null) return
+      return [{ expression }]
     }
   },
 }
 
-const toTable = computed(() => Pattern.parseExpression('__.to Table'))
-
-/** Create Enso Expression generating table from this tsvData. */
-export function tsvTableToEnsoExpression(tsvData: string) {
-  const textLiteral = Ast.TextLiteral.new(tsvData)
-  return toTable.value.instantiate(textLiteral.module, [textLiteral]).code()
-}
-
 /** @internal */
 export function isSpreadsheetTsv(htmlContent: string) {
   // This is a very general criterion that can have some false-positives (e.g. pasting rich text that includes a table).

diff --git a/...ject-view/components/GraphEditor/widgets/WidgetTableEditor/__tests__/tableParsing.test.ts b/...ject-view/components/GraphEditor/widgets/WidgetTableEditor/__tests__/tableParsing.test.ts
@@ -0,0 +1,54 @@
+import { assert } from '@/util/assert'
+import { expect, test } from 'vitest'
+import { parseTsvData, tableToEnsoExpression } from '../tableParsing'
+
+test.each([
+  {
+    description: 'Unpaired surrogate',
+    tableData: '𝌆\t\uDAAA',
+    expectedEnsoExpression: "Table.input [['Column #1', ['𝌆']], ['Column #2', ['\\u{daaa}']]]",
+  },
+  {
+    description: 'Empty cell',
+    tableData: '1\t2\n3\t',
+    expectedEnsoExpression:
+      "Table.input [['Column #1', ['1', '3']], ['Column #2', ['2', Nothing]]]",
+  },
+  {
+    description: 'Line feed in cell',
+    tableData: '1\t"2\n3"\n4\t5',
+    expectedEnsoExpression:
+      "Table.input [['Column #1', ['1', '4']], ['Column #2', ['2\\n3', '5']]]",
+  },
+  {
+    description: 'Line feed in quoted cell',
+    tableData: '1\t4\n2\t"""5\n6"""',
+    expectedEnsoExpression:
+      "Table.input [['Column #1', ['1', '2']], ['Column #2', ['4', '\"5\\n6\"']]]",
+  },
+  {
+    description: 'Multiple rows, empty cells',
+    tableData: [
+      '\t36\t52',
+      '11\t\t4.727272727',
+      '12\t\t4.333333333',
+      '13\t2.769230769\t4',
+      '14\t2.571428571\t3.714285714',
+      '15\t2.4\t3.466666667',
+      '16\t2.25\t3.25',
+      '17\t2.117647059\t3.058823529',
+      '19\t1.894736842\t2.736842105',
+      '21\t1.714285714\t2.476190476',
+      '24\t1.5\t2.166666667',
+      '27\t1.333333333\t1.925925926',
+      '30\t1.2\t',
+    ].join('\n'),
+    expectedEnsoExpression:
+      "Table.input [['Column #1', [Nothing, '11', '12', '13', '14', '15', '16', '17', '19', '21', '24', '27', '30']], ['Column #2', ['36', Nothing, Nothing, '2.769230769', '2.571428571', '2.4', '2.25', '2.117647059', '1.894736842', '1.714285714', '1.5', '1.333333333', '1.2']], ['Column #3', ['52', '4.727272727', '4.333333333', '4', '3.714285714', '3.466666667', '3.25', '3.058823529', '2.736842105', '2.476190476', '2.166666667', '1.925925926', Nothing]]]",
+  },
+])('Enso expression from Excel data: $description', ({ tableData, expectedEnsoExpression }) => {
+  const rows = parseTsvData(tableData)
+  expect(rows).not.toBeNull()
+  assert(rows != null)
+  expect(tableToEnsoExpression(rows)).toEqual(expectedEnsoExpression)
+})
diff --git a/...i/src/project-view/components/GraphEditor/widgets/WidgetTableEditor/tableInputArgument.ts b/...i/src/project-view/components/GraphEditor/widgets/WidgetTableEditor/tableInputArgument.ts
@@ -21,7 +21,7 @@ export const ROW_INDEX_HEADER = '#'
 /** A default prefix added to the column's index in newly created columns. */
 export const DEFAULT_COLUMN_PREFIX = 'Column #'
 const NOTHING_PATH = 'Standard.Base.Nothing.Nothing' as QualifiedName
-const NOTHING_NAME = qnLastSegment(NOTHING_PATH)
+export const NOTHING_NAME = qnLastSegment(NOTHING_PATH) as Ast.Identifier
 /**
  * The cells limit of the table; any modification which would exceed this limt should be
  * disallowed in UI
@@ -369,7 +369,7 @@ export function useTableInputArgument(
           contextMenuItems: [
             commonContextMenuActions.cut,
             commonContextMenuActions.copy,
-            'copyWithHeaders',
+            commonContextMenuActions.copyWithHeaders,
             commonContextMenuActions.paste,
             'separator',
             removeRowMenuItem,