feat: save file id for all fsspec connectors if present <- Ingest tes…

…t fixtures update (#3419) This pull request includes updated ingest test fixtures. Please review and merge if appropriate. Co-authored-by: rbiseck3 <[email protected]>
Unstructured-IO · Jul 19, 2024 · 83fada8 · 83fada8
1 parent 3fcf26c
commit 83fada8
Show file tree

Hide file tree

Showing 14 changed files with 158 additions and 79 deletions.
diff --git a/test_unstructured_ingest/expected-structured-output/box/handbook-1p.docx.json b/test_unstructured_ingest/expected-structured-output/box/handbook-1p.docx.json
@@ -14,7 +14,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -41,7 +42,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -68,7 +70,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -89,7 +92,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -110,7 +114,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -131,7 +136,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -152,7 +158,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -173,7 +180,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -194,7 +202,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -215,7 +224,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -236,7 +246,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -257,7 +268,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -278,7 +290,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -299,7 +312,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"
@@ -321,7 +335,8 @@
         "version": "83125548004193369404829885052395764226",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255888824139"
         },
         "date_created": "1688874451.0",
         "date_modified": "1688874451.0"

diff --git a/test_unstructured_ingest/expected-structured-output/box/nested-1/ideas-page.html.json b/test_unstructured_ingest/expected-structured-output/box/nested-1/ideas-page.html.json
@@ -14,7 +14,8 @@
         "version": "77943175838335685751163845636763163681",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255892530552"
         },
         "date_created": "1688874401.0",
         "date_modified": "1688874401.0"

diff --git a/...unstructured_ingest/expected-structured-output/box/nested-1/nested-2/ideas-page.html.json b/...unstructured_ingest/expected-structured-output/box/nested-1/nested-2/ideas-page.html.json
@@ -14,7 +14,8 @@
         "version": "293680985726204769765169474511274942733",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255884723846"
         },
         "date_created": "1688874389.0",
         "date_modified": "1688874389.0"

diff --git a/test_unstructured_ingest/expected-structured-output/box/science-exploration-1p.pptx.json b/test_unstructured_ingest/expected-structured-output/box/science-exploration-1p.pptx.json
@@ -14,7 +14,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -36,7 +37,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -58,7 +60,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -80,7 +83,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -102,7 +106,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -124,7 +129,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -146,7 +152,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -168,7 +175,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -190,7 +198,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -212,7 +221,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -234,7 +244,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -256,7 +267,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"
@@ -278,7 +290,8 @@
         "version": "309546934335254463247992132065898582121",
         "record_locator": {
           "protocol": "box",
-          "remote_file_path": "box://utic-test-ingest-fixtures"
+          "remote_file_path": "box://utic-test-ingest-fixtures",
+          "file_id": "1255894255490"
         },
         "date_created": "1688874428.0",
         "date_modified": "1688874428.0"