Merge pull request #248 from firebase/next

04/07/2020 Release

auth-mailchimp-sync/POSTINSTALL.md

@@ -1,8 +1,8 @@
 ### See it in action
 
-You can test out this extension right away:
+You can test out this extension right away!
 
-1.  Go to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users).
+1.  Go to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users) in the Firebase console.
 
 1.  Click **Add User** to add a test user.
 

delete-user-data/POSTINSTALL.md

@@ -1,12 +1,10 @@
 ### See it in action
 
-You can test out this extension right away:
+You can test out this extension right away!
 
-1.  Go to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users).
+1.  Go to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users) in the Firebase console.
 
-1.  Click **Add User** to add a test user.
-
-1.  Copy the test user's UID to your clipboard.
+1.  Click **Add User** to add a test user, then copy the test user's UID to your clipboard.
 
 1.  Create a new Cloud Firestore document, a new Realtime Database entry, or upload a new file to Storage - incorporating the user's UID into the path according to the schema that you configured.
 

firestore-bigquery-export/CHANGELOG.md

@@ -1,3 +1,12 @@
+## Version 0.1.3
+
+feature - Interpret data more easily with column descriptions in the exported BigQuery data (#138).
+
+- The raw changelog now includes column descriptions.
+- The schema-views script allows you to specify custom column descriptions.
+
+fixed - Updated `@firebaseextensions/firestore-bigquery-change-tracker` dependency (fixes issues #235 and #225).
+
 ## Version 0.1.2
 
 fixed - Added "IF NOT EXISTS" to safely run `fs-bq-schema-views` script multiple times (PR #193).

firestore-bigquery-export/DEVSETUP.md

@@ -0,0 +1,109 @@
+# Development Setup
+
+The firestore-bigquery-export extension is split out into 4 node.js packages, three of
+which are hosted on [npm](https://www.npmjs.com/search?q=firebaseextensions).
+
+**[firestore-bigquery-change-tracker](https://www.npmjs.com/package/@firebaseextensions/firestore-bigquery-change-tracker)**:
+Contains the core interface defintions for document changes. Also exposes an
+API for uploading changes to BigQuery. The business-logic associated with
+creating the raw changelog and the latest snapshot of live documents in the
+changelog also lives in this package.
+
+**[fs-bq-import-collection](https://www.npmjs.com/package/@firebaseextensions/fs-bq-import-collection)**:
+Contains a script for resumably importing a firestore collection into BigQuery
+using the interface definitions in `firestore-bigquery-change-tracker`.
+
+**[fs-bq-schema-views](https://www.npmjs.com/package/@firebaseextensions/fs-bq-schema-views)**:
+Contains a script for generating BigQuery views that provide typed-checked
+access to the changelog created in `firestore-bigquery-change-tracker`.
+
+**firestore-bigquery-export-functions (not hosted)**: Contains the definition
+for a Google Cloud function that is called on each write to some collection.
+The function constructs the relevant change event and calls the API in
+`firestore-bigquery-change-tracker` to upload the change to BigQuery.
+
+Here are the dependency edges:
+
+1. [fs-bq-import-collection -> firestore-bigquery-change-tracker](https://github.com/firebase/extensions/blob/next/firestore-bigquery-export/scripts/import/package.json#L27)
+1. [firestore-bigquery-export-functions -> firestore-bigquery-change-tracker](https://github.com/firebase/extensions/blob/next/firestore-bigquery-export/package.json#L17)
+
+Note that `fs-bq-schema-views` is a standalone package, and all of these
+packages are included at the root of this extension's folder in the firebase
+extensions repo.
+
+## Building Locally
+
+### Clean up any local changes (optional)
+
+Make sure you've cleaned up and partial builds:
+
+```
+export PKGS="firestore-bigquery-change-tracker scripts/gen-schema-view scripts/import functions ."
+
+for pkg in $PKGS;
+do
+  pushd . && cd $pkg && npm run clean && rm -rf node_modules
+  popd
+done;
+```
+
+### Local package.json file pointers
+
+Npm supports using [local paths as
+dependencies](https://docs.npmjs.com/files/package.json#local-paths) in package.json.
+You'll need to update the following package.json files with local pointers to
+the firestore-bigquery-change-tracker package:
+
+1. firestore-bigquery-export/package.json
+1. firestore-bigquery-export/scripts/import/package.json
+
+This can be done with jq from the root of this extension's folder:
+
+```
+jq '.dependencies."@firebaseextensions/firestore-bigquery-change-tracker" = "file:./firestore-bigquery-change-tracker"' package.json > package.local.json
+jq '.dependencies."@firebaseextensions/firestore-bigquery-change-tracker" = "file:../../firestore-bigquery-change-tracker"' scripts/import/package.json > scripts/import/package.local.json
+
+mv package.json package.json.bak
+mv scripts/import/package.json scripts/import/package.remote.json.bak
+
+mv package.local.json package.json
+mv scripts/import/package.local.json scripts/import/package.json
+```
+
+Now, build the components according to the dependency order.
+
+```
+export PKGS="firestore-bigquery-change-tracker scripts/import . scripts/gen-schema-view"
+
+for pkg in $PKGS;
+do
+ pushd . && cd $pkg && npm install && npm run build
+ popd
+done;
+```
+
+Finally, you can install the extension you just built onto a Firebase-enabled
+GCP project with:
+
+```
+firebase ext:install ./firestore-bigquery-export --project=project-id
+```
+
+## Publishing
+
+_The following instructions are for Firebase team members only._
+
+We publish 3 separate npm packages for this extension. Each follows semver, so
+make sure to update the version numbers and corresponding dependencies.
+
+For each package, `cd` into the appropriate directory, then run:
+
+```
+npm pack
+npm publish
+```
+
+In general, you should publish `firestore-bigquery-change-tracker` first, since
+it doesn't depend on anything else. You should also only publish _after_ your
+changes have been merged into the `next` branch (and before the extension is
+released).

firestore-bigquery-export/POSTINSTALL.md

@@ -1,10 +1,10 @@
 ### See it in action
 
-You can test out this extension right away:
+You can test out this extension right away!
 
-1.  Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data).
+1.  Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data) in the Firebase console.
 
-1.  If it doesn't already exist, create the collection you specified during installation: `${param:COLLECTION_PATH}`.
+1.  If it doesn't already exist, create the collection you specified during installation: `${param:COLLECTION_PATH}`
 
 1.  Create a document in the collection called `bigquery-mirror-test` that contains any fields with any values that you'd like.
 

firestore-bigquery-export/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-bigquery-export
-version: 0.1.2
+version: 0.1.3
 specVersion: v1beta
 
 displayName: Export Collections to BigQuery

firestore-bigquery-export/firestore-bigquery-change-tracker/package.json

@@ -5,19 +5,19 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.0.1",
+  "version": "1.1.2",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
     "build": "npm run clean && npm run compile",
-    "clean": "rimraf functions/lib",
+    "clean": "rimraf lib",
     "compile": "tsc",
-    "mocha": "nyc mocha -r ts-node/register --opts mocha.opts",
-    "test": "npm run mocha",
+    "test": "jest",
     "prepare": "npm run build"
   },
   "files": [
-    "lib"
+    "lib/*.js",
+    "lib/bigquery/*.js"
   ],
   "author": "Jan Wyszynski <[email protected]>",
   "license": "Apache-2.0",
@@ -31,13 +31,14 @@
     "sql-formatter": "^2.3.3"
   },
   "devDependencies": {
-    "chai": "^4.2.0",
+    "typescript": "^3.4.5",
     "rimraf": "^2.6.3",
-    "mocha": "^5.0.5",
     "nyc": "^14.0.0",
-    "@types/chai": "^4.1.6",
-    "@types/mocha": "^5.2.5",
+    "jest": "^24.9.0",
+    "chai": "^4.2.0",
     "ts-node": "^7.0.1",
-    "typescript": "^3.4.5"
+    "ts-jest": "^24.1.0",
+    "@types/jest": "^24.0.18",
+    "@types/chai": "^4.1.6"
   }
 }

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts

@@ -15,7 +15,7 @@
  */
 
 import * as bigquery from "@google-cloud/bigquery";
-import { firestoreToBQTable } from "./schema";
+import { RawChangelogSchema, RawChangelogViewSchema } from "./schema";
 import { latestConsistentSnapshotView } from "./snapshot";
 
 import {
@@ -25,6 +25,8 @@ import {
 } from "../tracker";
 import * as logs from "../logs";
 
+export { RawChangelogSchema, RawChangelogViewSchema } from "./schema";
+
 export interface FirestoreBigQueryEventHistoryTrackerConfig {
   datasetId: string;
   tableId: string;
@@ -134,7 +136,7 @@ export class FirestoreBigQueryEventHistoryTracker
       const options = {
         // `friendlyName` needs to be here to satisfy TypeScript
         friendlyName: changelogName,
-        schema: firestoreToBQTable(),
+        schema: RawChangelogSchema,
       };
       await table.create(options);
       logs.bigQueryTableCreated(changelogName);
@@ -164,6 +166,7 @@ export class FirestoreBigQueryEventHistoryTracker
         view: latestSnapshot,
       };
       await view.create(options);
+      await view.setMetadata({ schema: RawChangelogViewSchema });
       logs.bigQueryViewCreated(this.rawLatestView());
     }
     return view;

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts

@@ -64,23 +64,84 @@ export const timestampField = bigQueryField(
 export const latitudeField = bigQueryField("latitude", "NUMERIC");
 export const longitudeField = bigQueryField("longitude", "NUMERIC");
 
-/**
- * Convert from a list of Firestore field definitions into the schema
- * that will be used by the BigQuery `raw` data table.
- *
- * The `raw` data table schema is:
- * - event_id: The event ID of the function trigger invocation responsible for
- *   the row
- * - timestamp: A timestamp to be used for update ordering
- * - documentName: Stores the name of the Firestore document
- * - operation: The type of operation: CREATE, UPDATE, DELETE
- * - data: A record to contain the Firestore document data fields specified
- * in the schema
+/*
+ * We cannot specify a schema for view creation, and all view columns default
+ * to the NULLABLE mode.
  */
-export const firestoreToBQTable = (): BigQueryField[] => [
-  timestampField,
-  eventIdField,
-  documentNameField,
-  operationField,
-  dataField,
-];
+export const RawChangelogViewSchema: any = {
+  fields: [
+    {
+      name: "timestamp",
+      mode: "NULLABLE",
+      type: "TIMESTAMP",
+      description:
+        "The commit timestamp of this change in Cloud Firestore. If the operation is IMPORT, this timestamp is epoch to ensure that any operation on an imported document supersedes the IMPORT.",
+    },
+    {
+      name: "event_id",
+      mode: "NULLABLE",
+      type: "STRING",
+      description:
+        "The ID of the most-recent document change event that triggered the Cloud Function created by the extension. Empty for imports.",
+    },
+    {
+      name: "document_name",
+      mode: "NULLABLE",
+      type: "STRING",
+      description:
+        "The full name of the changed document, for example, projects/collection/databases/(default)/documents/users/me).",
+    },
+    {
+      name: "operation",
+      mode: "NULLABLE",
+      type: "STRING",
+      description: "One of CREATE, UPDATE, IMPORT.",
+    },
+    {
+      name: "data",
+      mode: "NULLABLE",
+      type: "STRING",
+      description:
+        "The full JSON representation of the current document state.",
+    },
+  ],
+};
+
+export const RawChangelogSchema: any = {
+  fields: [
+    {
+      name: "timestamp",
+      mode: "REQUIRED",
+      type: "TIMESTAMP",
+      description:
+        "The commit timestamp of this change in Cloud Firestore. If the operation is IMPORT, this timestamp is epoch to ensure that any operation on an imported document supersedes the IMPORT.",
+    },
+    {
+      name: "event_id",
+      mode: "REQUIRED",
+      type: "STRING",
+      description:
+        "The ID of the document change event that triggered the Cloud Function created by the extension. Empty for imports.",
+    },
+    {
+      name: "document_name",
+      mode: "REQUIRED",
+      type: "STRING",
+      description:
+        "The full name of the changed document, for example, projects/collection/databases/(default)/documents/users/me).",
+    },
+    {
+      name: "operation",
+      mode: "REQUIRED",
+      type: "STRING",
+      description: "One of CREATE, UPDATE, IMPORT, or DELETE.",
+    },
+    {
+      name: "data",
+      mode: "NULLABLE",
+      type: "STRING",
+      description:
+        "The full JSON representation of the document state after the indicated operation is applied. This field will be null for DELETE operations.",
+    },
+  ],
+};

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/snapshot.ts

@@ -17,7 +17,11 @@
 import * as sqlFormatter from "sql-formatter";
 
 import * as logs from "../logs";
-import { firestoreToBQTable, timestampField } from "./schema";
+import {
+  RawChangelogSchema,
+  RawChangelogViewSchema,
+  timestampField,
+} from "./schema";
 
 const excludeFields: string[] = ["document_name"];
 
@@ -29,9 +33,9 @@ export const latestConsistentSnapshotView = (
     datasetId,
     tableName,
     timestampField.name,
-    firestoreToBQTable()
+    RawChangelogViewSchema["fields"]
       .map((field) => field.name)
-      .filter((name) => excludeFields.indexOf(name) == -1)
+      .filter((name) => excludeFields.indexOf(name) === -1)
   ),
   useLegacySql: false,
 });

firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts

@@ -14,7 +14,10 @@
  * limitations under the License.
  */
 
-export { FirestoreBigQueryEventHistoryTracker } from "./bigquery";
+export {
+  FirestoreBigQueryEventHistoryTracker,
+  RawChangelogSchema,
+} from "./bigquery";
 export {
   ChangeType,
   FirestoreDocumentChangeEvent,