Incorporate metadata parsing package into data upload workflow

seasketch · Sep 9, 2024 · 29d7add · 29d7add
1 parent f3bba95
commit 29d7add
Show file tree

Hide file tree

Showing 17 changed files with 102 additions and 347 deletions.
diff --git a/packages/api/src/spatialUploads/index.ts b/packages/api/src/spatialUploads/index.ts
@@ -4,6 +4,7 @@ import { customAlphabet } from "nanoid";
 import { GeoJsonGeometryTypes } from "geojson";
 import {
   GeostatsLayer,
+  GeostatsMetadata,
   RasterInfo,
   SuggestedRasterPresentation,
   isRasterInfo,
@@ -195,7 +196,9 @@ export async function createDBRecordsForProcessedLayer(
             layerCount: 1,
           },
       pmtiles && !isVector ? 512 : null,
-      conversionTask?.attribution || null,
+      conversionTask?.attribution ||
+        layer.geostats?.metadata?.attribution ||
+        null,
       conversionTask?.location || null,
       Boolean(conversionTask),
       uploadedBy,
@@ -320,7 +323,7 @@ export async function createDBRecordsForProcessedLayer(
         `
           update table_of_contents_items set metadata = $1 where id = $2
         `,
-        [conversionTask.metadata || layer.geostats?.metadata, tocItem.id]
+        [conversionTask.metadata || layer.geostats?.metadata?.doc, tocItem.id]
       );
     }
 
@@ -380,15 +383,15 @@ export async function createDBRecordsForProcessedLayer(
       [
         projectId,
         nanoId(),
-        layer.name.replace("_", " "),
+        layer.geostats?.metadata?.title || layer.name.replace("_", " "),
         false,
         layer.bounds,
         dataLayerId,
         // 0,
         true,
         JSON.stringify(
           conversionTask?.metadata ||
-            layer.geostats?.metadata || {
+            layer.geostats?.metadata?.doc || {
               type: "doc",
               content: [
                 {

diff --git a/packages/geostats-types/lib/index.d.ts b/packages/geostats-types/lib/index.d.ts
@@ -2,18 +2,18 @@ import { GeoJsonGeometryTypes } from "geojson";
 /**
  * Attribute type as translated to a javacsript type
  */
-export declare type GeostatsAttributeType = "string" | "number" | "boolean" | "null" | "mixed" | "object" | "array";
+export type GeostatsAttributeType = "string" | "number" | "boolean" | "null" | "mixed" | "object" | "array";
 /**
  * A bucket is a tuple of [break, count]. Each bucket has a count of the number
  * of features between the break and the next break. The last bucket will have
  * a null count.
  */
-export declare type Bucket = [number, number | null];
+export type Bucket = [number, number | null];
 /**
  * A set of buckets for a given number of breaks. This way cartography
  * interfaces can give the user an option to choose the number of breaks
  */
-export declare type Buckets = {
+export type Buckets = {
     [numBreaks: number]: Bucket[];
 };
 export interface BaseGeostatsAttribute {
@@ -74,12 +74,31 @@ export interface NumericGeostatsAttribute extends BaseGeostatsAttribute {
         stdev: number;
     };
 }
-export declare type GeostatsAttribute = BaseGeostatsAttribute | NumericGeostatsAttribute;
-export declare type LegacyGeostatsAttribute = Omit<BaseGeostatsAttribute, "values" | "countDistinct"> & {
+export type GeostatsAttribute = BaseGeostatsAttribute | NumericGeostatsAttribute;
+export type LegacyGeostatsAttribute = Omit<BaseGeostatsAttribute, "values" | "countDistinct"> & {
     values: (string | number | boolean | null)[];
     quantiles?: number[];
 };
 export declare function isNumericGeostatsAttribute(attr: GeostatsAttribute): attr is NumericGeostatsAttribute;
+export declare enum MetadataType {
+    ISO19139 = "ISO19139",
+    FGDC = "FGDC"
+}
+export type GeostatsMetadata = {
+    /**
+     * metadata for the layer summarized as a prosemirror document
+     */
+    doc: any;
+    /**
+     * Attribution for the layer
+     */
+    attribution?: string;
+    type: MetadataType;
+    /**
+     * Suggested title based on the metadata
+     */
+    title?: string;
+};
 export interface GeostatsLayer {
     /**
      * Name for the layer
@@ -103,12 +122,9 @@ export interface GeostatsLayer {
      */
     attributes: GeostatsAttribute[];
     bounds?: number[];
-    /**
-     * Markdown-formatted metadata for the layer
-     */
-    metadata?: JSON;
+    metadata?: GeostatsMetadata;
 }
-export declare type LegacyGeostatsLayer = Omit<GeostatsLayer, "attributes" | "bounds"> & {
+export type LegacyGeostatsLayer = Omit<GeostatsLayer, "attributes" | "bounds"> & {
     attributes: LegacyGeostatsAttribute[];
 };
 export declare function isLegacyGeostatsLayer(layer: LegacyGeostatsLayer | GeostatsLayer): layer is LegacyGeostatsLayer;
@@ -118,21 +134,21 @@ export declare function isLegacyGeostatsAttribute(attr: LegacyGeostatsAttribute
  * number of features between the break and the next break. The last bucket will
  * have a null fraction.
  */
-export declare type RasterBucket = [number, number | null];
+export type RasterBucket = [number, number | null];
 /**
  * A set of buckets for a given number of breaks. This way cartography
  * interfaces can give the user an option to choose the number of breaks
  */
-export declare type RasterBuckets = {
+export type RasterBuckets = {
     [numBreaks: number]: RasterBucket[];
 };
 /**
  * A color table entry is a tuple of [value, color]. The value references the
  * raster value and the color is a CSS color string. Raster values should match
  * those in the stats.categories array.
  */
-export declare type ColorTableEntry = [number, string];
-export declare type RasterBandInfo = {
+export type ColorTableEntry = [number, string];
+export type RasterBandInfo = {
     name: string;
     colorInterpretation: "Red" | "Green" | "Blue" | "Alpha" | "Gray" | string | null;
     base: number;

diff --git a/packages/geostats-types/lib/index.js b/packages/geostats-types/lib/index.js
@@ -1,10 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isGeostatsLayer = exports.isRasterInfo = exports.SuggestedRasterPresentation = exports.isLegacyGeostatsAttribute = exports.isLegacyGeostatsLayer = exports.isNumericGeostatsAttribute = void 0;
+exports.SuggestedRasterPresentation = void 0;
+exports.isNumericGeostatsAttribute = isNumericGeostatsAttribute;
+exports.isLegacyGeostatsLayer = isLegacyGeostatsLayer;
+exports.isLegacyGeostatsAttribute = isLegacyGeostatsAttribute;
+exports.isRasterInfo = isRasterInfo;
+exports.isGeostatsLayer = isGeostatsLayer;
 function isNumericGeostatsAttribute(attr) {
     return attr.type === "number";
 }
-exports.isNumericGeostatsAttribute = isNumericGeostatsAttribute;
 function isLegacyGeostatsLayer(layer) {
     if ("attributesCount" in layer && layer.attributesCount) {
         return layer.attributes[0].countDistinct === undefined;
@@ -13,11 +17,9 @@ function isLegacyGeostatsLayer(layer) {
         return !("bounds" in layer);
     }
 }
-exports.isLegacyGeostatsLayer = isLegacyGeostatsLayer;
 function isLegacyGeostatsAttribute(attr) {
     return Array.isArray(attr.values);
 }
-exports.isLegacyGeostatsAttribute = isLegacyGeostatsAttribute;
 /**
  * SuggestedRasterPresentation is a hint to the client on how to present the
  * raster data. This can be used to determine the default visualization type for
@@ -33,12 +35,10 @@ var SuggestedRasterPresentation;
     SuggestedRasterPresentation[SuggestedRasterPresentation["categorical"] = 0] = "categorical";
     SuggestedRasterPresentation[SuggestedRasterPresentation["continuous"] = 1] = "continuous";
     SuggestedRasterPresentation[SuggestedRasterPresentation["rgb"] = 2] = "rgb";
-})(SuggestedRasterPresentation = exports.SuggestedRasterPresentation || (exports.SuggestedRasterPresentation = {}));
+})(SuggestedRasterPresentation || (exports.SuggestedRasterPresentation = SuggestedRasterPresentation = {}));
 function isRasterInfo(info) {
     return info.bands !== undefined;
 }
-exports.isRasterInfo = isRasterInfo;
 function isGeostatsLayer(data) {
     return (!Array.isArray(data) && data.attributes !== undefined);
 }
-exports.isGeostatsLayer = isGeostatsLayer;
diff --git a/packages/geostats-types/lib/index.ts b/packages/geostats-types/lib/index.ts
@@ -101,6 +101,27 @@ export function isNumericGeostatsAttribute(
   return attr.type === "number";
 }
 
+export declare enum MetadataType {
+  ISO19139 = "ISO19139",
+  FGDC = "FGDC",
+}
+
+export type GeostatsMetadata = {
+  /**
+   * metadata for the layer summarized as a prosemirror document
+   */
+  doc: any;
+  /**
+   * Attribution for the layer
+   */
+  attribution?: string;
+  type: MetadataType;
+  /**
+   * Suggested title based on the metadata
+   */
+  title?: string;
+};
+
 export interface GeostatsLayer {
   /**
    * Name for the layer
@@ -124,10 +145,7 @@ export interface GeostatsLayer {
    */
   attributes: GeostatsAttribute[];
   bounds?: number[];
-  /**
-   * Markdown-formatted metadata for the layer
-   */
-  metadata?: JSON;
+  metadata?: GeostatsMetadata;
 }
 
 export type LegacyGeostatsLayer = Omit<

diff --git a/packages/metadata-parser/dist/fgdcToProsemirror.d.ts b/packages/metadata-parser/dist/fgdcToProsemirror.d.ts
@@ -1,6 +1,3 @@
-export declare function fgdcToProseMirror(metadata: any): {
-    title: any;
-    doc: any;
-    attribution: string;
-};
-export declare function getAttribution(metadata: any): string;
+import { GeostatsMetadata } from "@seasketch/geostats-types";
+export declare function fgdcToProseMirror(metadata: any): GeostatsMetadata;
+export declare function getAttribution(metadata: any): string | undefined;
diff --git a/packages/metadata-parser/dist/fgdcToProsemirror.js b/packages/metadata-parser/dist/fgdcToProsemirror.js
@@ -182,6 +182,7 @@ function fgdcToProseMirror(metadata) {
         title: title,
         doc: doc,
         attribution: attribution,
+        type: "FGDC",
     };
 }
 function getAttribution(metadata) {
@@ -203,5 +204,5 @@ function getAttribution(metadata) {
     var cntinfo = getFirst(metc === null || metc === void 0 ? void 0 : metc.cntinfo);
     var cntorgp = getFirst(cntinfo === null || cntinfo === void 0 ? void 0 : cntinfo.cntorgp);
     var contactOrg = getFirst(cntorgp === null || cntorgp === void 0 ? void 0 : cntorgp.cntorg);
-    return contactOrg || "Unknown attribution";
+    return contactOrg || undefined;
 }
diff --git a/packages/metadata-parser/dist/iso19139ToProseMirror.d.ts b/packages/metadata-parser/dist/iso19139ToProseMirror.d.ts
@@ -1,6 +1,3 @@
-export declare function iso19139ToProseMirror(metadata: any): {
-    title: any;
-    doc: any;
-    attribution: string | null;
-};
-export declare function getAttribution(metadata: any): string | null;
+import { GeostatsMetadata } from "@seasketch/geostats-types";
+export declare function iso19139ToProseMirror(metadata: any): GeostatsMetadata;
+export declare function getAttribution(metadata: any): string | undefined;
diff --git a/packages/metadata-parser/dist/iso19139ToProseMirror.js b/packages/metadata-parser/dist/iso19139ToProseMirror.js
@@ -218,6 +218,7 @@ function iso19139ToProseMirror(metadata) {
         title: title,
         doc: doc,
         attribution: getAttribution(metadata),
+        type: "ISO19139",
     };
 }
 // Get attribution from ISO 19139 metadata
@@ -235,7 +236,7 @@ function getAttribution(metadata) {
     }
     // Fallback to other contact individual info if available
     var individual = (_f = (_e = responsibleParty === null || responsibleParty === void 0 ? void 0 : responsibleParty["gmd:individualName"]) === null || _e === void 0 ? void 0 : _e[0]["gco:CharacterString"]) === null || _f === void 0 ? void 0 : _f[0];
-    return individual || null;
+    return individual || undefined;
 }
 // Helper functions for parsing contact and constraints
 function parseContact(contact) {

diff --git a/packages/metadata-parser/dist/metadata-parser.d.ts b/packages/metadata-parser/dist/metadata-parser.d.ts
@@ -1,7 +1,3 @@
-export declare enum MetadataType {
-    ISO19139 = 0,
-    FGDC = 1
-}
 export interface Contact {
     name: string;
     organization: string;
@@ -13,8 +9,4 @@ export interface Contact {
     country: string;
     email: string;
 }
-export declare function metadataToProseMirror(xmlString: string): Promise<{
-    title: any;
-    doc: any;
-    attribution: string | null;
-} | null>;
+export declare function metadataToProseMirror(xmlString: string): Promise<import("@seasketch/geostats-types").GeostatsMetadata | null>;
diff --git a/packages/metadata-parser/dist/metadata-parser.js b/packages/metadata-parser/dist/metadata-parser.js
@@ -36,17 +36,11 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
     }
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MetadataType = void 0;
 exports.metadataToProseMirror = metadataToProseMirror;
 // @ts-ignore
 var xml2js_1 = require("xml2js");
 var fgdcToProsemirror_1 = require("./fgdcToProsemirror");
 var iso19139ToProseMirror_1 = require("./iso19139ToProseMirror");
-var MetadataType;
-(function (MetadataType) {
-    MetadataType[MetadataType["ISO19139"] = 0] = "ISO19139";
-    MetadataType[MetadataType["FGDC"] = 1] = "FGDC";
-})(MetadataType || (exports.MetadataType = MetadataType = {}));
 function metadataToProseMirror(xmlString) {
     return __awaiter(this, void 0, void 0, function () {
         var data, error_1;

diff --git a/packages/metadata-parser/package.json b/packages/metadata-parser/package.json
@@ -26,6 +26,7 @@
     "url": "https://github.com/seasketch/next/issues"
   },
   "dependencies": {
+    "@seasketch/geostats-types": "^1.0.0",
     "@types/xml2js": "^0.4.14",
     "xml2js": "^0.6.2"
   },

diff --git a/packages/metadata-parser/src/fgdcToProsemirror.ts b/packages/metadata-parser/src/fgdcToProsemirror.ts
@@ -1,3 +1,4 @@
+import { GeostatsMetadata } from "@seasketch/geostats-types";
 import { createParagraphNode } from "./createParagraphNode";
 
 // Helper function to safely access arrays
@@ -218,10 +219,11 @@ export function fgdcToProseMirror(metadata: any) {
     title,
     doc,
     attribution,
-  };
+    type: "FGDC",
+  } as GeostatsMetadata;
 }
 
-export function getAttribution(metadata: any): string {
+export function getAttribution(metadata: any): string | undefined {
   const getFirst = (value: any) =>
     Array.isArray(value) && value.length > 0 ? value[0] : "";
 
@@ -240,5 +242,5 @@ export function getAttribution(metadata: any): string {
   const cntinfo = getFirst(metc?.cntinfo);
   const cntorgp = getFirst(cntinfo?.cntorgp);
   const contactOrg = getFirst(cntorgp?.cntorg);
-  return contactOrg || "Unknown attribution";
+  return contactOrg || undefined;
 }
diff --git a/packages/metadata-parser/src/iso19139ToProseMirror.ts b/packages/metadata-parser/src/iso19139ToProseMirror.ts
@@ -1,3 +1,4 @@
+import { GeostatsMetadata } from "@seasketch/geostats-types";
 import { createParagraphNode } from "./createParagraphNode";
 
 // Helper function to safely access arrays
@@ -302,11 +303,12 @@ export function iso19139ToProseMirror(metadata: any) {
     title,
     doc,
     attribution: getAttribution(metadata),
-  };
+    type: "ISO19139",
+  } as GeostatsMetadata;
 }
 
 // Get attribution from ISO 19139 metadata
-export function getAttribution(metadata: any): string | null {
+export function getAttribution(metadata: any): string | undefined {
   if ("gmd:MD_Metadata" in metadata) {
     metadata = metadata["gmd:MD_Metadata"];
   }
@@ -323,7 +325,7 @@ export function getAttribution(metadata: any): string | null {
   // Fallback to other contact individual info if available
   const individual =
     responsibleParty?.["gmd:individualName"]?.[0]["gco:CharacterString"]?.[0];
-  return individual || null;
+  return individual || undefined;
 }
 
 // Helper functions for parsing contact and constraints

diff --git a/packages/metadata-parser/src/metadata-parser.ts b/packages/metadata-parser/src/metadata-parser.ts
@@ -3,11 +3,6 @@ import { parseStringPromise } from "xml2js";
 import { fgdcToProseMirror } from "./fgdcToProsemirror";
 import { iso19139ToProseMirror } from "./iso19139ToProseMirror";
 
-export enum MetadataType {
-  ISO19139,
-  FGDC,
-}
-
 export interface Contact {
   name: string;
   organization: string;

diff --git a/packages/spatial-uploads-handler/package.json b/packages/spatial-uploads-handler/package.json
@@ -42,6 +42,7 @@
     "@babel/preset-typescript": "^7.24.1",
     "@mapbox/mbtiles": "^0.12.1",
     "@seasketch/geostats-types": "^1.0.0",
+    "@seasketch/metadata-parser": "^1.0.0",
     "@slack/web-api": "^6.7.2",
     "@turf/bbox": "^6.5.0",
     "@types/bytes": "^3.1.1",