WIP - Add xml metadata file storage and handling as a data upload output

seasketch · Oct 3, 2024 · 12a6496 · 12a6496
1 parent f977f36
commit 12a6496
Show file tree

Hide file tree

Showing 12 changed files with 204 additions and 11 deletions.
diff --git a/packages/api/migrations/current.sql b/packages/api/migrations/current.sql
@@ -1 +1,50 @@
 -- Enter migration here
+alter type data_upload_output_type add value if not exists 'XMLMetadata';
+
+drop function if exists table_of_contents_items_metadata_xml;
+create or replace function table_of_contents_items_metadata_xml(item table_of_contents_items)
+    returns data_upload_outputs
+    language plpgsql
+    stable
+    security definer
+  as $$
+    declare
+      ds_id int;
+      data_upload_output data_upload_outputs;
+    begin
+      select data_source_id into ds_id from data_layers where id = item.data_layer_id;
+      if ds_id is null then
+        return null;
+      end if;
+      select * into data_upload_output from data_upload_outputs where data_source_id = ds_id and type = 'XMLMetadata';
+      return data_upload_output;
+    end;
+$$;
+
+grant execute on function table_of_contents_items_metadata_xml(table_of_contents_items) to anon;
+
+create or replace function create_metadata_xml_output(data_source_id int, url text, remote text, size bigint, filename text)
+    returns data_upload_outputs
+    language plpgsql
+    stable
+    security definer
+  as $$
+    declare
+      output_id int;
+      output data_upload_outputs;
+      original_fname text;
+      pid int;
+    begin
+      select original_filename, project_id into original_fname, pid from data_upload_outputs where data_upload_outputs.data_source_id = create_metadata_xml_output.data_source_id;
+      if session_is_admin(pid) = false then
+        raise exception 'Only admins can create metadata xml outputs';
+      end if;
+      insert into data_upload_outputs (data_source_id, type, url, remote, size, filename, original_filename, project_id) values (data_source_id, 'XMLMetadata', url, remote, size, filename, original_fnam, pid) returning id into output_id;
+      select * into output from data_upload_outputs where id = output_id;
+      return output;
+    end;
+$$;
+
+    grant execute on function create_metadata_xml_output to seasketch_user;
+
+    comment on function create_metadata_xml_output is '@omit';
diff --git a/packages/api/src/plugins/computedMetadataPlugin.ts b/packages/api/src/plugins/computedMetadataPlugin.ts
@@ -14,6 +14,29 @@ const ComputedMetadataPlugin = makeExtendSchemaPlugin((build) => {
     `,
     resolvers: {
       TableOfContentsItem: {
+        // metadataXMLUrl: async (item, args, context, info) => {
+        //   if (item.dataLayerId) {
+        //     // first, get the data_source_id
+        //     const q = await context.pgClient.query(
+        //       `select data_source_id from data_layers where id = $1`,
+        //       [item.dataLayerId]
+        //     );
+        //     if (q.rows.length === 0) {
+        //       return null;
+        //     }
+        //     // then look for a data_upload_output with type = XMLMetadata
+        //     const { data_source_id } = q.rows[0];
+        //     const { rows } = await context.pgClient.query(
+        //       `select url from data_upload_outputs where data_source_id = $1 and type = 'XMLMetadata'`,
+        //       [data_source_id]
+        //     );
+        //     if (rows.length === 0) {
+        //       return null;
+        //     }
+        //     return rows[0].url;
+        //   }
+        //   return null;
+        // },
         computedMetadata: async (item, args, context, info) => {
           if (item.metadata) {
             return item.metadata;

diff --git a/packages/api/src/plugins/metadataParserPlugin.ts b/packages/api/src/plugins/metadataParserPlugin.ts
@@ -1,8 +1,28 @@
 import { makeExtendSchemaPlugin, gql } from "graphile-utils";
 import { metadataToProseMirror } from "@seasketch/metadata-parser";
+import S3 from "aws-sdk/clients/s3";
+
+const r2 = new S3({
+  region: "auto",
+  endpoint: process.env.R2_ENDPOINT,
+  signatureVersion: "v4",
+  accessKeyId: process.env.R2_ACCESS_KEY_ID,
+  secretAccessKey: process.env.R2_SECRET_ACCESS_KEY,
+});
+
+const REQUIRED_ENV_VARS = [
+  "R2_ENDPOINT",
+  "R2_ACCESS_KEY_ID",
+  "R2_SECRET_ACCESS_KEY",
+];
 
 const MetadataParserPlugin = makeExtendSchemaPlugin((build) => {
   const { pgSql: sql } = build;
+  for (const envvar of REQUIRED_ENV_VARS) {
+    if (!process.env[envvar]) {
+      throw new Error(`Missing env var ${envvar}`);
+    }
+  }
   return {
     typeDefs: gql`
       extend type Mutation {

diff --git a/packages/client/src/admin/data/QuotaUsageTreemap.tsx b/packages/client/src/admin/data/QuotaUsageTreemap.tsx
@@ -398,6 +398,8 @@ export function humanizeOutputType(type: DataUploadOutputType | "Archives") {
       return "PNG Image";
     case "Archives":
       return "Archived Versions";
+    case DataUploadOutputType.Xmlmetadata:
+      return "XML Metadata";
     default:
       return type;
   }

diff --git a/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx b/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx
@@ -11,6 +11,7 @@ import * as Tooltip from "@radix-ui/react-tooltip";
 import { useTranslation } from "react-i18next";
 import "./TooltipContent.css";
 import { humanizeOutputType } from "../QuotaUsageTreemap";
+import { GeostatsLayer, isRasterInfo } from "@seasketch/geostats-types";
 
 export default function HostedLayerInfo({
   source,
@@ -26,6 +27,7 @@ export default function HostedLayerInfo({
     | "uploadedSourceFilename"
     | "hostingQuotaUsed"
     | "outputs"
+    | "geostats"
   >;
   readonly?: boolean;
   layerId: number;
@@ -38,6 +40,15 @@ export default function HostedLayerInfo({
   });
   const { t } = useTranslation("admin:data");
   const original = (source.outputs || []).find((output) => output.isOriginal);
+  const metadata = (source.outputs || []).find(
+    (output) => output.type === DataUploadOutputType.Xmlmetadata
+  );
+  const metadataFormat =
+    source.geostats &&
+    !isRasterInfo(source.geostats) &&
+    source.geostats.layers.length
+      ? source.geostats.layers[0].metadata?.type
+      : undefined;
   return (
     <>
       {original && (
@@ -57,6 +68,31 @@ export default function HostedLayerInfo({
           }
         />
       )}
+      {metadata && (
+        <SettingsDLListItem
+          term={"Metadata"}
+          description={
+            <div className="truncate">
+              <a
+                className="text-primary-500 underline"
+                href={metadata.url}
+                target="_blank"
+                download={metadata.filename}
+              >
+                {metadata.filename || metadata.url}
+              </a>{" "}
+              <span className="text-gray-500">
+                {metadata.createdAt && metadataFormat
+                  ? // eslint-disable-next-line i18next/no-literal-string
+                    `${metadataFormat}, created ${new Date(
+                      metadata.createdAt
+                    ).toLocaleDateString()}`
+                  : ""}
+              </span>
+            </div>
+          }
+        />
+      )}
       <SettingsDLListItem
         term={"Quota Used"}
         description={

diff --git a/packages/client/src/dataLayers/DataDownloadModal.tsx b/packages/client/src/dataLayers/DataDownloadModal.tsx
@@ -63,7 +63,10 @@ export default function DataDownloadModal({
         // a must be equal to b
         return 0;
       })
-      .filter((option) => !option.isOriginal);
+      .filter(
+        (option) =>
+          !option.isOriginal && option.type !== DataUploadOutputType.Xmlmetadata
+      );
   }, [data?.tableOfContentsItem?.downloadOptions]);
 
   const getTranslatedProp = useTranslatedProps(data?.tableOfContentsItem);

diff --git a/packages/client/src/dataLayers/MapContextManager.ts b/packages/client/src/dataLayers/MapContextManager.ts
@@ -3238,15 +3238,20 @@ class MapContextManager extends EventEmitter {
     }
     if (bounds && [180.0, 90.0, -180.0, -90.0].join(",") !== bounds.join(",")) {
       const sidebar = currentSidebarState();
-      this.map?.fitBounds(bounds, {
-        animate: true,
-        padding: {
-          bottom: 100,
-          top: 100,
-          left: sidebar.open ? sidebar.width + 100 : 100,
-          right: 100,
-        },
-      });
+      try {
+        this.map?.fitBounds(bounds, {
+          animate: true,
+          padding: {
+            bottom: 100,
+            top: 100,
+            left: sidebar.open ? sidebar.width + 100 : 100,
+            right: 100,
+          },
+        });
+      } catch (e) {
+        // may be invalid bounds
+        console.error(e);
+      }
     }
   }
 

diff --git a/packages/client/src/dataLayers/MetadataModal.tsx b/packages/client/src/dataLayers/MetadataModal.tsx
@@ -3,6 +3,8 @@ import { useEffect, useMemo, useRef } from "react";
 import Modal from "../components/Modal";
 import Spinner from "../components/Spinner";
 import { metadata as editorConfig } from "../editor/config";
+import { MetadataXmlFileFragment } from "../generated/graphql";
+import { Trans } from "react-i18next";
 
 const { schema } = editorConfig;
 
@@ -12,12 +14,14 @@ export default function MetadataModal({
   loading,
   error,
   title,
+  xml,
 }: {
   document?: any;
   onRequestClose: () => void;
   loading: boolean;
   error?: Error;
   title?: string;
+  xml?: MetadataXmlFileFragment | null;
 }) {
   const target = useRef<HTMLDivElement>(null);
   const serializer = useRef(DOMSerializer.fromSchema(schema));
@@ -77,6 +81,22 @@ export default function MetadataModal({
             <h1 className="text-2xl font-medium">{title}</h1>
           )}
           <div className="ProseMirror" ref={target}></div>
+          {xml && (
+            <div className="mt-5 bg-blue-50 p-2 border rounded text-sm">
+              <Trans ns="homepage">
+                This layer includes metadata in XML format, available for{" "}
+              </Trans>
+              <a
+                href={xml.url}
+                target="_blank"
+                rel="noopener noreferrer"
+                className="text-blue-600 underline"
+                download={xml.filename}
+              >
+                <Trans ns="homepage">Download</Trans>
+              </a>
+            </div>
+          )}
         </div>
       </>
     </Modal>

diff --git a/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx b/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx
@@ -31,6 +31,7 @@ export default function TableOfContentsMetadataModal({
   return (
     <MetadataModal
       document={data?.tableOfContentsItem?.computedMetadata}
+      xml={data?.tableOfContentsItem?.metadataXml}
       loading={loading}
       error={error}
       onRequestClose={onRequestClose}

diff --git a/packages/client/src/queries/DraftTableOfContents.graphql b/packages/client/src/queries/DraftTableOfContents.graphql
@@ -266,6 +266,8 @@ fragment FullAdminSource on DataSource {
     type
     size
     originalFilename
+    filename
+    createdAt
   }
   changelog
 }
@@ -616,12 +618,22 @@ mutation UpdateEnableHighDPIRequests(
   }
 }
 
+fragment MetadataXmlFile on DataUploadOutput {
+  url
+  createdAt
+  filename
+  size
+}
+
 query GetMetadata($itemId: Int!) {
   tableOfContentsItem(id: $itemId) {
     id
     computedMetadata
     usesDynamicMetadata
     isCustomGlSource
+    metadataXml {
+      ...MetadataXmlFile
+    }
   }
 }
 

diff --git a/packages/spatial-uploads-handler/src/handleUpload.ts b/packages/spatial-uploads-handler/src/handleUpload.ts
@@ -39,7 +39,8 @@ export interface ResponseOutput {
     | SupportedTypes
     | "PMTiles"
     // geotif may be converted to normalized png when processing gray -> rgb
-    | "PNG";
+    | "PNG"
+    | "XMLMetadata";
   /** URL of the tile service (or geojson if really small) */
   url?: string;
   /** in bytes */
@@ -241,6 +242,14 @@ export default async function handleUpload(
     await putObject(logPath, s3LogPath, logger);
     const geostats = Array.isArray(stats) ? stats[0] : stats;
 
+    console.log(
+      "buliding response",
+      outputs.map((o) => ({
+        ...o,
+        local: undefined,
+        filename: o.filename,
+      }))
+    );
     const response: { layers: ProcessedUploadLayer[]; logfile: string } = {
       layers: [
         {

diff --git a/packages/spatial-uploads-handler/src/processVectorUpload.ts b/packages/spatial-uploads-handler/src/processVectorUpload.ts
@@ -129,6 +129,19 @@ export async function processVectorUpload(options: {
             const parsedMetadata = await metadataToProseMirror(data);
             if (parsedMetadata && Object.keys(parsedMetadata).length > 0) {
               metadata = parsedMetadata;
+              outputs.push({
+                type: "XMLMetadata",
+                filename: xmlPath.split("/").pop()!,
+                remote: `${
+                  process.env.RESOURCES_REMOTE
+                }/${baseKey}/${jobId}/${xmlPath.split("/").pop()!}`,
+                local: xmlPath,
+                size: statSync(xmlPath).size,
+                url: `${
+                  process.env.UPLOADS_BASE_URL
+                }/${baseKey}/${jobId}/${xmlPath.split("/").pop()!}`,
+              });
+              console.log("outputs", outputs);
               break;
             }
           } catch (e) {