From 5e041c4e9efd3145f1fabe5c86e547465b063561 Mon Sep 17 00:00:00 2001 From: Chad Burt Date: Fri, 4 Oct 2024 14:29:48 -0700 Subject: [PATCH] Metadata file storage (#820) * WIP - Add xml metadata file storage and handling as a data upload output * WIP * WIP * Remove console.log * Move endpoint check to function body * Commit db changes * Fix for db migration issue * Fixing missing R2_TILES_BUCKET --- .github/workflows/deploy.yml | 1 + packages/api/generated-schema-clean.gql | 4 + packages/api/generated-schema.gql | 5 +- packages/api/migrations/committed/000335.sql | 119 ++++++++++ packages/api/process-env.d.ts | 1 + packages/api/schema.sql | 223 ++++++++++++++++-- .../api/src/plugins/computedMetadataPlugin.ts | 23 ++ .../api/src/plugins/metadataParserPlugin.ts | 72 ++++++ packages/client/src/admin/MetadataEditor.tsx | 20 ++ .../src/admin/data/OverlayMetadataEditor.tsx | 100 +++++++- .../src/admin/data/QuotaUsageTreemap.tsx | 2 + .../HostedLayerInfo.tsx | 52 ++++ .../data/TableOfContentsMetadataEditor.tsx | 9 + .../src/dataLayers/DataDownloadModal.tsx | 5 +- .../src/dataLayers/MapContextManager.ts | 23 +- .../client/src/dataLayers/MetadataModal.tsx | 21 ++ .../TableOfContentsMetadataModal.tsx | 8 + packages/client/src/editor/EditorMenuBar.tsx | 59 +---- packages/client/src/generated/graphql.ts | 59 ++++- packages/client/src/generated/queries.ts | 58 ++++- .../src/queries/DraftTableOfContents.graphql | 31 ++- .../infra/containers/maintenance/migrate.sh | 2 +- packages/infra/lib/GraphQLStack.ts | 6 +- .../src/handleUpload.ts | 11 +- .../src/processRasterUpload.ts | 27 ++- .../src/processVectorUpload.ts | 13 +- 26 files changed, 826 insertions(+), 128 deletions(-) create mode 100644 packages/api/migrations/committed/000335.sql diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7578749be..8d6f7f485 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -143,6 +143,7 @@ jobs: R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} R2_FILE_UPLOADS_BUCKET: ${{ secrets.R2_FILE_UPLOADS_BUCKET }} + R2_TILES_BUCKET: ${{ secrets.R2_TILES_BUCKET }} CLOUDFLARE_IMAGES_ACCOUNT: ${{ secrets.CLOUDFLARE_IMAGES_ACCOUNT }} CLOUDFLARE_IMAGES_TOKEN: ${{ secrets.CLOUDFLARE_IMAGES_TOKEN }} CLOUDFLARE_IMAGES_ACCOUNT_HASH: ${{ secrets.CLOUDFLARE_IMAGES_ACCOUNT_HASH }} diff --git a/packages/api/generated-schema-clean.gql b/packages/api/generated-schema-clean.gql index 7db352d4d..5d0ab111c 100644 --- a/packages/api/generated-schema-clean.gql +++ b/packages/api/generated-schema-clean.gql @@ -3390,6 +3390,7 @@ enum DataUploadOutputType { GEO_TIFF PMTILES PNG + XMLMETADATA ZIPPED_SHAPEFILE } @@ -9574,6 +9575,7 @@ type Mutation { """ input: UpdateTableOfContentsItemChildrenInput! ): UpdateTableOfContentsItemChildrenPayload + updateTocMetadataFromXML(filename: String, id: Int!, xmlMetadata: String!): TableOfContentsItem! """Updates a single `Topic` using a unique key and a patch.""" updateTopic( @@ -14329,6 +14331,8 @@ type TableOfContentsItem implements Node { DraftJS compatible representation of text content to display when a user requests layer metadata. Not valid for Folders """ metadata: JSON + metadataFormat: String + metadataXml: DataUploadOutput """ A globally unique identifier. Can be used in various places throughout the system to identify this single value. diff --git a/packages/api/generated-schema.gql b/packages/api/generated-schema.gql index ab59c8542..5d0ab111c 100644 --- a/packages/api/generated-schema.gql +++ b/packages/api/generated-schema.gql @@ -3390,6 +3390,7 @@ enum DataUploadOutputType { GEO_TIFF PMTILES PNG + XMLMETADATA ZIPPED_SHAPEFILE } @@ -9574,7 +9575,7 @@ type Mutation { """ input: UpdateTableOfContentsItemChildrenInput! ): UpdateTableOfContentsItemChildrenPayload - updateTocMetadataFromXML(id: Int!, xmlMetadata: String!): TableOfContentsItem! + updateTocMetadataFromXML(filename: String, id: Int!, xmlMetadata: String!): TableOfContentsItem! """Updates a single `Topic` using a unique key and a patch.""" updateTopic( @@ -14330,6 +14331,8 @@ type TableOfContentsItem implements Node { DraftJS compatible representation of text content to display when a user requests layer metadata. Not valid for Folders """ metadata: JSON + metadataFormat: String + metadataXml: DataUploadOutput """ A globally unique identifier. Can be used in various places throughout the system to identify this single value. diff --git a/packages/api/migrations/committed/000335.sql b/packages/api/migrations/committed/000335.sql new file mode 100644 index 000000000..15de0dacf --- /dev/null +++ b/packages/api/migrations/committed/000335.sql @@ -0,0 +1,119 @@ +--! Previous: sha1:b29221121218a7daf08c79f2e2580f66236bfdfc +--! Hash: sha1:fc2579e6e5275d22de4e81f534d1666a014721f8 + +-- Enter migration here +alter type data_upload_output_type add value if not exists 'XMLMetadata'; + +drop function if exists table_of_contents_items_metadata_xml; +create or replace function table_of_contents_items_metadata_xml(item table_of_contents_items) + returns data_upload_outputs + language plpgsql + stable + security definer + as $$ + declare + ds_id int; + data_upload_output data_upload_outputs; + begin + select data_source_id into ds_id from data_layers where id = item.data_layer_id; + if ds_id is null then + return null; + end if; + select * into data_upload_output from data_upload_outputs where data_source_id = ds_id and type = 'XMLMetadata'; + return data_upload_output; + end; +$$; + +grant execute on function table_of_contents_items_metadata_xml(table_of_contents_items) to anon; + +drop function if exists create_metadata_xml_output; +create or replace function create_metadata_xml_output(data_source_id int, url text, remote text, size bigint, filename text, metadata_type text) + returns data_upload_outputs + language plpgsql + security definer + as $$ + declare + source_exists boolean; + output_id int; + output data_upload_outputs; + original_fname text; + pid int; + begin + -- first, check if data_source even exists + select exists(select 1 from data_sources where id = data_source_id) into source_exists; + if source_exists = false then + raise exception 'Data source does not exist'; + end if; + + select + original_filename, + project_id + into original_fname, pid + from data_upload_outputs + where + data_upload_outputs.data_source_id = create_metadata_xml_output.data_source_id; + if session_is_admin(pid) = false then + raise exception 'Only admins can create metadata xml outputs'; + end if; + -- delete existing metadata xml output + delete from + data_upload_outputs + where + data_upload_outputs.data_source_id = create_metadata_xml_output.data_source_id and + type = 'XMLMetadata'; + insert into data_upload_outputs ( + data_source_id, + project_id, + type, + url, + remote, + size, + filename, + original_filename + ) values ( + create_metadata_xml_output.data_source_id, + pid, + 'XMLMetadata', + url, + remote, + size, + filename, + original_fname + ) returning * into output; + UPDATE data_sources + SET geostats = jsonb_set( + geostats, + '{layers,0,metadata,type}', + metadata_type::jsonb, + true + ) + WHERE data_sources.id = id and + jsonb_typeof(geostats->'layers'->0->'metadata'->'type') IS NOT NULL; + return output; + end; +$$; + +grant execute on function create_metadata_xml_output to seasketch_user; + +comment on function create_metadata_xml_output is '@omit'; + +drop function if exists table_of_contents_items_metadata_format; +create or replace function table_of_contents_items_metadata_format(item table_of_contents_items) + returns text + language plpgsql + stable + security definer + as $$ + declare + ds_id int; + metadata_type text; + begin + select data_source_id into ds_id from data_layers where id = item.data_layer_id; + SELECT geostats->'layers'->0->'metadata'->>'type' into metadata_type + FROM data_sources + WHERE jsonb_typeof(geostats->'layers'->0->'metadata'->'type') IS NOT NULL; + return metadata_type; + end; +$$; + +grant execute on function table_of_contents_items_metadata_format(table_of_contents_items) to anon; diff --git a/packages/api/process-env.d.ts b/packages/api/process-env.d.ts index 3ea45f97c..0e7decc66 100644 --- a/packages/api/process-env.d.ts +++ b/packages/api/process-env.d.ts @@ -49,5 +49,6 @@ declare namespace NodeJS { CLOUDFLARE_ACCOUNT_TAG: string; CLOUDFLARE_SITE_TAG: string; PMTILES_SERVER_ZONE: string; + R2_TILES_BUCKET: string; } } diff --git a/packages/api/schema.sql b/packages/api/schema.sql index a8f07087e..d9d1596ef 100644 --- a/packages/api/schema.sql +++ b/packages/api/schema.sql @@ -264,7 +264,8 @@ CREATE TYPE public.data_upload_output_type AS ENUM ( 'ZippedShapefile', 'PMTiles', 'GeoTIFF', - 'PNG' + 'PNG', + 'XMLMetadata' ); @@ -5940,6 +5941,101 @@ CREATE FUNCTION public.create_map_bookmark(slug text, "isPublic" boolean, style $$; +-- +-- Name: data_upload_outputs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.data_upload_outputs ( + id integer NOT NULL, + data_source_id integer, + project_id integer, + type public.data_upload_output_type NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + url text NOT NULL, + remote text NOT NULL, + is_original boolean DEFAULT false NOT NULL, + size bigint NOT NULL, + filename text NOT NULL, + original_filename text +); + + +-- +-- Name: create_metadata_xml_output(integer, text, text, bigint, text, text); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.create_metadata_xml_output(data_source_id integer, url text, remote text, size bigint, filename text, metadata_type text) RETURNS public.data_upload_outputs + LANGUAGE plpgsql SECURITY DEFINER + AS $$ + declare + source_exists boolean; + output_id int; + output data_upload_outputs; + original_fname text; + pid int; + begin + -- first, check if data_source even exists + select exists(select 1 from data_sources where id = data_source_id) into source_exists; + if source_exists = false then + raise exception 'Data source does not exist'; + end if; + + select + original_filename, + project_id + into original_fname, pid + from data_upload_outputs + where + data_upload_outputs.data_source_id = create_metadata_xml_output.data_source_id; + if session_is_admin(pid) = false then + raise exception 'Only admins can create metadata xml outputs'; + end if; + -- delete existing metadata xml output + delete from + data_upload_outputs + where + data_upload_outputs.data_source_id = create_metadata_xml_output.data_source_id and + type = 'XMLMetadata'; + insert into data_upload_outputs ( + data_source_id, + project_id, + type, + url, + remote, + size, + filename, + original_filename + ) values ( + create_metadata_xml_output.data_source_id, + pid, + 'XMLMetadata', + url, + remote, + size, + filename, + original_fname + ) returning * into output; + UPDATE data_sources + SET geostats = jsonb_set( + geostats, + '{layers,0,metadata,type}', + metadata_type::jsonb, + true + ) + WHERE data_sources.id = id and + jsonb_typeof(geostats->'layers'->0->'metadata'->'type') IS NOT NULL; + return output; + end; +$$; + + +-- +-- Name: FUNCTION create_metadata_xml_output(data_source_id integer, url text, remote text, size bigint, filename text, metadata_type text); Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON FUNCTION public.create_metadata_xml_output(data_source_id integer, url text, remote text, size bigint, filename text, metadata_type text) IS '@omit'; + + -- -- Name: extract_post_bookmark_attachments(jsonb); Type: FUNCTION; Schema: public; Owner: - -- @@ -8008,25 +8104,6 @@ CREATE FUNCTION public.data_sources_is_archived(data_source public.data_sources) $$; --- --- Name: data_upload_outputs; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.data_upload_outputs ( - id integer NOT NULL, - data_source_id integer, - project_id integer, - type public.data_upload_output_type NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL, - url text NOT NULL, - remote text NOT NULL, - is_original boolean DEFAULT false NOT NULL, - size bigint NOT NULL, - filename text NOT NULL, - original_filename text -); - - -- -- Name: data_sources_outputs(public.data_sources); Type: FUNCTION; Schema: public; Owner: - -- @@ -13950,6 +14027,40 @@ CREATE FUNCTION public.search_overlays(lang text, query text, "projectId" intege COMMENT ON FUNCTION public.search_overlays(lang text, query text, "projectId" integer, draft boolean, "limit" integer) IS '@simpleCollections only'; +-- +-- Name: search_overlays2(text, text, integer, boolean, integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.search_overlays2(lang text, query text, "projectId" integer, draft boolean, "limit" integer) RETURNS public.search_result[] + LANGUAGE plpgsql STABLE SECURITY DEFINER + AS $$ + declare + q tsquery := websearch_to_tsquery('english'::regconfig, query); + results search_result[]; + begin + IF position(' ' in query) <= 0 THEN + q := to_tsquery('english'::regconfig, query || ':*'); + end if; + select + id, + stable_id, + ts_headline('english', title, q, 'StartSel=<<<, StopSel=>>>') as title_headline, + ts_headline('english', jsonb_array_to_string(collect_prosemirror_text_nodes(metadata)), q, 'MaxFragments=10, MaxWords=7, MinWords=3, StartSel=<<<, StopSel=>>>') as metadata_headline, + is_folder + into results + from + table_of_contents_items + where + project_id = "projectId" and + is_draft = draft and + fts_en @@ q + limit + coalesce("limit", 10); + return results; + end; + $$; + + -- -- Name: invite_emails; Type: TABLE; Schema: public; Owner: - -- @@ -15706,6 +15817,47 @@ CREATE FUNCTION public.table_of_contents_items_is_downloadable_source_type(item $$; +-- +-- Name: table_of_contents_items_metadata_format(public.table_of_contents_items); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.table_of_contents_items_metadata_format(item public.table_of_contents_items) RETURNS text + LANGUAGE plpgsql STABLE SECURITY DEFINER + AS $$ + declare + ds_id int; + metadata_type text; + begin + select data_source_id into ds_id from data_layers where id = item.data_layer_id; + SELECT geostats->'layers'->0->'metadata'->>'type' into metadata_type + FROM data_sources + WHERE jsonb_typeof(geostats->'layers'->0->'metadata'->'type') IS NOT NULL; + return metadata_type; + end; +$$; + + +-- +-- Name: table_of_contents_items_metadata_xml(public.table_of_contents_items); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.table_of_contents_items_metadata_xml(item public.table_of_contents_items) RETURNS public.data_upload_outputs + LANGUAGE plpgsql STABLE SECURITY DEFINER + AS $$ + declare + ds_id int; + data_upload_output data_upload_outputs; + begin + select data_source_id into ds_id from data_layers where id = item.data_layer_id; + if ds_id is null then + return null; + end if; + select * into data_upload_output from data_upload_outputs where data_source_id = ds_id and type = 'XMLMetadata'; + return data_upload_output; + end; +$$; + + -- -- Name: table_of_contents_items_primary_download_url(public.table_of_contents_items); Type: FUNCTION; Schema: public; Owner: - -- @@ -25029,6 +25181,14 @@ REVOKE ALL ON FUNCTION public.create_map_bookmark(slug text, "isPublic" boolean, GRANT ALL ON FUNCTION public.create_map_bookmark(slug text, "isPublic" boolean, style jsonb, "visibleDataLayers" text[], "selectedBasemap" integer, "basemapOptionalLayerStates" jsonb, "cameraOptions" jsonb, "mapDimensions" integer[], "visibleSketches" integer[], "sidebarState" jsonb, "basemapName" text, "layerNames" jsonb, "sketchNames" jsonb, "clientGeneratedThumbnail" text) TO seasketch_user; +-- +-- Name: FUNCTION create_metadata_xml_output(data_source_id integer, url text, remote text, size bigint, filename text, metadata_type text); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.create_metadata_xml_output(data_source_id integer, url text, remote text, size bigint, filename text, metadata_type text) FROM PUBLIC; +GRANT ALL ON FUNCTION public.create_metadata_xml_output(data_source_id integer, url text, remote text, size bigint, filename text, metadata_type text) TO seasketch_user; + + -- -- Name: FUNCTION extract_post_bookmark_attachments(doc jsonb); Type: ACL; Schema: public; Owner: - -- @@ -29206,6 +29366,13 @@ REVOKE ALL ON FUNCTION public.search_overlays(lang text, query text, "projectId" GRANT ALL ON FUNCTION public.search_overlays(lang text, query text, "projectId" integer, draft boolean, "limit" integer) TO anon; +-- +-- Name: FUNCTION search_overlays2(lang text, query text, "projectId" integer, draft boolean, "limit" integer); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.search_overlays2(lang text, query text, "projectId" integer, draft boolean, "limit" integer) FROM PUBLIC; + + -- -- Name: COLUMN invite_emails.id; Type: ACL; Schema: public; Owner: - -- @@ -32626,6 +32793,22 @@ REVOKE ALL ON FUNCTION public.table_of_contents_items_is_downloadable_source_typ GRANT ALL ON FUNCTION public.table_of_contents_items_is_downloadable_source_type(item public.table_of_contents_items) TO anon; +-- +-- Name: FUNCTION table_of_contents_items_metadata_format(item public.table_of_contents_items); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.table_of_contents_items_metadata_format(item public.table_of_contents_items) FROM PUBLIC; +GRANT ALL ON FUNCTION public.table_of_contents_items_metadata_format(item public.table_of_contents_items) TO anon; + + +-- +-- Name: FUNCTION table_of_contents_items_metadata_xml(item public.table_of_contents_items); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.table_of_contents_items_metadata_xml(item public.table_of_contents_items) FROM PUBLIC; +GRANT ALL ON FUNCTION public.table_of_contents_items_metadata_xml(item public.table_of_contents_items) TO anon; + + -- -- Name: FUNCTION table_of_contents_items_primary_download_url(item public.table_of_contents_items); Type: ACL; Schema: public; Owner: - -- diff --git a/packages/api/src/plugins/computedMetadataPlugin.ts b/packages/api/src/plugins/computedMetadataPlugin.ts index 458cd3294..000f2e76b 100644 --- a/packages/api/src/plugins/computedMetadataPlugin.ts +++ b/packages/api/src/plugins/computedMetadataPlugin.ts @@ -14,6 +14,29 @@ const ComputedMetadataPlugin = makeExtendSchemaPlugin((build) => { `, resolvers: { TableOfContentsItem: { + // metadataXMLUrl: async (item, args, context, info) => { + // if (item.dataLayerId) { + // // first, get the data_source_id + // const q = await context.pgClient.query( + // `select data_source_id from data_layers where id = $1`, + // [item.dataLayerId] + // ); + // if (q.rows.length === 0) { + // return null; + // } + // // then look for a data_upload_output with type = XMLMetadata + // const { data_source_id } = q.rows[0]; + // const { rows } = await context.pgClient.query( + // `select url from data_upload_outputs where data_source_id = $1 and type = 'XMLMetadata'`, + // [data_source_id] + // ); + // if (rows.length === 0) { + // return null; + // } + // return rows[0].url; + // } + // return null; + // }, computedMetadata: async (item, args, context, info) => { if (item.metadata) { return item.metadata; diff --git a/packages/api/src/plugins/metadataParserPlugin.ts b/packages/api/src/plugins/metadataParserPlugin.ts index 173b43e50..f5831d25b 100644 --- a/packages/api/src/plugins/metadataParserPlugin.ts +++ b/packages/api/src/plugins/metadataParserPlugin.ts @@ -1,5 +1,21 @@ import { makeExtendSchemaPlugin, gql } from "graphile-utils"; import { metadataToProseMirror } from "@seasketch/metadata-parser"; +import { v4 as uuid } from "uuid"; +import S3 from "aws-sdk/clients/s3"; + +const r2 = new S3({ + region: "auto", + endpoint: process.env.R2_ENDPOINT, + signatureVersion: "v4", + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY, +}); + +const REQUIRED_ENV_VARS = [ + "R2_ENDPOINT", + "R2_ACCESS_KEY_ID", + "R2_SECRET_ACCESS_KEY", +]; const MetadataParserPlugin = makeExtendSchemaPlugin((build) => { const { pgSql: sql } = build; @@ -9,6 +25,7 @@ const MetadataParserPlugin = makeExtendSchemaPlugin((build) => { updateTocMetadataFromXML( id: Int! xmlMetadata: String! + filename: String ): TableOfContentsItem! } `, @@ -20,9 +37,64 @@ const MetadataParserPlugin = makeExtendSchemaPlugin((build) => { context, resolveInfo ) => { + for (const envvar of REQUIRED_ENV_VARS) { + if (!process.env[envvar]) { + throw new Error(`Missing env var ${envvar}`); + } + } + const { pgClient } = context; + const data = await metadataToProseMirror(args.xmlMetadata); if (data?.doc) { + // get project id related to table of contents item from db + const { rows: projectRows } = await pgClient.query( + `select project_id from public.table_of_contents_items where id = $1`, + [args.id] + ); + if (!projectRows?.[0]?.project_id) { + throw new Error("Table of contents item not found"); + } + const projectId = projectRows[0].project_id; + + // get the project slug + const { rows: slugRows } = await pgClient.query( + `select slug from public.projects where id = $1`, + [projectId] + ); + if (!slugRows?.[0]?.slug) { + throw new Error("Project not found"); + } + const slug = slugRows[0].slug; + + const objectPath = `projects/${slug}/metadata-updates/${uuid()}.xml`; + const remote = `r2://${process.env.R2_TILES_BUCKET}/${objectPath}`; + const url = `https://uploads.seasketch.org/${objectPath}`; + // get byte length of xmlMetadata + const byteLength = Buffer.byteLength(args.xmlMetadata, "utf8"); + // create a record in the db + await pgClient.query( + `select create_metadata_xml_output(( + select data_source_id from data_layers where id = (select data_layer_id from public.table_of_contents_items where id = $1) + ), $2, $3, $4, $5, $6)`, + [ + args.id, + url, + remote, + byteLength, + args.filename || "metadata.xml", + `"${data.type}"`, + ] + ); + // Save to r2 + const uploadParams = { + Bucket: process.env.R2_TILES_BUCKET, + Key: objectPath, + Body: args.xmlMetadata, // The XML content + ContentType: "application/xml", // MIME type for XML files + }; + + await r2.upload(uploadParams).promise(); const { rows } = await pgClient.query( `update public.table_of_contents_items set metadata = $1 where id = $2 returning data_layer_id`, [data.doc, args.id] diff --git a/packages/client/src/admin/MetadataEditor.tsx b/packages/client/src/admin/MetadataEditor.tsx index 0a8ecfe5c..e2fa19ac1 100644 --- a/packages/client/src/admin/MetadataEditor.tsx +++ b/packages/client/src/admin/MetadataEditor.tsx @@ -11,6 +11,7 @@ import useDialog from "../components/useDialog"; import Modal from "../components/Modal"; import { Pencil1Icon } from "@radix-ui/react-icons"; import useMetadataEditor from "./data/useMetadataEditor"; +import { MetadataXmlFileFragment } from "../generated/graphql"; const { schema, plugins } = editorConfig; interface MetadataEditorProps { @@ -28,6 +29,7 @@ interface MetadataEditorProps { /* whether the metadata document is using dynamic metadata from a CustomGLSource */ usingDynamicMetadata?: boolean; dynamicMetadataAvailable?: boolean; + xml?: (MetadataXmlFileFragment & { format: string }) | null; } export default function MetadataEditor({ @@ -39,6 +41,7 @@ export default function MetadataEditor({ startingDocument, usingDynamicMetadata, dynamicMetadataAvailable, + xml, }: MetadataEditorProps) { const { t } = useTranslation("admin"); const { confirm } = useDialog(); @@ -160,6 +163,23 @@ export default function MetadataEditor({ onChange={onChange} /> )} + {xml && ( +
+ + This layer includes metadata in {xml.format} XML format, available + for{" "} + + + Download + +
+ )} ); diff --git a/packages/client/src/admin/data/OverlayMetadataEditor.tsx b/packages/client/src/admin/data/OverlayMetadataEditor.tsx index 83d54b129..825c1427e 100644 --- a/packages/client/src/admin/data/OverlayMetadataEditor.tsx +++ b/packages/client/src/admin/data/OverlayMetadataEditor.tsx @@ -1,6 +1,7 @@ import { Trans, useTranslation } from "react-i18next"; import { useGetMetadataQuery, + useUpdateMetadataFromXmlMutation, useUpdateMetadataMutation, } from "../../generated/graphql"; import Skeleton from "../../components/Skeleton"; @@ -11,8 +12,9 @@ import useMetadataEditor from "./useMetadataEditor"; import Warning from "../../components/Warning"; import EditorMenuBar from "../../editor/EditorMenuBar"; import useDialog from "../../components/useDialog"; -import { Dispatch, SetStateAction, useEffect } from "react"; +import { Dispatch, SetStateAction, useCallback, useEffect } from "react"; import Button from "../../components/Button"; +import { useGlobalErrorHandler } from "../../components/GlobalErrorHandler"; export default function OverlayMetataEditor({ id, @@ -41,7 +43,14 @@ export default function OverlayMetataEditor({ loading, }); - const { confirm } = useDialog(); + const { confirm, loadingMessage } = useDialog(); + + const xml = data?.tableOfContentsItem?.metadataXml + ? { + ...data.tableOfContentsItem.metadataXml, + format: data.tableOfContentsItem.metadataFormat!, + } + : undefined; useEffect(() => { if (registerPreventUnload) { @@ -61,6 +70,68 @@ export default function OverlayMetataEditor({ } }, [hasChanges, registerPreventUnload, t]); + const onError = useGlobalErrorHandler(); + + const [uploadXMLMutation, uploadXMLMutationState] = + useUpdateMetadataFromXmlMutation(); + + const onUploadMetadataClick = useCallback(() => { + // create an input element to trigger the file upload dialog + var input = document.createElement("input"); + input.type = "file"; + // only accept xml files + input.accept = ".xml"; + input.onchange = (e: any) => { + if (e.target.files.length > 0 && id) { + const file = e.target.files[0]; + // verify that the file is an xml file + if (file.type !== "text/xml") { + alert("Please upload an XML file"); + return; + } + const loader = loadingMessage(t("Reading XML metadata")); + // read the xml file as a string + const reader = new FileReader(); + reader.onload = async (e) => { + let xml = e.target?.result; + if (xml) { + xml = xml.toString(); + try { + loader.updateLoadingMessage(t("Uploading XML metadata")); + const response = await uploadXMLMutation({ + variables: { + itemId: id!, + xml, + filename: file.name, + }, + }); + loader.hideLoadingMessage(); + if (!response.errors?.length && viewRef.current?.view) { + viewRef.current.view.focus(); + const tr = viewRef.current?.view.state.tr; + const node = schema.nodeFromJSON( + response.data?.updateTocMetadataFromXML.computedMetadata + ); + tr.replaceWith( + 0, + viewRef.current?.view.state.doc.content.size, + node + ); + viewRef.current?.view!.dispatch(tr); + } + } catch (e) { + console.error(e); + onError(e); + loader.hideLoadingMessage(); + } + } + }; + reader.readAsText(file); + } + }; + input.click(); + }, [id, uploadXMLMutation, viewRef.current?.view]); + return ( <> {/*

@@ -123,6 +194,7 @@ export default function OverlayMetataEditor({ className="border-t border-b pl-0 bg-gray-100 shadow-sm mb-1 border-black border-opacity-10 flex-none" state={state} schema={schema} + onUploadMetadataClick={onUploadMetadataClick} dynamicMetadataAvailable={dynamicMetadataAvailable} >
@@ -151,6 +223,30 @@ export default function OverlayMetataEditor({ // @ts-ignore ref={viewRef} /> + {!usingDynamicMetadata && xml && ( +
+ + This layer includes metadata in {xml.format} XML format. + +
+ + Download + + +
+
+ )} {dynamicMetadataAvailable && !usingDynamicMetadata && (

diff --git a/packages/client/src/admin/data/QuotaUsageTreemap.tsx b/packages/client/src/admin/data/QuotaUsageTreemap.tsx index 6a8dfbaca..7b30f5e28 100644 --- a/packages/client/src/admin/data/QuotaUsageTreemap.tsx +++ b/packages/client/src/admin/data/QuotaUsageTreemap.tsx @@ -398,6 +398,8 @@ export function humanizeOutputType(type: DataUploadOutputType | "Archives") { return "PNG Image"; case "Archives": return "Archived Versions"; + case DataUploadOutputType.Xmlmetadata: + return "XML Metadata"; default: return type; } diff --git a/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx b/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx index 1bca4c04f..631718298 100644 --- a/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx +++ b/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx @@ -11,6 +11,7 @@ import * as Tooltip from "@radix-ui/react-tooltip"; import { useTranslation } from "react-i18next"; import "./TooltipContent.css"; import { humanizeOutputType } from "../QuotaUsageTreemap"; +import { GeostatsLayer, isRasterInfo } from "@seasketch/geostats-types"; export default function HostedLayerInfo({ source, @@ -26,6 +27,7 @@ export default function HostedLayerInfo({ | "uploadedSourceFilename" | "hostingQuotaUsed" | "outputs" + | "geostats" >; readonly?: boolean; layerId: number; @@ -38,6 +40,22 @@ export default function HostedLayerInfo({ }); const { t } = useTranslation("admin:data"); const original = (source.outputs || []).find((output) => output.isOriginal); + const metadata = (source.outputs || []).find( + (output) => output.type === DataUploadOutputType.Xmlmetadata + ); + const metadataFormat = + source.geostats && + !isRasterInfo(source.geostats) && + source.geostats.layers.length + ? source.geostats.layers[0].metadata?.type + : undefined; + const metadataWasUpdated = + metadata && + Math.abs( + new Date(metadata.createdAt).getTime() - + new Date(original?.createdAt || 0).getTime() + ) > 1000; + return ( <> {original && ( @@ -57,6 +75,40 @@ export default function HostedLayerInfo({ } /> )} + {metadata && ( + + + + {metadata.filename || metadata.url} + {" "} + + + {metadata.createdAt && metadataFormat + ? // eslint-disable-next-line i18next/no-literal-string + `${metadataFormat}${ + metadataWasUpdated + ? ". Updated " + + new Date(metadata.createdAt).toLocaleString() + : "" + }` + : ""} + +

+ } + /> + )} ); } diff --git a/packages/client/src/dataLayers/DataDownloadModal.tsx b/packages/client/src/dataLayers/DataDownloadModal.tsx index a975b7d9e..f6ce2be82 100644 --- a/packages/client/src/dataLayers/DataDownloadModal.tsx +++ b/packages/client/src/dataLayers/DataDownloadModal.tsx @@ -63,7 +63,10 @@ export default function DataDownloadModal({ // a must be equal to b return 0; }) - .filter((option) => !option.isOriginal); + .filter( + (option) => + !option.isOriginal && option.type !== DataUploadOutputType.Xmlmetadata + ); }, [data?.tableOfContentsItem?.downloadOptions]); const getTranslatedProp = useTranslatedProps(data?.tableOfContentsItem); diff --git a/packages/client/src/dataLayers/MapContextManager.ts b/packages/client/src/dataLayers/MapContextManager.ts index ff4e829c3..ec7830081 100644 --- a/packages/client/src/dataLayers/MapContextManager.ts +++ b/packages/client/src/dataLayers/MapContextManager.ts @@ -3238,15 +3238,20 @@ class MapContextManager extends EventEmitter { } if (bounds && [180.0, 90.0, -180.0, -90.0].join(",") !== bounds.join(",")) { const sidebar = currentSidebarState(); - this.map?.fitBounds(bounds, { - animate: true, - padding: { - bottom: 100, - top: 100, - left: sidebar.open ? sidebar.width + 100 : 100, - right: 100, - }, - }); + try { + this.map?.fitBounds(bounds, { + animate: true, + padding: { + bottom: 100, + top: 100, + left: sidebar.open ? sidebar.width + 100 : 100, + right: 100, + }, + }); + } catch (e) { + // may be invalid bounds + console.error(e); + } } } diff --git a/packages/client/src/dataLayers/MetadataModal.tsx b/packages/client/src/dataLayers/MetadataModal.tsx index 985870b28..cd4e1d76c 100644 --- a/packages/client/src/dataLayers/MetadataModal.tsx +++ b/packages/client/src/dataLayers/MetadataModal.tsx @@ -3,6 +3,8 @@ import { useEffect, useMemo, useRef } from "react"; import Modal from "../components/Modal"; import Spinner from "../components/Spinner"; import { metadata as editorConfig } from "../editor/config"; +import { MetadataXmlFileFragment } from "../generated/graphql"; +import { Trans } from "react-i18next"; const { schema } = editorConfig; @@ -12,12 +14,14 @@ export default function MetadataModal({ loading, error, title, + xml, }: { document?: any; onRequestClose: () => void; loading: boolean; error?: Error; title?: string; + xml?: (MetadataXmlFileFragment & { format?: string }) | null; }) { const target = useRef(null); const serializer = useRef(DOMSerializer.fromSchema(schema)); @@ -77,6 +81,23 @@ export default function MetadataModal({

{title}

)}
+ {xml && ( +
+ + This layer includes metadata in {xml.format} XML format, + available for{" "} + + + Download + +
+ )} diff --git a/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx b/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx index 77926da73..4474c1612 100644 --- a/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx +++ b/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx @@ -31,6 +31,14 @@ export default function TableOfContentsMetadataModal({ return ( void; } export default function EditorMenuBar(props: EditorMenuBarProps) { @@ -154,60 +155,6 @@ export default function EditorMenuBar(props: EditorMenuBarProps) { [props.view, sketchingContext] ); - const [uploadXMLMutation, uploadXMLMutationState] = - useUpdateMetadataFromXmlMutation(); - - const onUploadButtonClick = useCallback(() => { - // create an input element to trigger the file upload dialog - var input = document.createElement("input"); - input.type = "file"; - // only accept xml files - input.accept = ".xml"; - input.onchange = (e: any) => { - if (e.target.files.length > 0 && props.tocId) { - const file = e.target.files[0]; - // verify that the file is an xml file - if (file.type !== "text/xml") { - alert("Please upload an XML file"); - return; - } - const loader = dialog.loadingMessage(t("Reading XML metadata")); - // read the xml file as a string - const reader = new FileReader(); - reader.onload = async (e) => { - let xml = e.target?.result; - if (xml) { - xml = xml.toString(); - try { - loader.updateLoadingMessage(t("Uploading XML metadata")); - const response = await uploadXMLMutation({ - variables: { - itemId: props.tocId!, - xml, - }, - }); - loader.hideLoadingMessage(); - if (!response.errors?.length && props.view) { - props.view!.focus(); - const tr = props.view!.state.tr; - const node = schema.nodeFromJSON( - response.data?.updateTocMetadataFromXML.computedMetadata - ); - tr.replaceWith(0, props.view!.state.doc.content.size, node); - props.view!.dispatch(tr); - } - } catch (e) { - console.error(e); - onError(e); - } - } - }; - reader.readAsText(file); - } - }; - input.click(); - }, [props.tocId, uploadXMLMutation, props.view]); - const [contextMenuTarget, setContextMenuTarget] = useState(null); @@ -461,9 +408,9 @@ export default function EditorMenuBar(props: EditorMenuBarProps) { /> - {props.showUploadOption && ( + {props.showUploadOption && props.onUploadMetadataClick && (