Adding archived_data_sources table for tracking uploaded source repla…

…cements (#735)

packages/api/generated-schema-clean.gql

@@ -244,6 +244,91 @@ enum ArcgisSourceType {
   ARCGIS_VECTOR
 }
 
+"""
+Admins can upload new version of data sources, and these are tracked from this
+table. This is used to track changes to data sources over time with a version
+number and optional changelog.
+"""
+type ArchivedDataSource implements Node {
+  """
+  Optional changelog so that admins can explain what changed in the new version.
+  """
+  changelog: String
+  dataLayerId: Int!
+
+  """
+  Reads a single `DataSource` that is related to this `ArchivedDataSource`.
+  """
+  dataSource: DataSource
+  dataSourceId: Int!
+
+  """
+  Mapbox GL style from the associated data layer at the time of upload of the
+  new version. This is tracked in case the data source is significantly changed
+  such that rolling back to a previous version also requires style changes
+  """
+  mapboxGlStyle: JSON!
+
+  """
+  A globally unique identifier. Can be used in various places throughout the system to identify this single value.
+  """
+  nodeId: ID!
+
+  """Array of sprite ids used in the archived mapbox_gl_style."""
+  spriteIds: [Int]
+
+  """
+  Version number of the data source. Incremented each time a new version is uploaded.
+  """
+  version: Int!
+}
+
+"""
+A condition to be used against `ArchivedDataSource` object types. All fields are
+tested for equality and combined with a logical ‘and.’
+"""
+input ArchivedDataSourceCondition {
+  """Checks for equality with the object’s `dataSourceId` field."""
+  dataSourceId: Int
+}
+
+"""A connection to a list of `ArchivedDataSource` values."""
+type ArchivedDataSourcesConnection {
+  """
+  A list of edges which contains the `ArchivedDataSource` and cursor to aid in pagination.
+  """
+  edges: [ArchivedDataSourcesEdge!]!
+
+  """A list of `ArchivedDataSource` objects."""
+  nodes: [ArchivedDataSource!]!
+
+  """Information to aid in pagination."""
+  pageInfo: PageInfo!
+
+  """
+  The count of *all* `ArchivedDataSource` you could get from the connection.
+  """
+  totalCount: Int!
+}
+
+"""A `ArchivedDataSource` edge in the connection."""
+type ArchivedDataSourcesEdge {
+  """A cursor for use in pagination."""
+  cursor: Cursor
+
+  """The `ArchivedDataSource` at the end of the edge."""
+  node: ArchivedDataSource!
+}
+
+"""Methods to use when ordering `ArchivedDataSource`."""
+enum ArchivedDataSourcesOrderBy {
+  DATA_SOURCE_ID_ASC
+  DATA_SOURCE_ID_DESC
+  NATURAL
+  PRIMARY_KEY_ASC
+  PRIMARY_KEY_DESC
+}
+
 """All input for the `archiveResponses` mutation."""
 input ArchiveResponsesInput {
   """
@@ -2412,6 +2497,23 @@ When documentation is lacking for any of these properties, consult the [MapBox G
 type DataSource implements Node {
   arcgisFetchStrategy: ArcgisFeatureLayerFetchStrategy!
 
+  """Reads and enables pagination through a set of `ArchivedDataSource`."""
+  archivedDataSources(
+    """
+    A condition to be used in determining which values should be returned by the collection.
+    """
+    condition: ArchivedDataSourceCondition
+
+    """Only read the first `n` values of the set."""
+    first: Int
+
+    """Skip the first `n` values."""
+    offset: Int
+
+    """The method to use when ordering `ArchivedDataSource`."""
+    orderBy: [ArchivedDataSourcesOrderBy!]
+  ): [ArchivedDataSource!]!
+
   """
   Contains an attribution to be displayed when the map is shown to a user.
   """
@@ -11201,6 +11303,44 @@ type Query implements Node {
     offset: Int
     period: ActivityStatsPeriod
   ): [Project!]
+  archivedDataSource(dataLayerId: Int!, dataSourceId: Int!, version: Int!): ArchivedDataSource
+
+  """Reads a single `ArchivedDataSource` using its globally unique `ID`."""
+  archivedDataSourceByNodeId(
+    """
+    The globally unique `ID` to be used in selecting a single `ArchivedDataSource`.
+    """
+    nodeId: ID!
+  ): ArchivedDataSource
+
+  """Reads and enables pagination through a set of `ArchivedDataSource`."""
+  archivedDataSourcesConnection(
+    """Read all values in the set after (below) this cursor."""
+    after: Cursor
+
+    """Read all values in the set before (above) this cursor."""
+    before: Cursor
+
+    """
+    A condition to be used in determining which values should be returned by the collection.
+    """
+    condition: ArchivedDataSourceCondition
+
+    """Only read the first `n` values of the set."""
+    first: Int
+
+    """Only read the last `n` values of the set."""
+    last: Int
+
+    """
+    Skip the first `n` values from our `after` cursor, an alternative to cursor
+    based pagination. May not be used with `last`.
+    """
+    offset: Int
+
+    """The method to use when ordering `ArchivedDataSource`."""
+    orderBy: [ArchivedDataSourcesOrderBy!] = [PRIMARY_KEY_ASC]
+  ): ArchivedDataSourcesConnection
   basemap(id: Int!): Basemap
 
   """Reads a single `Basemap` using its globally unique `ID`."""

packages/api/generated-schema.gql

@@ -244,6 +244,91 @@ enum ArcgisSourceType {
   ARCGIS_VECTOR
 }
 
+"""
+Admins can upload new version of data sources, and these are tracked from this
+table. This is used to track changes to data sources over time with a version
+number and optional changelog.
+"""
+type ArchivedDataSource implements Node {
+  """
+  Optional changelog so that admins can explain what changed in the new version.
+  """
+  changelog: String
+  dataLayerId: Int!
+
+  """
+  Reads a single `DataSource` that is related to this `ArchivedDataSource`.
+  """
+  dataSource: DataSource
+  dataSourceId: Int!
+
+  """
+  Mapbox GL style from the associated data layer at the time of upload of the
+  new version. This is tracked in case the data source is significantly changed
+  such that rolling back to a previous version also requires style changes
+  """
+  mapboxGlStyle: JSON!
+
+  """
+  A globally unique identifier. Can be used in various places throughout the system to identify this single value.
+  """
+  nodeId: ID!
+
+  """Array of sprite ids used in the archived mapbox_gl_style."""
+  spriteIds: [Int]
+
+  """
+  Version number of the data source. Incremented each time a new version is uploaded.
+  """
+  version: Int!
+}
+
+"""
+A condition to be used against `ArchivedDataSource` object types. All fields are
+tested for equality and combined with a logical ‘and.’
+"""
+input ArchivedDataSourceCondition {
+  """Checks for equality with the object’s `dataSourceId` field."""
+  dataSourceId: Int
+}
+
+"""A connection to a list of `ArchivedDataSource` values."""
+type ArchivedDataSourcesConnection {
+  """
+  A list of edges which contains the `ArchivedDataSource` and cursor to aid in pagination.
+  """
+  edges: [ArchivedDataSourcesEdge!]!
+
+  """A list of `ArchivedDataSource` objects."""
+  nodes: [ArchivedDataSource!]!
+
+  """Information to aid in pagination."""
+  pageInfo: PageInfo!
+
+  """
+  The count of *all* `ArchivedDataSource` you could get from the connection.
+  """
+  totalCount: Int!
+}
+
+"""A `ArchivedDataSource` edge in the connection."""
+type ArchivedDataSourcesEdge {
+  """A cursor for use in pagination."""
+  cursor: Cursor
+
+  """The `ArchivedDataSource` at the end of the edge."""
+  node: ArchivedDataSource!
+}
+
+"""Methods to use when ordering `ArchivedDataSource`."""
+enum ArchivedDataSourcesOrderBy {
+  DATA_SOURCE_ID_ASC
+  DATA_SOURCE_ID_DESC
+  NATURAL
+  PRIMARY_KEY_ASC
+  PRIMARY_KEY_DESC
+}
+
 """All input for the `archiveResponses` mutation."""
 input ArchiveResponsesInput {
   """
@@ -2412,6 +2497,23 @@ When documentation is lacking for any of these properties, consult the [MapBox G
 type DataSource implements Node {
   arcgisFetchStrategy: ArcgisFeatureLayerFetchStrategy!
 
+  """Reads and enables pagination through a set of `ArchivedDataSource`."""
+  archivedDataSources(
+    """
+    A condition to be used in determining which values should be returned by the collection.
+    """
+    condition: ArchivedDataSourceCondition
+
+    """Only read the first `n` values of the set."""
+    first: Int
+
+    """Skip the first `n` values."""
+    offset: Int
+
+    """The method to use when ordering `ArchivedDataSource`."""
+    orderBy: [ArchivedDataSourcesOrderBy!]
+  ): [ArchivedDataSource!]!
+
   """
   Contains an attribution to be displayed when the map is shown to a user.
   """
@@ -11201,6 +11303,44 @@ type Query implements Node {
     offset: Int
     period: ActivityStatsPeriod
   ): [Project!]
+  archivedDataSource(dataLayerId: Int!, dataSourceId: Int!, version: Int!): ArchivedDataSource
+
+  """Reads a single `ArchivedDataSource` using its globally unique `ID`."""
+  archivedDataSourceByNodeId(
+    """
+    The globally unique `ID` to be used in selecting a single `ArchivedDataSource`.
+    """
+    nodeId: ID!
+  ): ArchivedDataSource
+
+  """Reads and enables pagination through a set of `ArchivedDataSource`."""
+  archivedDataSourcesConnection(
+    """Read all values in the set after (below) this cursor."""
+    after: Cursor
+
+    """Read all values in the set before (above) this cursor."""
+    before: Cursor
+
+    """
+    A condition to be used in determining which values should be returned by the collection.
+    """
+    condition: ArchivedDataSourceCondition
+
+    """Only read the first `n` values of the set."""
+    first: Int
+
+    """Only read the last `n` values of the set."""
+    last: Int
+
+    """
+    Skip the first `n` values from our `after` cursor, an alternative to cursor
+    based pagination. May not be used with `last`.
+    """
+    offset: Int
+
+    """The method to use when ordering `ArchivedDataSource`."""
+    orderBy: [ArchivedDataSourcesOrderBy!] = [PRIMARY_KEY_ASC]
+  ): ArchivedDataSourcesConnection
   basemap(id: Int!): Basemap
 
   """Reads a single `Basemap` using its globally unique `ID`."""

packages/api/migrations/committed/000329.sql

@@ -0,0 +1,73 @@
+--! Previous: sha1:5a121ffbecd4ef38ef0cfe56433328d5d5c3dd39
+--! Hash: sha1:70af5c278ab4307d6445e6dcce482838d3d4ef96
+
+-- Enter migration here
+drop table if exists archived_data_sources;
+
+create table archived_data_sources (
+  data_source_id integer not null references data_sources(id) on delete cascade,
+  data_layer_id integer not null references data_layers(id) on delete cascade,
+  version integer not null,
+  mapbox_gl_style jsonb not null,
+  sprite_ids integer[] generated always as (extract_sprite_ids(mapbox_gl_style::text)) stored,
+  changelog text,
+  primary key (data_source_id, data_layer_id, version)
+);
+
+grant select on archived_data_sources to seasketch_user;
+
+alter table archived_data_sources enable row level security;
+
+create policy select_archived_data_sources_policy on archived_data_sources for select using (session_is_admin((select project_id from data_sources where id = data_source_id)));
+
+COMMENT ON CONSTRAINT archived_data_sources_data_source_id_fkey ON public.archived_data_sources IS '
+@simpleCollections only
+';
+
+comment on table archived_data_sources is 'Admins can upload new version of data sources, and these are tracked from this table. This is used to track changes to data sources over time with a version number and optional changelog.';
+
+comment on column archived_data_sources.version is 'Version number of the data source. Incremented each time a new version is uploaded.';
+
+comment on column archived_data_sources.mapbox_gl_style is 'Mapbox GL style from the associated data layer at the time of upload of the new version. This is tracked in case the data source is significantly changed such that rolling back to a previous version also requires style changes';
+
+comment on column archived_data_sources.sprite_ids is 'Array of sprite ids used in the archived mapbox_gl_style.';
+
+comment on column archived_data_sources.changelog is 'Optional changelog so that admins can explain what changed in the new version.';
+
+CREATE OR REPLACE FUNCTION public.projects_data_hosting_quota_used(p public.projects) RETURNS bigint
+    LANGUAGE plpgsql STABLE SECURITY DEFINER
+    AS $$
+    declare
+      sum_bytes bigint;
+      quota bigint;
+    begin
+    if session_is_admin(p.id) != true then
+      raise 'Permission denied';
+    end if;
+    select 
+      sum(size) 
+    into 
+      sum_bytes 
+    from 
+      data_upload_outputs
+    where
+      data_source_id = any (
+        select id from data_sources where id = any (
+          select data_source_id from data_layers where id = any (
+            select data_layer_id from table_of_contents_items where project_id = p.id and data_layer_id is not null and is_draft = true
+          ) union
+          select data_source_id from archived_data_sources where data_source_id = any (
+            select id from data_sources where project_id = p.id
+          )
+        )
+      );
+    select projects_data_hosting_quota(p) into quota;
+    if sum_bytes < quota then
+      return sum_bytes;
+    end if;
+    if sum_bytes is null then
+      return 0;
+    end if;
+    return quota;
+    end;
+  $$;