[scd/crdb] Add columns ovn and past_ovns to scd_operations table; Bum…

…p schema_version to v3.2.0
interuss · Sep 2, 2024 · 8097b40 · 8097b40
1 parent 02f8ac9
commit 8097b40
Show file tree

Hide file tree

Showing 10 changed files with 68 additions and 11 deletions.
diff --git a/build/db_schemas/README.md b/build/db_schemas/README.md
@@ -18,6 +18,6 @@ places:
  schema_versions.schema_version
 * [DSS main.jsonnet](../deploy/examples/minimum/main.jsonnet)
 * [Schema manager main.jsonnet](../deploy/examples/schema_manager/main.jsonnet)
-* scd_ or rid_ bootstrapper.sh in [dev/startup](../dev/startup)
 * /pkg/{rid|scd}/store/cockroach/store.go
 * /deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf
+* /deploy/services/helm-charts/dss/templates/schema-manager.yaml
diff --git a/build/db_schemas/scd.libsonnet b/build/db_schemas/scd.libsonnet
@@ -4,6 +4,8 @@
  "upto-v2.0.0-support_api_1_0_0.sql": importstr "scd/upto-v2.0.0-support_api_1_0_0.sql",
  "upto-v3.0.0-add_inverted_indices.sql": importstr "scd/upto-v3.0.0-add_inverted_indices.sql",
  "upto-v3.1.0-create_uss_availability.sql": importstr "scd/upto-v3.1.0-create_uss_availability.sql",
+ "upto-v3.2.0-add_ovn_columns.sql": importstr "scd/upto-v3.2.0-add_ovn_columns.sql",
+ "downfrom-v3.2.0-remove_ovn_columns.sql": importstr "scd/downfrom-v3.2.0-remove_ovn_columns.sql",
  "downfrom-v3.1.0-remove_uss_availability.sql": importstr "scd/downfrom-v3.1.0-remove_uss_availability.sql",
  "downfrom-v3.0.0-remove_inverted_indices.sql": importstr "scd/downfrom-v3.0.0-remove_inverted_indices.sql",
  "downfrom-v2.0.0-remove_api_1_0_0_support.sql": importstr "scd/downfrom-v2.0.0-remove_api_1_0_0_support.sql",

diff --git a/build/db_schemas/scd/downfrom-v3.2.0-remove_ovn_columns.sql b/build/db_schemas/scd/downfrom-v3.2.0-remove_ovn_columns.sql
@@ -0,0 +1,7 @@
+ALTER TABLE scd_operations
+ DROP IF EXISTS ovn,
+ DROP IF EXISTS past_ovns;
+
+UPDATE schema_versions
+SET schema_version = 'v3.1.0'
+WHERE onerow_enforcer = TRUE;
diff --git a/build/db_schemas/scd/upto-v3.2.0-add_ovn_columns.sql b/build/db_schemas/scd/upto-v3.2.0-add_ovn_columns.sql
@@ -0,0 +1,13 @@
+ALTER TABLE scd_operations
+ ADD COLUMN IF NOT EXISTS ovn STRING
+ CHECK (ovn != ''), -- ovn must be NULL if unspecified, not an empty string
+ ADD COLUMN IF NOT EXISTS past_ovns STRING[] NOT NULL
+ DEFAULT ARRAY []::STRING[]
+ CHECK (
+ array_position(past_ovns, NULL) IS NULL AND
+ array_position(past_ovns, NULL) IS NULL
+ ); -- past_ovns must not contain NULL elements nor empty strings
+
+UPDATE schema_versions
+SET schema_version = 'v3.2.0'
+WHERE onerow_enforcer = TRUE;
diff --git a/build/deploy/examples/minimum/main.jsonnet b/build/deploy/examples/minimum/main.jsonnet
@@ -31,7 +31,7 @@ local metadata = metadataBase {
  schema_manager+: {
  image: 'VAR_DOCKER_IMAGE_NAME',
  desired_rid_db_version: '4.0.0',
- desired_scd_db_version: '3.1.0',
+ desired_scd_db_version: '3.2.0',
  },
  prometheus+: {
  storageClass: 'VAR_STORAGE_CLASS',

diff --git a/build/deploy/examples/schema_manager/main.jsonnet b/build/deploy/examples/schema_manager/main.jsonnet
@@ -15,7 +15,7 @@ local metadata = metadataBase {
  schema_manager+: {
  image: 'VAR_DOCKER_IMAGE_NAME',
  desired_rid_db_version: '4.0.0',
- desired_scd_db_version: '3.1.0',
+ desired_scd_db_version: '3.2.0',
  },
 };
 

diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf
@@ -1,4 +1,4 @@
 locals {
  rid_db_schema = var.desired_rid_db_version == "latest" ? "4.0.0" : var.desired_rid_db_version
- scd_db_schema = var.desired_scd_db_version == "latest" ? "3.1.0" : var.desired_scd_db_version
+ scd_db_schema = var.desired_scd_db_version == "latest" ? "3.2.0" : var.desired_scd_db_version
 }
diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml
@@ -3,7 +3,7 @@
 {{- $jobVersion := .Release.Revision -}} {{/* Jobs template definition is immutable, using the revision in the name forces the job to be recreated at each helm upgrade. */}}
 {{- $waitForCockroachDB := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $cockroachHost)) -}}
 
-{{- range $service, $schemaVersion := dict "rid" "4.0.0" "scd" "3.1.0" }}
+{{- range $service, $schemaVersion := dict "rid" "4.0.0" "scd" "3.2.0" }}
 ---
 apiVersion: batch/v1
 kind: Job
@@ -50,4 +50,4 @@ spec:
  volumes:
  {{- include "ca-certs:volume" . | nindent 8 }}
  {{- include "client-certs:volume" . | nindent 8 }}
-{{- end -}}
+{{- end -}}
diff --git a/pkg/scd/models/operational_intents.go b/pkg/scd/models/operational_intents.go
@@ -76,6 +76,7 @@ type OperationalIntent struct {
  Version VersionNumber
  State OperationalIntentState
  OVN OVN
+ PastOVNs []OVN
  StartTime *time.Time
  EndTime *time.Time
  USSBaseURL string

diff --git a/pkg/scd/store/cockroach/operational_intents.go b/pkg/scd/store/cockroach/operational_intents.go
@@ -11,13 +11,14 @@ import (
  scdmodels "github.com/interuss/dss/pkg/scd/models"
  dsssql "github.com/interuss/dss/pkg/sql"
  "github.com/interuss/stacktrace"
+ "github.com/jackc/pgx/v5/pgtype"
 
  "github.com/jackc/pgx/v5"
  "github.com/pkg/errors"
 )
 
 var (
- operationFieldsWithIndices [12]string
+ operationFieldsWithIndices [14]string
  operationFieldsWithPrefix string
  operationFieldsWithoutPrefix string
 )
@@ -36,6 +37,8 @@ func init() {
  operationFieldsWithIndices[9] = "updated_at"
  operationFieldsWithIndices[10] = "state"
  operationFieldsWithIndices[11] = "cells"
+ operationFieldsWithIndices[12] = "ovn"
+ operationFieldsWithIndices[13] = "past_ovns"
 
  operationFieldsWithoutPrefix = strings.Join(
  operationFieldsWithIndices[:], ",",
@@ -58,8 +61,12 @@ func (s *repo) fetchOperationalIntents(ctx context.Context, q dsssql.Queryable,
  }
  defer rows.Close()
 
- var payload []*scdmodels.OperationalIntent
- var cids []int64
+ var (
+ payload []*scdmodels.OperationalIntent
+ cids []int64
+ ovn pgtype.Text
+ pastOVNs []string
+ )
  ussAvailabilities := map[dssmodels.Manager]scdmodels.UssAvailabilityState{}
  for rows.Next() {
  var (
@@ -79,11 +86,27 @@ func (s *repo) fetchOperationalIntents(ctx context.Context, q dsssql.Queryable,
  &updatedAt,
  &o.State,
  &cids,
+ &ovn,
+ &pastOVNs,
  )
  if err != nil {
  return nil, stacktrace.Propagate(err, "Error scanning Operation row")
  }
- o.OVN = scdmodels.NewOVNFromTime(updatedAt, o.ID.String())
+
+ // If the managing USS has requested a specific OVN on this operational intent, it will be persisted in DB.
+ // If not, a default DSS-generated OVN based on the last update time is used.
+ // See https://github.com/interuss/dss/issues/1078 for more details.
+ if ovn.Valid {
+ o.OVN = scdmodels.OVN(ovn.String)
+ } else {
+ o.OVN = scdmodels.NewOVNFromTime(updatedAt, o.ID.String())
+ }
+
+ o.PastOVNs = make([]scdmodels.OVN, 0, len(pastOVNs))
+ for _, pastOVN := range pastOVNs {
+ o.PastOVNs = append(o.PastOVNs, scdmodels.OVN(pastOVN))
+ }
+
  o.SetCells(cids)
  ussAvailabilities[o.Manager] = scdmodels.UssAvailabilityStateUnknown
  payload = append(payload, o)
@@ -177,7 +200,7 @@ func (s *repo) UpsertOperationalIntent(ctx context.Context, operation *scdmodels
  scd_operations
  (%s)
  VALUES
- ($1, $2, $3, $4, $5, $6, $7, $8, $9, transaction_timestamp(), $10, $11)
+ ($1, $2, $3, $4, $5, $6, $7, $8, $9, transaction_timestamp(), $10, $11, $12, $13)
  RETURNING
  %s`, operationFieldsWithoutPrefix, operationFieldsWithPrefix)
  )
@@ -199,6 +222,15 @@ func (s *repo) UpsertOperationalIntent(ctx context.Context, operation *scdmodels
  if err != nil {
  return nil, stacktrace.Propagate(err, "Failed to convert id to PgUUID")
  }
+
+ var ovn pgtype.Text // if the OVN is to be generated by the DSS, it must be NULL in DB, not just an empty string
+ if operation.OVN != "" {
+ ovn = pgtype.Text{
+ String: operation.OVN.String(),
+ Valid: true,
+ }
+ }
+
  operation, err = s.fetchOperationalIntent(ctx, s.q, upsertOperationsQuery,
  opid,
  operation.Manager,
@@ -211,6 +243,8 @@ func (s *repo) UpsertOperationalIntent(ctx context.Context, operation *scdmodels
  subid,
  operation.State,
  cids,
+ ovn,
+ operation.PastOVNs,
  )
  if err != nil {
  return nil, stacktrace.Propagate(err, "Error fetching Operation")