[rid] Add v2 endpoints

Makefile

@@ -39,6 +39,7 @@ format:
  clang-format -style=file -i pkg/api/v1/scdpb/scd.proto
  clang-format -style=file -i pkg/api/v1/auxpb/aux_service.proto
  cd monitoring/uss_qualifier && make format
+ gofmt -s -w .
 
 .PHONY: lint
 lint: go_lint shell_lint

README.md

@@ -8,7 +8,9 @@ framework to test UAS Service Suppliers (USS). See the [InterUSS website](https:
 The DSS implementation and associated monitoring tools target compliance with the following standards and regulations:
 
 - [ASTM F3411-19](https://www.astm.org/Standards/F3411.htm): Remote ID.
-[See OpenAPI interface](./interfaces/rid/v1/remoteid)
+ - [F3411-19](./interfaces/rid/v1/remoteid)
+ - [F3411-xx](./interfaces/rid/v2/remoteid)
+ - See [documentation](./interfaces/rid/README.md) before mixing versions in a single ecosystem.
 - [ASTM WK63418](https://www.astm.org/DATABASE.CART/WORKITEMS/WK63418.htm): UAS Traffic Management (UTM) UAS
 Service Supplier (USS) Interoperability Specification.
 [See OpenAPI interface](./interfaces/astm-utm)

cmds/core-service/main.go

@@ -18,6 +18,7 @@ import (
  "github.com/interuss/dss/pkg/api/v1/auxpb"
  "github.com/interuss/dss/pkg/api/v1/ridpbv1"
  "github.com/interuss/dss/pkg/api/v1/scdpb"
+ "github.com/interuss/dss/pkg/api/v2/ridpbv2"
  "github.com/interuss/dss/pkg/auth"
  aux "github.com/interuss/dss/pkg/aux_"
  "github.com/interuss/dss/pkg/build"
@@ -26,7 +27,8 @@ import (
  uss_errors "github.com/interuss/dss/pkg/errors"
  "github.com/interuss/dss/pkg/logging"
  application "github.com/interuss/dss/pkg/rid/application"
- ridserverv1 "github.com/interuss/dss/pkg/rid/server/v1"
+ rid_v1 "github.com/interuss/dss/pkg/rid/server/v1"
+ rid_v2 "github.com/interuss/dss/pkg/rid/server/v2"
  ridc "github.com/interuss/dss/pkg/rid/store/cockroach"
  "github.com/interuss/dss/pkg/scd"
  scdc "github.com/interuss/dss/pkg/scd/store/cockroach"
@@ -105,16 +107,16 @@ func createKeyResolver() (auth.KeyResolver, error) {
  }
 }
 
-func createRIDServer(ctx context.Context, locality string, logger *zap.Logger) (*ridserverv1.Server, error) {
+func createRIDServer(ctx context.Context, locality string, logger *zap.Logger) (*rid_v1.Server, *rid_v2.Server, error) {
  connectParameters := flags.ConnectParameters()
  connectParameters.DBName = "rid"
  ridCrdb, err := cockroach.Dial(ctx, connectParameters)
  if err != nil {
  // TODO: More robustly detect failure to create RID server is due to a problem that may be temporary
  if strings.Contains(err.Error(), "connect: connection refused") {
- return nil, stacktrace.PropagateWithCode(err, codeRetryable, "Failed to connect to CRDB server for remote ID store")
+ return nil, nil, stacktrace.PropagateWithCode(err, codeRetryable, "Failed to connect to CRDB server for remote ID store")
  }
- return nil, stacktrace.Propagate(err, "Failed to connect to remote ID database; verify your database configuration is current with https://github.com/interuss/dss/tree/master/build#upgrading-database-schemas")
+ return nil, nil, stacktrace.Propagate(err, "Failed to connect to remote ID database; verify your database configuration is current with https://github.com/interuss/dss/tree/master/build#upgrading-database-schemas")
  }
 
  ridStore, err := ridc.NewStore(ctx, ridCrdb, connectParameters.DBName, logger)
@@ -124,45 +126,52 @@ func createRIDServer(ctx context.Context, locality string, logger *zap.Logger) (
  connectParameters.DBName = "defaultdb"
  ridCrdb, err := cockroach.Dial(ctx, connectParameters)
  if err != nil {
- return nil, stacktrace.Propagate(err, "Failed to connect to remote ID database for older version <defaultdb>; verify your database configuration is current with https://github.com/interuss/dss/tree/master/build#upgrading-database-schemas")
+ return nil, nil, stacktrace.Propagate(err, "Failed to connect to remote ID database for older version <defaultdb>; verify your database configuration is current with https://github.com/interuss/dss/tree/master/build#upgrading-database-schemas")
  }
  ridStore, err = ridc.NewStore(ctx, ridCrdb, connectParameters.DBName, logger)
  if err != nil {
  // TODO: More robustly detect failure to create RID server is due to a problem that may be temporary
  if strings.Contains(err.Error(), "connect: connection refused") || strings.Contains(err.Error(), "database has not been bootstrapped with Schema Manager") {
  ridCrdb.Pool.Close()
- return nil, stacktrace.PropagateWithCode(err, codeRetryable, "Failed to connect to CRDB server for remote ID store")
+ return nil, nil, stacktrace.PropagateWithCode(err, codeRetryable, "Failed to connect to CRDB server for remote ID store")
  }
- return nil, stacktrace.Propagate(err, "Failed to create remote ID store")
+ return nil, nil, stacktrace.Propagate(err, "Failed to create remote ID store")
  }
  }
 
  repo, err := ridStore.Interact(ctx)
  if err != nil {
- return nil, stacktrace.Propagate(err, "Unable to interact with store")
+ return nil, nil, stacktrace.Propagate(err, "Unable to interact with store")
  }
  gc := ridc.NewGarbageCollector(repo, locality)
 
  // schedule period tasks for RID Server
  ridCron := cron.New()
  // schedule printing of DB connection stats every minute for the underlying storage for RID Server
  if _, err := ridCron.AddFunc("@every 1m", func() { getDBStats(ctx, ridCrdb, connectParameters.DBName) }); err != nil {
- return nil, stacktrace.Propagate(err, "Failed to schedule periodic db stat check to %s", connectParameters.DBName)
+ return nil, nil, stacktrace.Propagate(err, "Failed to schedule periodic db stat check to %s", connectParameters.DBName)
  }
 
  cronLogger := cron.VerbosePrintfLogger(log.New(os.Stdout, "RIDGarbageCollectorJob: ", log.LstdFlags))
  if _, err = ridCron.AddJob(*garbageCollectorSpec, cron.NewChain(cron.SkipIfStillRunning(cronLogger)).Then(RIDGarbageCollectorJob{"delete rid expired records", *gc, ctx})); err != nil {
- return nil, stacktrace.Propagate(err, "Failed to schedule periodic delete rid expired records to %s", connectParameters.DBName)
+ return nil, nil, stacktrace.Propagate(err, "Failed to schedule periodic delete rid expired records to %s", connectParameters.DBName)
  }
  ridCron.Start()
 
- return &ridserverv1.Server{
- App: application.NewFromTransactor(ridStore, logger),
- Timeout: *timeout,
- Locality: locality,
- EnableHTTP: *enableHTTP,
- Cron: ridCron,
- }, nil
+ app := application.NewFromTransactor(ridStore, logger)
+ return &rid_v1.Server{
+ App: app,
+ Timeout: *timeout,
+ Locality: locality,
+ EnableHTTP: *enableHTTP,
+ Cron: ridCron,
+ }, &rid_v2.Server{
+ App: app,
+ Timeout: *timeout,
+ Locality: locality,
+ EnableHTTP: *enableHTTP,
+ Cron: ridCron,
+ }, nil
 }
 
 func createSCDServer(ctx context.Context, logger *zap.Logger) (*scd.Server, error) {
@@ -211,20 +220,25 @@ func RunGRPCServer(ctx context.Context, ctxCanceler func(), address string, loca
  }
 
  var (
- ridServerV1 *ridserverv1.Server
+ ridServerV1 *rid_v1.Server
+ ridServerV2 *rid_v2.Server
  scdServer *scd.Server
  auxServer = &aux.Server{}
  )
 
  // Initialize remote ID
- server, err := createRIDServer(ctx, locality, logger)
+ serverV1, serverV2, err := createRIDServer(ctx, locality, logger)
  if err != nil {
  return stacktrace.Propagate(err, "Failed to create remote ID server")
  }
- ridServerV1 = server
+ ridServerV1 = serverV1
+ ridServerV2 = serverV2
 
  scopesValidators := auth.MergeOperationsAndScopesValidators(
- ridServerV1.AuthScopes(), auxServer.AuthScopes(),
+ ridServerV1.AuthScopes(), ridServerV2.AuthScopes(),
+ )
+ scopesValidators = auth.MergeOperationsAndScopesValidators(
+ scopesValidators, auxServer.AuthScopes(),
  )
 
  // Initialize strategic conflict detection
@@ -233,6 +247,7 @@ func RunGRPCServer(ctx context.Context, ctxCanceler func(), address string, loca
  server, err := createSCDServer(ctx, logger)
  if err != nil {
  ridServerV1.Cron.Stop()
+ ridServerV2.Cron.Stop()
  return stacktrace.Propagate(err, "Failed to create strategic conflict detection server")
  }
  scdServer = server
@@ -285,6 +300,7 @@ func RunGRPCServer(ctx context.Context, ctxCanceler func(), address string, loca
  logger.Info("build", zap.Any("description", build.Describe()))
 
  ridpbv1.RegisterDiscoveryAndSynchronizationServiceServer(s, ridServerV1)
+ ridpbv2.RegisterStandardRemoteIDAPIInterfacesServiceServer(s, ridServerV2)
  auxpb.RegisterDSSAuxServiceServer(s, auxServer)
  if *enableSCD {
  logger.Info("config", zap.Any("scd", "enabled"))

cmds/http-gateway/main.go

@@ -18,6 +18,7 @@ import (
  "github.com/interuss/dss/pkg/api/v1/auxpb"
  "github.com/interuss/dss/pkg/api/v1/ridpbv1"
  "github.com/interuss/dss/pkg/api/v1/scdpb"
+ "github.com/interuss/dss/pkg/api/v2/ridpbv2"
  "github.com/interuss/dss/pkg/build"
  "github.com/interuss/dss/pkg/errors"
  "github.com/interuss/dss/pkg/logging"
@@ -74,13 +75,21 @@ func RunHTTPProxy(ctx context.Context, ctxCanceler func(), address, endpoint str
  grpc.WithTimeout(10 * time.Second),
  }
 
- logger.Info("Registering RID service")
+ logger.Info("Registering RID v1 service")
  if err := ridpbv1.RegisterDiscoveryAndSynchronizationServiceHandlerFromEndpoint(ctx, grpcMux, endpoint, opts); err != nil {
  // TODO: More robustly detect failure to create RID server is due to a problem that may be temporary
  if strings.Contains(err.Error(), "context deadline exceeded") {
- return stacktrace.PropagateWithCode(err, codeRetryable, "Failed to connect to core-service for remote ID")
+ return stacktrace.PropagateWithCode(err, codeRetryable, "Failed to connect to core-service for remote ID v1")
  }
- return stacktrace.Propagate(err, "Error registering RID service handler")
+ return stacktrace.Propagate(err, "Error registering RID v1 service handler")
+ }
+
+ logger.Info("Registering RID v2 service")
+ if err := ridpbv2.RegisterStandardRemoteIDAPIInterfacesServiceHandlerFromEndpoint(ctx, grpcMux, endpoint, opts); err != nil {
+ if strings.Contains(err.Error(), "context deadline exceeded") {
+ return stacktrace.PropagateWithCode(err, codeRetryable, "Failed to connect to core-service for remote ID v2")
+ }
+ return stacktrace.Propagate(err, "Error registering RID v2 service handler")
  }
 
  logger.Info("Registering aux service")

interfaces/rid/README.md

@@ -0,0 +1,49 @@
+# ASTM F3411 Network remote identification
+
+## F3411-19
+
+[OpenAPI specification](v1/remoteid/augmented.yaml)
+
+## F3411-xx (second version)
+
+[OpenAPI specification](v2/remoteid/canonical.yaml)
+
+## Mixing versions in a single ecosystem
+
+If all USSs in an ecosystem use the v1 API, then everything is fine. If all USSs in an ecosystem use the v2 API, then everything is fine. If some USSs in an ecosystem use v1 while others use v2, there may be interoperability problems. To avoid accidentally missing ISAs, this DSS implementation stores v1 and v2 ISAs alongside each other. The URL field for v1 ISAs contains the `/flights` resource URL (e.g., `https://example.com/v1/uss/flights`), but this same URL field contains the base URL (e.g., `http://example.com/rid/v2`) for v2 ISAs. This means a v1 USS may try to query `http://example.com/rid/v2` if reading a v2 USS's ISA, or a v2 USS may try to query `http://example.com/v1/uss/flights/uss/flights` if reading a v1 USS's ISA. This issue is somewhat intentional because even though v1 and v2 both have a `/flights` endpoint, the communications protocol for these two endpoints is not compatible. If v1 and v2 ISAs are going to co-exist in the same ecosystem, then every USS in that ecosystem must infer the USS-USS communications protocol based on the content of the URL field (`flights_url` and `identification_service_area_url` in v1 and `uss_base_url` in v2).
+
+### v1 ISAs
+
+If a USS in a mixed-version ecosystem reads a v1 ISA, it must communicate with the managing USS in the following ways:
+
+| If the `flights_url` | Then reach `/flights` URL at | Using data exchange protocol |
+| --- | --- | --- |
+| Ends with "/flights" | `flights_url` without any changes | [F3411-19 (v1)](https://github.com/uastech/standards/blob/36e7ea23a010ff91053f82ac4f6a9bfc698503f9/remoteid/canonical.yaml#L325) |
+| Does not end with "/flights" | `flights_url` + "[/uss/flights](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L338)" | [F3411-xx (v2)](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L338) |
+
+### v1 SubscriberToNotify
+
+If a USS in a mixed-version ecosystem makes a change to a v1 ISA, the response will [contain](https://github.com/uastech/standards/blob/36e7ea23a010ff91053f82ac4f6a9bfc698503f9/remoteid/canonical.yaml#L1263) a list of SubscriberToNotify. The POST notification should be sent differently depending on the contents of the [`url` field](https://github.com/uastech/standards/blob/36e7ea23a010ff91053f82ac4f6a9bfc698503f9/remoteid/canonical.yaml#L1330):
+
+| If the `url` | Then reach `/identification_service_areas` URL at | Using data exchange protocol |
+| --- | --- | --- |
+| Ends with "/identification_service_areas" | `url` + "/" + ISA ID | [F3411-19 (v1)](https://github.com/uastech/standards/blob/36e7ea23a010ff91053f82ac4f6a9bfc698503f9/remoteid/canonical.yaml#L767) |
+| Does not end with "/identification_service_areas" | `url` + ["/uss/identification_service_areas/" + ISA ID](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L778) | [F3411-xx (v2)](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L822) |
+
+### v2 ISAs
+
+If a USS in a mixed-version ecosystem reads a v2 ISA, it must communicate with the managing USS in the following ways:
+
+| If the `uss_base_url` | Then reach `/flights` URL at | Using data exchange protocol |
+| --- | --- | --- |
+| Ends with "/flights" | `uss_base_url` without any changes | [F3411-19 (v1)](https://github.com/uastech/standards/blob/36e7ea23a010ff91053f82ac4f6a9bfc698503f9/remoteid/canonical.yaml#L325) |
+| Does not end with "/flights" | `uss_base_url` + "[/uss/flights](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L338)" | [F3411-xx (v2)](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L338) |
+
+### v2 SubscriberToNotify
+
+If a USS in a mixed-version ecosystem makes a change to a v2 ISA, the response will [contain](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L1475) a list of SubscriberToNotify. The POST notification should be sent differently depending on the contents of the [`url` field](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L1545):
+
+| If the `url` | Then reach `/identification_service_areas` URL at | Using data exchange protocol |
+| --- | --- | --- |
+| Ends with "/identification_service_areas" | `url` + "/" + ISA ID | [F3411-19 (v1)](https://github.com/uastech/standards/blob/36e7ea23a010ff91053f82ac4f6a9bfc698503f9/remoteid/canonical.yaml#L767) |
+| Does not end with "/identification_service_areas" | `url` + ["/uss/identification_service_areas/" + ISA ID](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L778) | [F3411-xx (v2)](https://github.com/uastech/standards/blob/ab6037442d97e868183ed60d35dab4954d9f15d0/remoteid/canonical.yaml#L822) |

monitoring/monitorlib/rid_v2.py

@@ -0,0 +1,39 @@
+import datetime
+from typing import Dict, Literal
+
+from monitoring.monitorlib.typing import ImplicitDict, StringBasedDateTime
+from . import rid as rid_v1
+
+
+ISA_PATH = '/dss/identification_service_areas'
+SUBSCRIPTION_PATH = '/dss/subscriptions'
+SCOPE_DP = 'rid.display_provider'
+SCOPE_SP = 'rid.service_provider'
+
+
+class Time(ImplicitDict):
+ value: StringBasedDateTime
+ format: Literal['RFC3339']
+
+ @classmethod
+ def make(cls, t: datetime.datetime):
+ return Time(format='RFC3339', value=t.strftime(DATE_FORMAT))
+
+
+class Altitude(ImplicitDict):
+ reference: Literal['WGS84']
+ units: Literal['M']
+ value: float
+
+ @classmethod
+ def make(cls, altitude_meters: float):
+ return Altitude(reference='WGS84', units='M', value=altitude_meters)
+
+
+MAX_SUB_PER_AREA = rid_v1.MAX_SUB_PER_AREA
+MAX_SUB_TIME_HRS = rid_v1.MAX_SUB_TIME_HRS
+DATE_FORMAT = rid_v1.DATE_FORMAT
+NetMaxNearRealTimeDataPeriod = rid_v1.NetMaxNearRealTimeDataPeriod
+NetMaxDisplayAreaDiagonal = rid_v1.NetMaxDisplayAreaDiagonal
+NetDetailsMaxDisplayAreaDiagonal = rid_v1.NetDetailsMaxDisplayAreaDiagonal
+geo_polygon_string = rid_v1.geo_polygon_string

monitoring/prober/conftest.py

@@ -9,10 +9,12 @@
 
 
 OPT_RID_AUTH = 'rid_auth'
+OPT_RID_V2_AUTH = 'rid_v2_auth'
 OPT_SCD_AUTH1 = 'scd_auth1'
 OPT_SCD_AUTH2 = 'scd_auth2'
 
 BASE_URL_RID = ''
+BASE_URL_RID_V2 = '/rid/v2'
 BASE_URL_SCD = '/dss/v1'
 BASE_URL_AUX = '/aux/v1'
 
@@ -37,10 +39,16 @@ def pytest_addoption(parser):
 
  parser.addoption(
  '--rid-auth',
- help='Auth spec (see Authorization section of README.md) for performing remote ID actions in the DSS',
+ help='Auth spec (see Authorization section of README.md) for performing remote ID v1 actions in the DSS',
  metavar='SPEC',
  dest='rid_auth')
 
+ parser.addoption(
+ '--rid-v2-auth',
+ help='Auth spec (see Authorization section of README.md) for performing remote ID v2 actions in the DSS',
+ metavar='SPEC',
+ dest='rid_v2_auth')
+
  parser.addoption(
  '--scd-auth1',
  help='Auth spec (see Authorization section of README.md) for performing primary strategic deconfliction actions in the DSS',
@@ -120,6 +128,11 @@ def session_ridv1(pytestconfig) -> UTMClientSession:
  return make_session(pytestconfig, BASE_URL_RID, OPT_RID_AUTH)
 
 
+@pytest.fixture(scope='session')
+def session_ridv2(pytestconfig) -> UTMClientSession:
+ return make_session(pytestconfig, BASE_URL_RID_V2, OPT_RID_V2_AUTH)
+
+
 @pytest.fixture(scope='session')
 def session_ridv1_async(pytestconfig):
  session = make_session_async(pytestconfig, BASE_URL_RID, OPT_RID_AUTH)
@@ -207,6 +220,11 @@ def no_auth_session_ridv1(pytestconfig) -> UTMClientSession:
  return make_session(pytestconfig, BASE_URL_RID)
 
 
+@pytest.fixture(scope='function')
+def no_auth_session_ridv2(pytestconfig) -> UTMClientSession:
+ return make_session(pytestconfig, BASE_URL_RID_V2)
+
+
 @pytest.fixture(scope='session')
 def scd_api(pytestconfig) -> str:
  api = pytestconfig.getoption('scd_api_version')

monitoring/prober/infrastructure.py

@@ -86,7 +86,7 @@ def wrapper_default_scope(*args, **kwargs):
 resource_type_code_descriptions: Dict[ResourceType, str] = {}
 
 
-# Next code: 347
+# Next code: 367
 def register_resource_type(code: int, description: str) -> ResourceType:
  """Register that the specified code refers to the described resource.
 

monitoring/prober/rid/v2/common.py

@@ -0,0 +1,8 @@
+from monitoring.prober.rid.v1 import common as common_v1
+
+
+VERTICES = common_v1.VERTICES
+GEO_POLYGON_STRING = common_v1.GEO_POLYGON_STRING
+HUGE_GEO_POLYGON_STRING = common_v1.HUGE_GEO_POLYGON_STRING
+LOOP_GEO_POLYGON_STRING = common_v1.LOOP_GEO_POLYGON_STRING
+HUGE_VERTICES = common_v1.HUGE_VERTICES