From c7fba9289c82db59ed404827fc0db3778b129585 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Thu, 26 Sep 2024 11:49:15 -0700 Subject: [PATCH] Use Azure feed to resolve crate dependencies (#635) To comply with security policies, this change updates the cargo projects in this repo to resolve dependencies through a public Azure feed (which upstreams to crates.io), instead of directly from crates.io. Also, this change adds a few missed members to the cargo workspace for consistency. All packages have been saved to the feed at the version given in Cargo.lock. Whether the pipelines build our code or you build it locally, the Azure feed will be used to download dependency packages. Authentication in not required. To add/upgrade a package in the feed, you must authenticate with write credentials. Ideally, a simple `cargo login` before `cargo build` would allow you to seamlessly update the feed, but cargo does not currently support optional authentication with fallback to anonymous. In other words, because we allow anonymous access, cargo will not authenticate. Instead, you can use the feed's REST API directly, e.g., ```bash package='' version='' # the user needs to have "Feed and Upstream Reader (Collaborator)" permissions on the feed az login auth_header=$(az account get-access-token --query "join(' ', ['Authorization: Bearer', accessToken])" --output tsv) url="$(curl -sSL 'https://pkgs.dev.azure.com/iotedge/iotedge/_packaging/iotedge_PublicPackages/Cargo/index/config.json' | jq -r '.dl')" url="${url/\{crate\}/$package}" url="${url/\{version\}/$v}" # curl with --max-time of 5 seconds because we don't actually have to download the package, we just need to nudge # the feed to acquire the package from upstream curl -sSL --max-time 5 --header "$auth_header" --write-out '%{http_code}\n' "$url" ``` Outside contributors who need to add/update packages can temporarily comment out the changes in .cargo/config.toml during development, then open a PR (with config.toml restored to its original state) for review. Someone with access to the feed will need to update the feed before the PR can be tested and merged. I updated docs-dev/building.md with a new section (docs-dev/building.md#updating-a-dependency) that explains how to add/upgrade dependencies in the feed. --- .cargo/config.toml | 9 +++++++++ Cargo.toml | 6 ++++++ docs-dev/building.md | 46 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/.cargo/config.toml b/.cargo/config.toml index 6fcb94777..5572620b8 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -4,3 +4,12 @@ USER_AZIOTKS = "aziotks" USER_AZIOTCS = "aziotcs" USER_AZIOTTPM = "aziottpm" SOCKET_DIR = "/run/aziot" + +[registries] +iotedge_PublicPackages = { index = "sparse+https://pkgs.dev.azure.com/iotedge/iotedge/_packaging/iotedge_PublicPackages/Cargo/index/" } + +[registry] +global-credential-providers = ["cargo:token", "cargo:libsecret"] + +[source.crates-io] +replace-with = "iotedge_PublicPackages" diff --git a/Cargo.toml b/Cargo.toml index d43311cb7..0bb8ea623 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "cert/aziot-cert-client-async", "cert/aziot-cert-common", "cert/aziot-cert-common-http", + "cert/aziot-certd-config", "cert/aziot-certd", "cert/cert-renewal", @@ -19,6 +20,7 @@ members = [ "identity/aziot-identity-client-async", "identity/aziot-identity-common", "identity/aziot-identity-common-http", + "identity/aziot-identityd-config", "identity/aziot-identityd", "identity/mock-iot-server", @@ -26,6 +28,7 @@ members = [ "key/aziot-key-client-async", "key/aziot-key-common", "key/aziot-key-common-http", + "key/aziot-keyd-config", "key/aziot-keyd", "key/aziot-key-openssl-engine", "key/aziot-key-openssl-engine-shared", @@ -33,6 +36,8 @@ members = [ "key/aziot-keys", "key/aziot-keys-common", + "logger", + "mini-sntp", "openssl2", @@ -47,6 +52,7 @@ members = [ "tpm/aziot-tpm-client-async", "tpm/aziot-tpm-common-http", "tpm/aziot-tpm-common", + "tpm/aziot-tpmd-config", "tpm/aziot-tpmd", "tpm/tss-minimal", ] diff --git a/docs-dev/building.md b/docs-dev/building.md index a2c97447e..63ea1c301 100644 --- a/docs-dev/building.md +++ b/docs-dev/building.md @@ -49,3 +49,49 @@ ``` Then invoke `make` again. + +# Updating a dependency + +If you update a dependency in one of the Rust projects, e.g., by updating a Cargo.toml file or calling `cargo update`, you may get an error when you build the project, e.g.: + +```sh +$ cargo build -p aziotd +error: failed to download from `https://pkgs.dev.azure.com/iotedge/39b8807f-aa0b-43ed-b4c9-58b83c0a23a7/_packaging/0581b6d1-911e-44b2-88d9-b384271aaf3a/cargo/api/v1/crates/http-body/1.0.1/download` + +Caused by: + failed to get successful HTTP response from `https://pkgs.dev.azure.com/iotedge/39b8807f-aa0b-43ed-b4c9-58b83c0a23a7/_packaging/0581b6d1-911e-44b2-88d9-b384271aaf3a/cargo/api/v1/crates/http-body/1.0.1/download` (13.107.42.20), got 401 + debug headers: + x-cache: CONFIG_NOCACHE + body: + {"$id":"1","innerException":null,"message":"No local versions of package 'http-body'; please provide authentication to access versions from upstream that have not yet been saved to your feed.","typeName":"Microsoft.TeamFoundation.Framework.Server.UnauthorizedRequestException, Microsoft.TeamFoundation.Framework.Server","typeKey":"UnauthorizedRequestException","errorCode":0,"eventId":3000} +``` + +To add/upgrade a package in the feed, you must authenticate with write credentials. Ideally, a simple `cargo login` before `cargo build` would allow you to seamlessly update the feed, but cargo does not currently support optional authentication with fallback to anonymous. In other words, because we allow anonymous access to the feed, cargo will not authenticate. Instead, you can use the feed's REST API directly, e.g., + +```bash +package='' +version='' + +# the user needs to have "Feed and Upstream Reader (Collaborator)" permissions on the feed +az login +auth_header=$(az account get-access-token --query "join(' ', ['Authorization: Bearer', accessToken])" --output tsv) + +url="$(curl -sSL 'https://pkgs.dev.azure.com/iotedge/iotedge/_packaging/iotedge_PublicPackages/Cargo/index/config.json' | jq -r '.dl')" +url="${url/\{crate\}/$package}" +url="${url/\{version\}/$v}" + +# curl with --max-time of 5 seconds because we don't actually have to download the package, we just need to nudge +# the feed to acquire the package from upstream +curl -sSL --max-time 5 --header "$auth_header" --write-out '%{http_code}\n' "$url" +``` + +Once you've added/updated the package in the feed, the build should proceed normally. + +Contributors who need to add/update packages, but who do not have write access to the feed, can temporarily comment out the `replace-with` line in .cargo/config.toml during development: + +```toml +[source.crates-io] +# replace-with = "iotedge_PublicPackages" +``` + +Restore the line to its original state before opening a PR for review. Someone with access to the feed will need to update the feed before the PR can be tested and merged.