fix: ensure upload of bcc installation logs

.github/renovate.json

@@ -87,6 +87,13 @@
     {
       "matchPackageNames": ["moby-runc", "moby-containerd"],
       "extractVersion": "^v?(?<version>.+)$"
+    },
+    {      
+      "matchPackageNames": ["aks/aks-gpu-cuda", "aks/aks-gpu-grid"],      
+      "versioning": "regex:^(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)-(?<prerelease>\\d{14})$",
+      "automerge": false,      
+      "enabled": true,
+      "ignoreUnstable": false
     }
   ],
   "customManagers": [

.github/workflows/check-coverage.yml

@@ -6,13 +6,6 @@ permissions:
   id-token: write
   contents: read
 
-env:
-  SUBSCRIPTION_ID: "8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8"
-  RESOURCE_GROUP_NAME: "agentbaker-e2e-tests"
-  LOCATION: "eastus"
-  CLUSTER_NAME: "agentbaker-e2e-test-cluster"
-  AZURE_TENANT_ID: "72f988bf-86f1-41af-91ab-2d7cd011db47"
-
 jobs:
   unit_tests:
     runs-on: ubuntu-latest

.pipelines/.vsts-vhd-builder-release.yaml

@@ -1308,7 +1308,7 @@ stages:
               useOverrides: ${{ parameters.useOverrides }}
               overrideBranch: ${{ parameters.overrideBranch }}
               artifactName: 2204-minimal-gen2-containerd
-  - stage: Run_E2E_Tests
+  - stage: e2e
     condition: and(ne(variables.SKIP_E2E_TESTS, 'true'), eq('${{ parameters.dryrun }}', false))
     variables:
       VHD_BUILD_ID: $(Build.BuildId)

.pipelines/.vsts-vhd-builder.yaml

@@ -1,4 +1,4 @@
-name: $(Date:yyyyMMdd)$(Rev:.r)_$(OS_SKU)$(OS_VERSION)_$(HYPERV_GENERATION)_$(FEATURE_FLAGS)_$(Build.SourceBranchName)
+name: $(Date:yyyyMMdd)$(Rev:.r)_$(Build.SourceBranchName)_$(BuildID)
 trigger: none
 pr:
   branches:
@@ -97,7 +97,7 @@ stages:
         - template: ./templates/.builder-release-template.yaml
           parameters:
             artifactName: marinerv2-gen2
-  - stage: Run_E2E_Tests
+  - stage: e2e
     condition: ne(variables.SKIP_E2E_TESTS, 'true')
     variables:
       VHD_BUILD_ID: $(Build.BuildId)

.pipelines/templates/.builder-release-template.yaml

@@ -126,6 +126,7 @@ steps:
   - bash: |
       CAPTURED_SIG_VERSION="$(cat vhdbuilder/packer/settings.json | grep "captured_sig_version" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       SIG_IMAGE_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_image_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
+      ARTIFACT_ID=$(date +%s) && \
       echo "##vso[task.setvariable variable=VHD_NAME]${CAPTURED_SIG_VERSION}.vhd" && \
       echo "##vso[task.setvariable variable=IMAGE_NAME]${SIG_IMAGE_NAME}-${CAPTURED_SIG_VERSION}" && \
       echo "##vso[task.setvariable variable=SIG_IMAGE_NAME]${SIG_IMAGE_NAME}" && \
@@ -146,6 +147,7 @@ steps:
       echo "##vso[task.setvariable variable=BRANCH]$(Build.SourceBranch)" && \
       echo "##vso[task.setvariable variable=GIT_BRANCH]$(Build.SourceBranch)" && \
       echo "##vso[task.setvariable variable=JOB_STATUS]$(Agent.JobStatus)" && \
+      echo "##vso[task.setvariable variable=ARTIFACT_ID]${ARTIFACT_ID}" && \
       echo "##vso[task.setvariable variable=VHD_DEBUG]False"
     condition: always()
     displayName: Set Pipeline Variables
@@ -164,7 +166,7 @@ steps:
     condition: always()
     displayName: Publish BCC Tools Installation Log
     inputs:
-      artifactName: 'bcc-tools-installation-log-${{ parameters.artifactName }}'
+      artifactName: 'bcc-tools-installation-log-$(ARTIFACT_ID)-${{ parameters.artifactName }}'
       targetPath: 'bcc-tools-installation.log'
 
   - task: CopyFiles@2

.pipelines/templates/e2e-template.yaml

@@ -1,7 +1,7 @@
 jobs:
-  - job: E2E_Tests
+  - job: e2e
     pool:
-      name: nodesigtest-pool
+      name: $(POOL_NAME)
     timeoutInMinutes: 90
     displayName: Run AgentBaker E2E
     variables:
@@ -10,7 +10,7 @@ jobs:
       - bash: |
           set -ex
           az login --identity
-          az account set -s $(AZURE_SUBSCRIPTION_ID)
+          az account set -s $(SUBSCRIPTION_ID)
         displayName: Azure login
       - bash: bash .pipelines/scripts/setup_go.sh
         displayName: Setup go
@@ -32,7 +32,7 @@ jobs:
             export SIG_VERSION_TAG_VALUE=$VHD_BUILD_ID
             export IGNORE_SCENARIOS_WITH_MISSING_VHD=true
           else
-            echo "Build.BuildId is not specified. Running default e2e tests."
+            echo "VHD_BUILD_ID is not specified. Running tests with default SIG version tag selectors."
           fi
 
           cd e2e

README.md

@@ -136,4 +136,3 @@ Reference: https://docs.opensource.microsoft.com/tools/cg/cgmanifest.html
 Package:
 
 - Calico Windows: https://docs.projectcalico.org/release-notes/
-

e2e/config/azure.go

@@ -57,16 +57,16 @@ type AzureClient struct {
 	VirutalNetworkLinksClient *armprivatedns.VirtualNetworkLinksClient
 }
 
-func mustNewAzureClient(subscription string) *AzureClient {
-	client, err := NewAzureClient(subscription)
+func mustNewAzureClient() *AzureClient {
+	client, err := NewAzureClient()
 	if err != nil {
 		panic(err)
 	}
 	return client
 
 }
 
-func NewAzureClient(subscription string) (*AzureClient, error) {
+func NewAzureClient() (*AzureClient, error) {
 	httpClient := &http.Client{
 		// use a bunch of connections for load balancing
 		// ensure all timeouts are defined and reasonable
@@ -128,87 +128,87 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 		return nil, fmt.Errorf("create core client: %w", err)
 	}
 
-	cloud.RegistriesClient, err = armcontainerregistry.NewRegistriesClient(subscription, credential, opts)
+	cloud.RegistriesClient, err = armcontainerregistry.NewRegistriesClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create registry client: %w", err)
 	}
 
-	cloud.CacheRulesClient, err = armcontainerregistry.NewCacheRulesClient(subscription, credential, opts)
+	cloud.CacheRulesClient, err = armcontainerregistry.NewCacheRulesClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create cache rules client: %w", err)
 	}
 
-	cloud.PrivateEndpointClient, err = armnetwork.NewPrivateEndpointsClient(subscription, credential, opts)
+	cloud.PrivateEndpointClient, err = armnetwork.NewPrivateEndpointsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create private endpoint client: %w", err)
 	}
 
-	cloud.PrivateZonesClient, err = armprivatedns.NewPrivateZonesClient(subscription, credential, opts)
+	cloud.PrivateZonesClient, err = armprivatedns.NewPrivateZonesClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create private dns zones client: %w", err)
 	}
 
-	cloud.VirutalNetworkLinksClient, err = armprivatedns.NewVirtualNetworkLinksClient(subscription, credential, opts)
+	cloud.VirutalNetworkLinksClient, err = armprivatedns.NewVirtualNetworkLinksClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create virtual network links client: %w", err)
 	}
 
-	cloud.RecordSetClient, err = armprivatedns.NewRecordSetsClient(subscription, credential, opts)
+	cloud.RecordSetClient, err = armprivatedns.NewRecordSetsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create record set client: %w", err)
 	}
 
-	cloud.PrivateDNSZoneGroup, err = armnetwork.NewPrivateDNSZoneGroupsClient(subscription, credential, opts)
+	cloud.PrivateDNSZoneGroup, err = armnetwork.NewPrivateDNSZoneGroupsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create private dns zone group client: %w", err)
 	}
 
-	cloud.SecurityGroup, err = armnetwork.NewSecurityGroupsClient(subscription, credential, opts)
+	cloud.SecurityGroup, err = armnetwork.NewSecurityGroupsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create security group client: %w", err)
 	}
 
-	cloud.Subnet, err = armnetwork.NewSubnetsClient(subscription, credential, opts)
+	cloud.Subnet, err = armnetwork.NewSubnetsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create subnet client: %w", err)
 	}
 
-	cloud.AKS, err = armcontainerservice.NewManagedClustersClient(subscription, credential, opts)
+	cloud.AKS, err = armcontainerservice.NewManagedClustersClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create aks client: %w", err)
 	}
 
-	cloud.Maintenance, err = armcontainerservice.NewMaintenanceConfigurationsClient(subscription, credential, opts)
+	cloud.Maintenance, err = armcontainerservice.NewMaintenanceConfigurationsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create maintenance client: %w", err)
 	}
 
-	cloud.VMSS, err = armcompute.NewVirtualMachineScaleSetsClient(subscription, credential, opts)
+	cloud.VMSS, err = armcompute.NewVirtualMachineScaleSetsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create vmss client: %w", err)
 	}
 
-	cloud.VMSSVM, err = armcompute.NewVirtualMachineScaleSetVMsClient(subscription, credential, opts)
+	cloud.VMSSVM, err = armcompute.NewVirtualMachineScaleSetVMsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create vmss vm client: %w", err)
 	}
 
-	cloud.Resource, err = armresources.NewClient(subscription, credential, opts)
+	cloud.Resource, err = armresources.NewClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create resource client: %w", err)
 	}
 
-	cloud.ResourceGroup, err = armresources.NewResourceGroupsClient(subscription, credential, opts)
+	cloud.ResourceGroup, err = armresources.NewResourceGroupsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create resource group client: %w", err)
 	}
 
-	cloud.VNet, err = armnetwork.NewVirtualNetworksClient(subscription, credential, opts)
+	cloud.VNet, err = armnetwork.NewVirtualNetworksClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create vnet client: %w", err)
 	}
 
-	cloud.GalleryImageVersion, err = armcompute.NewGalleryImageVersionsClient(subscription, credential, opts)
+	cloud.GalleryImageVersion, err = armcompute.NewGalleryImageVersionsClient(Config.GallerySubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create a new images client: %v", err)
 	}

e2e/config/config.go

@@ -1,6 +1,8 @@
 package config
 
 import (
+	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
@@ -10,34 +12,37 @@ import (
 
 var (
 	Config            = mustLoadConfig()
-	Azure             = mustNewAzureClient(Config.SubscriptionID)
+	Azure             = mustNewAzureClient()
 	ResourceGroupName = "abe2e-" + Config.Location
 	VMIdentityName    = "abe2e-vm-identity"
-	PrivateACRName    = "privateacre2e"
+	PrivateACRName    = "privateacre2e" + Config.Location
 
 	DefaultPollUntilDoneOptions = &runtime.PollUntilDoneOptions{
 		Frequency: time.Second,
 	}
 )
 
 type Configuration struct {
-	AirgapNSGName                 string        `env:"AIRGAP_NSG_NAME" envDefault:"abe2e-airgap-securityGroup"`
-	DefaultSubnetName             string        `env:"DEFAULT_SUBNET_NAME" envDefault:"aks-subnet"`
-	BuildID                       string        `env:"BUILD_ID" envDefault:"local"`
-	Location                      string        `env:"LOCATION" envDefault:"westus3"`
-	SubscriptionID                string        `env:"SUBSCRIPTION_ID" envDefault:"8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8"`
-	SIGVersionTagName             string        `env:"SIG_VERSION_TAG_NAME" envDefault:"branch"`
-	SIGVersionTagValue            string        `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/master"`
-	TagsToRun                     string        `env:"TAGS_TO_RUN"`
-	TagsToSkip                    string        `env:"TAGS_TO_SKIP"`
-	TestTimeout                   time.Duration `env:"TEST_TIMEOUT" envDefault:"35m"`
-	E2ELoggingDir                 string        `env:"LOGGING_DIR" envDefault:"scenario-logs"`
-	IgnoreScenariosWithMissingVHD bool          `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"`
-	SkipTestsWithSKUCapacityIssue bool          `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"`
-	KeepVMSS                      bool          `env:"KEEP_VMSS"`
-	BlobStorageAccountPrefix      string        `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e"`
-	BlobContainer                 string        `env:"BLOB_CONTAINER" envDefault:"abe2e"`
-	EnableNodeBootstrapperTest    bool          `env:"ENABLE_NODE_BOOTSTRAPPER_TEST"`
+	AirgapNSGName                 string        `env:"AIRGAP_NSG_NAME" envDefault:"abe2e-airgap-securityGroup" json:"airgapNSGName"`
+	DefaultSubnetName             string        `env:"DEFAULT_SUBNET_NAME" envDefault:"aks-subnet" json:"defaultSubnetName"`
+	BuildID                       string        `env:"BUILD_ID" envDefault:"local" json:"buildID"`
+	Location                      string        `env:"LOCATION" envDefault:"westus3" json:"location"`
+	SubscriptionID                string        `env:"SUBSCRIPTION_ID" envDefault:"8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8" json:"subscriptionID"`
+	GallerySubscriptionID         string        `env:"GALLERY_SUBSCRIPTION_ID" envDefault:"c4c3550e-a965-4993-a50c-628fd38cd3e1" json:"gallerySubscriptionID"`
+	GalleryResourceGroupName      string        `env:"GALLERY_RESOURCE_GROUP_NAME" envDefault:"aksvhdtestbuildrg" json:"galleryResourceGroupName"`
+	GalleryName                   string        `env:"GALLERY_NAME" envDefault:"PackerSigGalleryEastUS" json:"galleryName"`
+	SIGVersionTagName             string        `env:"SIG_VERSION_TAG_NAME" envDefault:"branch" json:"sigVersionTagName"`
+	SIGVersionTagValue            string        `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/master" json:"sigVersionTagValue"`
+	TagsToRun                     string        `env:"TAGS_TO_RUN" json:"tagsToRun"`
+	TagsToSkip                    string        `env:"TAGS_TO_SKIP" json:"tagsToSkip"`
+	TestTimeout                   time.Duration `env:"TEST_TIMEOUT" envDefault:"35m" json:"testTimeout"`
+	E2ELoggingDir                 string        `env:"LOGGING_DIR" envDefault:"scenario-logs" json:"e2eLoggingDir"`
+	IgnoreScenariosWithMissingVHD bool          `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD" json:"ignoreScenariosWithMissingVHD"`
+	SkipTestsWithSKUCapacityIssue bool          `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE" json:"skipTestsWithSKUCapacityIssue"`
+	KeepVMSS                      bool          `env:"KEEP_VMSS" json:"keepVMSS"`
+	BlobStorageAccountPrefix      string        `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e" json:"blobStorageAccountPrefix"`
+	BlobContainer                 string        `env:"BLOB_CONTAINER" envDefault:"abe2e" json:"blobContainer"`
+	EnableNodeBootstrapperTest    bool          `env:"ENABLE_NODE_BOOTSTRAPPER_TEST" json:"enableNodeBootstrapperTest"`
 }
 
 func (c *Configuration) BlobStorageAccount() string {
@@ -48,6 +53,18 @@ func (c *Configuration) BlobStorageAccountURL() string {
 	return "https://" + c.BlobStorageAccount() + ".blob.core.windows.net"
 }
 
+func (c *Configuration) GalleryResourceID() string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/galleries/%s", c.GallerySubscriptionID, c.GalleryResourceGroupName, c.GalleryName)
+}
+
+func (c Configuration) String() string {
+	content, err := json.MarshalIndent(c, "", "	")
+	if err != nil {
+		panic(err)
+	}
+	return string(content)
+}
+
 func mustLoadConfig() Configuration {
 	_ = godotenv.Load(".env")
 	cfg := Configuration{}

e2e/config/vhd.go

@@ -14,7 +14,6 @@ import (
 )
 
 const (
-	imageGallery       = "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/"
 	noSelectionTagName = "abe2e-ignore"
 )
 
@@ -64,6 +63,7 @@ var (
 	// this is a particular 2204gen2containerd image originally built with private packages,
 	// if we ever want to update this then we'd need to run a new VHD build using private package overrides
 	VHDUbuntu2204Gen2ContainerdPrivateKubePkg = &Image{
+		// 2204Gen2 is a special image definition holding historical VHDs used by agentbaker e2e's.
 		Name:    "2204Gen2",
 		OS:      "ubuntu",
 		Arch:    "amd64",
@@ -73,7 +73,7 @@ var (
 
 	// without kubelet, kubectl, credential-provider and wasm
 	VHDUbuntu2204Gen2ContainerdAirgapped = &Image{
-		Name:    "2204gen2containerd",
+		Name:    "2204Gen2",
 		OS:      "ubuntu",
 		Arch:    "amd64",
 		Version: "1.1725612526.29638",
@@ -102,7 +102,7 @@ func (i *Image) String() string {
 
 func (i *Image) VHDResourceID(ctx context.Context, t *testing.T) (VHDResourceID, error) {
 	i.vhdOnce.Do(func() {
-		imageDefinitionResourceID := imageGallery + i.Name
+		imageDefinitionResourceID := fmt.Sprintf("%s/images/%s", Config.GalleryResourceID(), i.Name)
 		if i.Version != "" {
 			i.vhd, i.vhdErr = ensureStaticSIGImageVersion(ctx, t, imageDefinitionResourceID+"/versions/"+i.Version)
 		} else {

e2e/scenario_test.go

@@ -16,12 +16,14 @@ import (
 )
 
 func TestMain(m *testing.M) {
+	fmt.Printf("using E2E environment configuration:\n%s\n", config.Config)
 	// delete scenario-logs folder if it exists
 	if _, err := os.Stat("scenario-logs"); err == nil {
 		_ = os.RemoveAll("scenario-logs")
 	}
 	m.Run()
 }
+
 func Test_azurelinuxv2(t *testing.T) {
 	RunScenario(t, &Scenario{
 		Description: "Tests that a node using a AzureLinuxV2 (CgroupV2) VHD can be properly bootstrapped",