Skip to content

Commit

Permalink
[node-bootstrapper] set ENV variables without intermediate template s…
Browse files Browse the repository at this point in the history
…tep (#5176)
  • Loading branch information
r2k1 authored Oct 29, 2024
1 parent f0b1d57 commit 0f260a8
Show file tree
Hide file tree
Showing 5 changed files with 263 additions and 361 deletions.
30 changes: 12 additions & 18 deletions node-bootstrapper/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
Expand All @@ -12,7 +13,7 @@ import (
"os/exec"

"github.com/Azure/agentbaker/node-bootstrapper/parser"
"github.com/Azure/agentbaker/node-bootstrapper/utils"
nbcontractv1 "github.com/Azure/agentbaker/pkg/proto/nbcontract/v1"
)

type App struct {
Expand Down Expand Up @@ -67,30 +68,23 @@ func (a *App) Provision(ctx context.Context, flags ProvisionFlags) error {
return fmt.Errorf("open proision file %s: %w", flags.ProvisionConfig, err)
}

cseCmd, err := parser.Parse(inputJSON)
config := &nbcontractv1.Configuration{}
err = json.Unmarshal(inputJSON, config)
if err != nil {
return fmt.Errorf("parse config: %w", err)
return fmt.Errorf("unmarshal provision config: %w", err)
}

if err := a.provisionStart(ctx, cseCmd); err != nil {
return fmt.Errorf("provision start: %w", err)
if config.Version != "v0" {
return fmt.Errorf("unsupported version: %s", config.Version)
}
return nil
}

func (a *App) provisionStart(ctx context.Context, cse utils.SensitiveString) error {
// CSEScript can't be logged because it contains sensitive information.
slog.Info("Running CSE script")
//nolint:gosec // we generate the script, so it's safe to execute
cmd := exec.CommandContext(ctx, "/bin/bash", "-c", cse.UnsafeValue())
cmd.Dir = "/"
cmd, err := parser.BuildCSECmd(ctx, config)
if err != nil {
return fmt.Errorf("build CSE command: %w", err)
}
var stdoutBuf, stderrBuf bytes.Buffer
// We want to preserve the original stdout and stderr to avoid any issues during migration to the "scriptless" approach
// RP may rely on stdout and stderr for error handling
// it's also nice to have a single log file for all the important information, so write to both places
cmd.Stdout = io.MultiWriter(os.Stdout, &stdoutBuf)
cmd.Stderr = io.MultiWriter(os.Stderr, &stderrBuf)
err := a.cmdRunner(cmd)
err = a.cmdRunner(cmd)
exitCode := -1
if cmd.ProcessState != nil {
exitCode = cmd.ProcessState.ExitCode()
Expand Down
55 changes: 2 additions & 53 deletions node-bootstrapper/parser/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,59 +51,8 @@ var (

func getFuncMap() template.FuncMap {
return template.FuncMap{
"derefString": deref[string],
"derefBool": deref[bool],
"getStringFromVMType": getStringFromVMType,
"getStringFromNetworkPluginType": getStringFromNetworkPluginType,
"getStringFromNetworkPolicyType": getStringFromNetworkPolicyType,
"getStringFromLoadBalancerSkuType": getStringFromLoadBalancerSkuType,
"getKubenetTemplate": getKubenetTemplate,
"getSysctlContent": getSysctlContent,
"getUlimitContent": getUlimitContent,
"getContainerdConfig": getContainerdConfig,
"getStringifiedStringArray": getStringifiedStringArray,
"getIsMIGNode": getIsMIGNode,
"getCustomCACertsStatus": getCustomCACertsStatus,
"getEnableTLSBootstrap": getEnableTLSBootstrap,
"getEnableSecureTLSBootstrap": getEnableSecureTLSBootstrap,
"getTLSBootstrapToken": getTLSBootstrapToken,
"getCustomSecureTLSBootstrapAADServerAppID": getCustomSecureTLSBootstrapAADServerAppID,
"getIsKrustlet": getIsKrustlet,
"getEnsureNoDupePromiscuousBridge": getEnsureNoDupePromiscuousBridge,
"getHasSearchDomain": getHasSearchDomain,
"getCSEHelpersFilepath": getCSEHelpersFilepath,
"getCSEDistroHelpersFilepath": getCSEDistroHelpersFilepath,
"getCSEInstallFilepath": getCSEInstallFilepath,
"getCSEDistroInstallFilepath": getCSEDistroInstallFilepath,
"getCSEConfigFilepath": getCSEConfigFilepath,
"getCustomSearchDomainFilepath": getCustomSearchDomainFilepath,
"getDHCPV6ConfigFilepath": getDHCPV6ConfigFilepath,
"getDHCPV6ServiceFilepath": getDHCPV6ServiceFilepath,
"getShouldConfigContainerdUlimits": getShouldConfigContainerdUlimits,
"createSortedKeyValueStringPairs": createSortedKeyValuePairs[string],
"createSortedKeyValueInt32Pairs": createSortedKeyValuePairs[int32],
"getExcludeMasterFromStandardLB": getExcludeMasterFromStandardLB,
"getMaxLBRuleCount": getMaxLBRuleCount,
"getGpuImageSha": getGpuImageSha,
"getGpuDriverVersion": getGpuDriverVersion,
"getIsSgxEnabledSKU": getIsSgxEnabledSKU,
"getShouldConfigureHTTPProxy": getShouldConfigureHTTPProxy,
"getShouldConfigureHTTPProxyCA": getShouldConfigureHTTPProxyCA,
"getAzureEnvironmentFilepath": getAzureEnvironmentFilepath,
"getLinuxAdminUsername": getLinuxAdminUsername,
"getTargetEnvironment": getTargetEnvironment,
"getIsVHD": getIsVHD,
"getDisableSSH": getDisableSSH,
"getServicePrincipalFileContent": getServicePrincipalFileContent,
"getEnableSwapConfig": getEnableSwapConfig,
"getShouldConfigTransparentHugePage": getShouldConfigTransparentHugePage,
"getProxyVariables": getProxyVariables,
"getHasKubeletDiskType": getHasKubeletDiskType,
"getInitAKSCustomCloudFilepath": getInitAKSCustomCloudFilepath,
"getTargetCloud": getTargetCloud,
"getIsAksCustomCloud": getIsAksCustomCloud,
"getGPUNeedsFabricManager": getGPUNeedsFabricManager,
"getEnableNvidia": getEnableNvidia,
"getInitAKSCustomCloudFilepath": getInitAKSCustomCloudFilepath,
"getIsAksCustomCloud": getIsAksCustomCloud,
}
}

Expand Down
173 changes: 156 additions & 17 deletions node-bootstrapper/parser/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ package parser

import (
"bytes"
"context"
_ "embed"
"encoding/json"
"fmt"
"os/exec"
"sort"
"strings"
"text/template"

"github.com/Azure/agentbaker/node-bootstrapper/utils"
nbcontractv1 "github.com/Azure/agentbaker/pkg/proto/nbcontract/v1"
)

Expand All @@ -27,25 +28,163 @@ func executeBootstrapTemplate(inputContract *nbcontractv1.Configuration) (string
return buffer.String(), nil
}

// this function will eventually take a pointer to the bootstrap contract struct.
// it will then template out the variables into the final bootstrap trigger script.
func Parse(inputJSON []byte) (utils.SensitiveString, error) {
// Parse the JSON into a nbcontractv1.Configuration struct
var nbc nbcontractv1.Configuration
err := json.Unmarshal(inputJSON, &nbc)
if err != nil {
return "", fmt.Errorf("failed to unmarshal the json to nbcontractv1: %w", err)
}
func getCSEEnv(config *nbcontractv1.Configuration) map[string]string {
env := make(map[string]string)

if nbc.Version != "v0" {
return "", fmt.Errorf("unsupported version: %s", nbc.Version)
env["PROVISION_OUTPUT"] = "/var/log/azure/cluster-provision.log;"
env["MOBY_VERSION"] = ""
env["CLOUDPROVIDER_BACKOFF"] = "true"
env["CLOUDPROVIDER_BACKOFF_MODE"] = "v2"
env["CLOUDPROVIDER_BACKOFF_RETRIES"] = "6"
env["CLOUDPROVIDER_BACKOFF_EXPONENT"] = "0"
env["CLOUDPROVIDER_BACKOFF_DURATION"] = "5"
env["CLOUDPROVIDER_BACKOFF_JITTER"] = "0"
env["CLOUDPROVIDER_RATELIMIT"] = "true"
env["CLOUDPROVIDER_RATELIMIT_QPS"] = "10"
env["CLOUDPROVIDER_RATELIMIT_QPS_WRITE"] = "10"
env["CLOUDPROVIDER_RATELIMIT_BUCKET"] = "100"
env["CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE"] = "100"
env["CONTAINER_RUNTIME"] = " containerd"
env["CLI_TOOL"] = "ctr"
env["NETWORK_MODE"] = "transparent"
env["NEEDS_CONTAINERD"] = "true"
env["NEEDS_DOCKER_LOGIN"] = "false"

env["ADMINUSER"] = getLinuxAdminUsername(config.GetLinuxAdminUsername())
env["TENANT_ID"] = config.AuthConfig.GetTenantId()
env["KUBERNETES_VERSION"] = config.GetKubernetesVersion()
env["KUBE_BINARY_URL"] = config.KubeBinaryConfig.GetKubeBinaryUrl()
env["CUSTOM_KUBE_BINARY_URL"] = config.KubeBinaryConfig.GetCustomKubeBinaryUrl()
env["PRIVATE_KUBE_BINARY_URL"] = config.KubeBinaryConfig.GetPrivateKubeBinaryUrl()
env["KUBEPROXY_URL"] = config.GetKubeProxyUrl()
env["APISERVER_PUBLIC_KEY"] = config.ApiServerConfig.GetApiServerPublicKey()
env["SUBSCRIPTION_ID"] = config.AuthConfig.GetSubscriptionId()
env["RESOURCE_GROUP"] = config.ClusterConfig.GetResourceGroup()
env["LOCATION"] = config.ClusterConfig.GetLocation()
env["VM_TYPE"] = getStringFromVMType(config.ClusterConfig.GetVmType())
env["SUBNET"] = config.ClusterConfig.GetClusterNetworkConfig().GetSubnet()
env["NETWORK_SECURITY_GROUP"] = config.ClusterConfig.GetClusterNetworkConfig().GetSecurityGroupName()
env["VIRTUAL_NETWORK"] = config.ClusterConfig.GetClusterNetworkConfig().GetVnetName()
env["VIRTUAL_NETWORK_RESOURCE_GROUP"] = config.ClusterConfig.GetClusterNetworkConfig().GetVnetResourceGroup()
env["ROUTE_TABLE"] = config.ClusterConfig.GetClusterNetworkConfig().GetRouteTable()
env["PRIMARY_AVAILABILITY_SET"] = config.ClusterConfig.GetPrimaryAvailabilitySet()
env["PRIMARY_SCALE_SET"] = config.ClusterConfig.GetPrimaryScaleSet()
env["SERVICE_PRINCIPAL_CLIENT_ID"] = config.AuthConfig.GetServicePrincipalId()
env["NETWORK_PLUGIN"] = getStringFromNetworkPluginType(config.GetNetworkConfig().GetNetworkPlugin())
env["NETWORK_POLICY"] = getStringFromNetworkPolicyType(config.GetNetworkConfig().GetNetworkPolicy())
env["VNET_CNI_PLUGINS_URL"] = config.GetNetworkConfig().GetVnetCniPluginsUrl()
env["LOAD_BALANCER_DISABLE_OUTBOUND_SNAT"] = fmt.Sprintf("%v", config.ClusterConfig.GetLoadBalancerConfig().GetDisableOutboundSnat())
env["USE_MANAGED_IDENTITY_EXTENSION"] = fmt.Sprintf("%v", config.AuthConfig.GetUseManagedIdentityExtension())
env["USE_INSTANCE_METADATA"] = fmt.Sprintf("%v", config.ClusterConfig.GetUseInstanceMetadata())
env["LOAD_BALANCER_SKU"] = getStringFromLoadBalancerSkuType(config.ClusterConfig.GetLoadBalancerConfig().GetLoadBalancerSku())
env["EXCLUDE_MASTER_FROM_STANDARD_LB"] = fmt.Sprintf("%v", getExcludeMasterFromStandardLB(config.ClusterConfig.GetLoadBalancerConfig()))
env["MAXIMUM_LOADBALANCER_RULE_COUNT"] = fmt.Sprintf("%v", getMaxLBRuleCount(config.ClusterConfig.GetLoadBalancerConfig()))
env["CONTAINERD_DOWNLOAD_URL_BASE"] = config.ContainerdConfig.GetContainerdDownloadUrlBase()
env["USER_ASSIGNED_IDENTITY_ID"] = config.AuthConfig.GetAssignedIdentityId()
env["API_SERVER_NAME"] = config.ApiServerConfig.GetApiServerName()
env["IS_VHD"] = fmt.Sprintf("%v", getIsVHD(config.IsVhd))
env["GPU_NODE"] = fmt.Sprintf("%v", getEnableNvidia(config))
env["SGX_NODE"] = fmt.Sprintf("%v", getIsSgxEnabledSKU(config.VmSize))
env["MIG_NODE"] = fmt.Sprintf("%v", getIsMIGNode(config.GpuConfig.GetGpuInstanceProfile()))
env["CONFIG_GPU_DRIVER_IF_NEEDED"] = fmt.Sprintf("%v", config.GpuConfig.GetConfigGpuDriver())
env["ENABLE_GPU_DEVICE_PLUGIN_IF_NEEDED"] = fmt.Sprintf("%v", config.GpuConfig.GetGpuDevicePlugin())
env["TELEPORTD_PLUGIN_DOWNLOAD_URL"] = config.TeleportConfig.GetTeleportdPluginDownloadUrl()
env["CREDENTIAL_PROVIDER_DOWNLOAD_URL"] = config.KubeBinaryConfig.GetLinuxCredentialProviderUrl()
env["CONTAINERD_VERSION"] = config.ContainerdConfig.GetContainerdVersion()
env["CONTAINERD_PACKAGE_URL"] = config.ContainerdConfig.GetContainerdPackageUrl()
env["RUNC_VERSION"] = config.RuncConfig.GetRuncVersion()
env["RUNC_PACKAGE_URL"] = config.RuncConfig.GetRuncPackageUrl()
env["ENABLE_HOSTS_CONFIG_AGENT"] = fmt.Sprintf("%v", config.GetEnableHostsConfigAgent())
env["DISABLE_SSH"] = fmt.Sprintf("%v", getDisableSSH(config))
env["TELEPORT_ENABLED"] = fmt.Sprintf("%v", config.TeleportConfig.GetStatus())
env["SHOULD_CONFIGURE_HTTP_PROXY"] = fmt.Sprintf("%v", getShouldConfigureHTTPProxy(config.HttpProxyConfig))
env["SHOULD_CONFIGURE_HTTP_PROXY_CA"] = fmt.Sprintf("%v", getShouldConfigureHTTPProxyCA(config.HttpProxyConfig))
env["HTTP_PROXY_TRUSTED_CA"] = config.HttpProxyConfig.GetProxyTrustedCa()
env["SHOULD_CONFIGURE_CUSTOM_CA_TRUST"] = fmt.Sprintf("%v", getCustomCACertsStatus(config.GetCustomCaCerts()))
env["CUSTOM_CA_TRUST_COUNT"] = fmt.Sprintf("%v", len(config.GetCustomCaCerts()))
for i, cert := range config.CustomCaCerts {
env[fmt.Sprintf("CUSTOM_CA_CERT_%d", i)] = cert
}
env["IS_KRUSTLET"] = fmt.Sprintf("%v", getIsKrustlet(config.GetWorkloadRuntime()))
env["GPU_NEEDS_FABRIC_MANAGER"] = fmt.Sprintf("%v", getGPUNeedsFabricManager(config.VmSize))
env["IPV6_DUAL_STACK_ENABLED"] = fmt.Sprintf("%v", config.GetIpv6DualStackEnabled())
env["OUTBOUND_COMMAND"] = config.GetOutboundCommand()
env["ENABLE_UNATTENDED_UPGRADES"] = fmt.Sprintf("%v", config.GetEnableUnattendedUpgrade())
env["ENSURE_NO_DUPE_PROMISCUOUS_BRIDGE"] = fmt.Sprintf("%v", getEnsureNoDupePromiscuousBridge(config.GetNetworkConfig()))
env["SHOULD_CONFIG_SWAP_FILE"] = fmt.Sprintf("%v", getEnableSwapConfig(config.CustomLinuxOsConfig))
env["SHOULD_CONFIG_TRANSPARENT_HUGE_PAGE"] = fmt.Sprintf("%v", getShouldConfigTransparentHugePage(config.CustomLinuxOsConfig))
env["SHOULD_CONFIG_CONTAINERD_ULIMITS"] = fmt.Sprintf("%v", getShouldConfigContainerdUlimits(config.CustomLinuxOsConfig.GetUlimitConfig()))
env["CONTAINERD_ULIMITS"] = getUlimitContent(config.CustomLinuxOsConfig.GetUlimitConfig())
env["TARGET_CLOUD"] = getTargetCloud(config)
env["TARGET_ENVIRONMENT"] = getTargetEnvironment(config)
env["CUSTOM_ENV_JSON"] = config.CustomCloudConfig.GetCustomEnvJsonContent()
env["IS_CUSTOM_CLOUD"] = fmt.Sprintf("%v", getIsAksCustomCloud(config.CustomCloudConfig))
env["AKS_CUSTOM_CLOUD_CONTAINER_REGISTRY_DNS_SUFFIX"] = config.CustomCloudConfig.GetContainerRegistryDnsSuffix()
env["CSE_HELPERS_FILEPATH"] = getCSEHelpersFilepath()
env["CSE_DISTRO_HELPERS_FILEPATH"] = getCSEDistroHelpersFilepath()
env["CSE_INSTALL_FILEPATH"] = getCSEInstallFilepath()
env["CSE_DISTRO_INSTALL_FILEPATH"] = getCSEDistroInstallFilepath()
env["CSE_CONFIG_FILEPATH"] = getCSEConfigFilepath()
env["AZURE_PRIVATE_REGISTRY_SERVER"] = config.GetAzurePrivateRegistryServer()
env["HAS_CUSTOM_SEARCH_DOMAIN"] = fmt.Sprintf("%v", getHasSearchDomain(config.GetCustomSearchDomainConfig()))
env["CUSTOM_SEARCH_DOMAIN_FILEPATH"] = getCustomSearchDomainFilepath()
env["HTTP_PROXY_URLS"] = config.HttpProxyConfig.GetHttpProxy()
env["HTTPS_PROXY_URLS"] = config.HttpProxyConfig.GetHttpsProxy()
env["NO_PROXY_URLS"] = getStringifiedStringArray(config.HttpProxyConfig.GetNoProxyEntries(), ",")
env["PROXY_VARS"] = getProxyVariables(config.HttpProxyConfig)
env["ENABLE_TLS_BOOTSTRAPPING"] = fmt.Sprintf("%v", getEnableTLSBootstrap(config.TlsBootstrappingConfig))
env["ENABLE_SECURE_TLS_BOOTSTRAPPING"] = fmt.Sprintf("%v", getEnableSecureTLSBootstrap(config.TlsBootstrappingConfig))
env["CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID"] = getCustomSecureTLSBootstrapAADServerAppID(config.TlsBootstrappingConfig)
env["DHCPV6_SERVICE_FILEPATH"] = getDHCPV6ServiceFilepath()
env["DHCPV6_CONFIG_FILEPATH"] = getDHCPV6ConfigFilepath()
env["THP_ENABLED"] = config.CustomLinuxOsConfig.GetTransparentHugepageSupport()
env["THP_DEFRAG"] = config.CustomLinuxOsConfig.GetTransparentDefrag()
env["SERVICE_PRINCIPAL_FILE_CONTENT"] = getServicePrincipalFileContent(config.AuthConfig)
env["KUBELET_CLIENT_CONTENT"] = config.KubeletConfig.GetKubeletClientKey()
env["KUBELET_CLIENT_CONTENT"] = config.KubeletConfig.GetKubeletClientKey()
env["KUBELET_CLIENT_CERT_CONTENT"] = config.KubeletConfig.GetKubeletClientCertContent()
env["KUBELET_CONFIG_FILE_ENABLED"] = fmt.Sprintf("%v", config.KubeletConfig.GetEnableKubeletConfigFile())
env["KUBELET_CONFIG_FILE_CONTENT"] = config.KubeletConfig.GetKubeletConfigFileContent()
env["SWAP_FILE_SIZE_MB"] = fmt.Sprintf("%v", config.CustomLinuxOsConfig.GetSwapFileSize())
env["GPU_DRIVER_VERSION"] = getGpuDriverVersion(config.VmSize)
env["GPU_IMAGE_SHA"] = getGpuImageSha(config.VmSize)
env["GPU_INSTANCE_PROFILE"] = config.GpuConfig.GetGpuInstanceProfile()
env["CUSTOM_SEARCH_DOMAIN_NAME"] = config.CustomSearchDomainConfig.GetDomainName()
env["CUSTOM_SEARCH_REALM_USER"] = config.CustomSearchDomainConfig.GetRealmUser()
env["CUSTOM_SEARCH_REALM_PASSWORD"] = config.CustomSearchDomainConfig.GetRealmPassword()
env["MESSAGE_OF_THE_DAY"] = config.GetMessageOfTheDay()
env["HAS_KUBELET_DISK_TYPE"] = fmt.Sprintf("%v", getHasKubeletDiskType(config.KubeletConfig))
env["NEEDS_CGROUPV2"] = fmt.Sprintf("%v", config.GetNeedsCgroupv2())
env["TLS_BOOTSTRAP_TOKEN"] = getTLSBootstrapToken(config.TlsBootstrappingConfig)
env["KUBELET_FLAGS"] = createSortedKeyValuePairs(config.KubeletConfig.GetKubeletFlags(), " ")
env["NETWORK_POLICY"] = getStringFromNetworkPolicyType(config.NetworkConfig.GetNetworkPolicy())
env["KUBELET_NODE_LABELS"] = createSortedKeyValuePairs(config.KubeletConfig.GetKubeletNodeLabels(), ",")
env["AZURE_ENVIRONMENT_FILEPATH"] = getAzureEnvironmentFilepath(config)
env["KUBE_CA_CRT"] = config.GetKubernetesCaCert()
env["KUBENET_TEMPLATE"] = getKubenetTemplate()
env["CONTAINERD_CONFIG_CONTENT"] = getContainerdConfig(config)
env["IS_KATA"] = fmt.Sprintf("%v", config.GetIsKata())
env["ARTIFACT_STREAMING_ENABLED"] = fmt.Sprintf("%v", config.GetEnableArtifactStreaming())
env["SYSCTL_CONTENT"] = getSysctlContent(config.CustomLinuxOsConfig.GetSysctlConfig())
env["PRIVATE_EGRESS_PROXY_ADDRESS"] = config.GetPrivateEgressProxyAddress()
env["BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER"] = config.GetBootstrapProfileContainerRegistryServer()
env["ENABLE_IMDS_RESTRICTION"] = fmt.Sprintf("%v", config.ImdsRestrictionConfig.GetEnableImdsRestriction())
env["INSERT_IMDS_RESTRICTION_RULE_TO_MANGLE_TABLE"] = fmt.Sprintf("%v", config.ImdsRestrictionConfig.GetInsertImdsRestrictionRuleToMangleTable())

triggerBootstrapScript, err := executeBootstrapTemplate(&nbc)
return env
}

func BuildCSECmd(ctx context.Context, config *nbcontractv1.Configuration) (*exec.Cmd, error) {
triggerBootstrapScript, err := executeBootstrapTemplate(config)
if err != nil {
return "", fmt.Errorf("failed to execute the template: %w", err)
return nil, fmt.Errorf("failed to execute the template: %w", err)
}

// Convert to one-liner
return utils.SensitiveString(strings.ReplaceAll(triggerBootstrapScript, "\n", " ")), nil
triggerBootstrapScript = strings.ReplaceAll(triggerBootstrapScript, "\n", " ")
cmd := exec.CommandContext(ctx, "/bin/bash", "-c", triggerBootstrapScript)
for k, v := range getCSEEnv(config) {
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
}
sort.Strings(cmd.Env) // produce deterministic output
return cmd, nil
}
Loading

0 comments on commit 0f260a8

Please sign in to comment.