DEVPROD-6193 Use IRSA to sign mciuploads bucket urls

agent/command/s3_put.go

@@ -125,7 +125,8 @@ type s3put struct {
 	isPatchable      bool
 	isPatchOnly      bool
 
-	bucket pail.Bucket
+	bucket              pail.Bucket
+	devprodOwnedBuckets []string
 
 	taskdata client.TaskData
 	base
@@ -312,6 +313,8 @@ func (s3pc *s3put) Execute(ctx context.Context,
 		attribute.String(s3PutRemotePathAttribute, s3pc.remoteFile),
 	)
 
+	s3pc.devprodOwnedBuckets = conf.DevProdOwnedBuckets
+
 	// create pail bucket
 	httpClient := utility.GetHTTPClient()
 	httpClient.Timeout = s3HTTPClientTimeout
@@ -543,10 +546,11 @@ func (s3pc *s3put) attachFiles(ctx context.Context, comm client.Communicator, lo
 		}
 		var key, secret, bucket, fileKey string
 		if s3pc.Visibility == artifact.Signed {
-			key = s3pc.AwsKey
-			secret = s3pc.AwsSecret
 			bucket = s3pc.Bucket
 			fileKey = remoteFileName
+			// TODO (DEVPROD-13658): Check if the bucket is devprod owned and do not send the credentials if it is.
+			key = s3pc.AwsKey
+			secret = s3pc.AwsSecret
 		}
 
 		files = append(files, &artifact.File{

agent/internal/task_config.go

@@ -60,6 +60,10 @@ type TaskConfig struct {
 	// message of a version to be used in the otel attributes.
 	PatchOrVersionDescription string
 
+	// DevProdOwnedBuckets holds the list of buckets that are owned by
+	// DevProd.
+	DevProdOwnedBuckets []string
+
 	mu sync.RWMutex
 }
 

agent/task_context.go

@@ -372,6 +372,7 @@ func (a *Agent) makeTaskConfig(ctx context.Context, tc *taskContext) (*internal.
 	taskConfig.TaskSync = a.opts.SetupData.TaskSync
 	taskConfig.EC2Keys = a.opts.SetupData.EC2Keys
 	taskConfig.MaxExecTimeoutSecs = a.opts.SetupData.MaxExecTimeoutSecs
+	taskConfig.DevProdOwnedBuckets = a.opts.SetupData.DevProdOwnedBuckets
 
 	// Set AWS credentials for task output buckets.
 	awsCreds := pail.CreateAWSCredentials(taskConfig.TaskOutput.Key, taskConfig.TaskOutput.Secret, "")

apimodels/agent_models.go

@@ -122,6 +122,7 @@ type AgentSetupData struct {
 	EC2Keys                []evergreen.EC2Key      `json:"ec2_keys"`
 	TraceCollectorEndpoint string                  `json:"trace_collector_endpoint"`
 	MaxExecTimeoutSecs     int                     `json:"max_exec_timeout_secs"`
+	DevProdOwnedBuckets    []string                `json:"dev_prod_owned_buckets"`
 }
 
 // NextTaskResponse represents the response sent back when an agent asks for a next task

config.go

@@ -37,7 +37,7 @@ var (
 
 	// Agent version to control agent rollover. The format is the calendar date
 	// (YYYY-MM-DD).
-	AgentVersion = "2024-12-10"
+	AgentVersion = "2024-12-18"
 )
 
 const (
@@ -75,10 +75,12 @@ type Settings struct {
 	ConfigDir           string                    `yaml:"configdir" bson:"configdir" json:"configdir"`
 	ContainerPools      ContainerPoolsConfig      `yaml:"container_pools" bson:"container_pools" json:"container_pools" id:"container_pools"`
 	Database            DBSettings                `yaml:"database" json:"database" bson:"database"`
+	DevProdOwnedBuckets []string                  `yaml:"dev_prod_owned_buckets" bson:"dev_prod_owned_buckets" json:"dev_prod_owned_buckets"`
 	DomainName          string                    `yaml:"domain_name" bson:"domain_name" json:"domain_name"`
 	Expansions          map[string]string         `yaml:"expansions" bson:"expansions" json:"expansions"`
 	ExpansionsNew       util.KeyValuePairSlice    `yaml:"expansions_new" bson:"expansions_new" json:"expansions_new"`
 	GithubPRCreatorOrg  string                    `yaml:"github_pr_creator_org" bson:"github_pr_creator_org" json:"github_pr_creator_org"`
+	GitHubCheckRun      GitHubCheckRunConfig      `yaml:"github_check_run" bson:"github_check_run" json:"github_check_run" id:"github_check_run"`
 	GithubOrgs          []string                  `yaml:"github_orgs" bson:"github_orgs" json:"github_orgs"`
 	GithubWebhookSecret string                    `yaml:"github_webhook_secret" bson:"github_webhook_secret" json:"github_webhook_secret"`
 	DisabledGQLQueries  []string                  `yaml:"disabled_gql_queries" bson:"disabled_gql_queries" json:"disabled_gql_queries"`
@@ -102,19 +104,18 @@ type Settings struct {
 	RuntimeEnvironments RuntimeEnvironmentsConfig `yaml:"runtime_environments" bson:"runtime_environments" json:"runtime_environments" id:"runtime_environments"`
 	Scheduler           SchedulerConfig           `yaml:"scheduler" bson:"scheduler" json:"scheduler" id:"scheduler"`
 	ServiceFlags        ServiceFlags              `bson:"service_flags" json:"service_flags" id:"service_flags" yaml:"service_flags"`
-	SSHKeyDirectory     string                    `yaml:"ssh_key_directory" bson:"ssh_key_directory" json:"ssh_key_directory"`
-	SSHKeyPairs         []SSHKeyPair              `yaml:"ssh_key_pairs" bson:"ssh_key_pairs" json:"ssh_key_pairs"`
+	ShutdownWaitSeconds int                       `yaml:"shutdown_wait_seconds" bson:"shutdown_wait_seconds" json:"shutdown_wait_seconds"`
 	Slack               SlackConfig               `yaml:"slack" bson:"slack" json:"slack" id:"slack"`
 	SleepSchedule       SleepScheduleConfig       `yaml:"sleep_schedule" bson:"sleep_schedule" json:"sleep_schedule" id:"sleep_schedule"`
+	Spawnhost           SpawnHostConfig           `yaml:"spawnhost" bson:"spawnhost" json:"spawnhost" id:"spawnhost"`
 	Splunk              SplunkConfig              `yaml:"splunk" bson:"splunk" json:"splunk" id:"splunk"`
+	SSHKeyDirectory     string                    `yaml:"ssh_key_directory" bson:"ssh_key_directory" json:"ssh_key_directory"`
+	SSHKeyPairs         []SSHKeyPair              `yaml:"ssh_key_pairs" bson:"ssh_key_pairs" json:"ssh_key_pairs"`
 	TaskLimits          TaskLimitsConfig          `yaml:"task_limits" bson:"task_limits" json:"task_limits" id:"task_limits"`
 	TestSelection       TestSelectionConfig       `yaml:"test_selection" bson:"test_selection" json:"test_selection" id:"test_selection"`
+	Tracer              TracerConfig              `yaml:"tracer" bson:"tracer" json:"tracer" id:"tracer"`
 	Triggers            TriggerConfig             `yaml:"triggers" bson:"triggers" json:"triggers" id:"triggers"`
 	Ui                  UIConfig                  `yaml:"ui" bson:"ui" json:"ui" id:"ui"`
-	Spawnhost           SpawnHostConfig           `yaml:"spawnhost" bson:"spawnhost" json:"spawnhost" id:"spawnhost"`
-	ShutdownWaitSeconds int                       `yaml:"shutdown_wait_seconds" bson:"shutdown_wait_seconds" json:"shutdown_wait_seconds"`
-	Tracer              TracerConfig              `yaml:"tracer" bson:"tracer" json:"tracer" id:"tracer"`
-	GitHubCheckRun      GitHubCheckRunConfig      `yaml:"github_check_run" bson:"github_check_run" json:"github_check_run" id:"github_check_run"`
 }
 
 func (c *Settings) SectionId() string { return ConfigDocID }

config_db.go

@@ -27,6 +27,7 @@ var (
 	awsInstanceRoleKey     = bsonutil.MustHaveTag(Settings{}, "AWSInstanceRole")
 	cedarKey               = bsonutil.MustHaveTag(Settings{}, "Cedar")
 	hostJasperKey          = bsonutil.MustHaveTag(Settings{}, "HostJasper")
+	devProdOwnedBucketsKey = bsonutil.MustHaveTag(Settings{}, "DevProdOwnedBuckets")
 	domainNameKey          = bsonutil.MustHaveTag(Settings{}, "DomainName")
 	jiraKey                = bsonutil.MustHaveTag(Settings{}, "Jira")
 	splunkKey              = bsonutil.MustHaveTag(Settings{}, "Splunk")

globals.go

@@ -382,6 +382,10 @@ const (
 	RedactedValue       = "{REDACTED}"
 	RedactedAfterValue  = "{REDACTED_AFTER}"
 	RedactedBeforeValue = "{REDACTED_BEFORE}"
+
+	// PresignMinimumValidTime is the minimum amount of time that a presigned URL
+	// should be valid for.
+	PresignMinimumValidTime = 15 * time.Minute
 )
 
 var TaskStatuses = []string{

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/evergreen-ci/cocoa v0.0.0-20240523192623-2e730fcd1784
 	github.com/evergreen-ci/gimlet v0.0.0-20241003144629-4e8f8a178646
 	github.com/evergreen-ci/juniper v0.0.0-20230901183147-c805ea7351aa
-	github.com/evergreen-ci/pail v0.0.0-20240812165850-4ccf32c50e99
+	github.com/evergreen-ci/pail v0.0.0-20241203190230-cd4f3226065f
 	github.com/evergreen-ci/poplar v0.0.0-20241121172741-9545e54b1b67
 	github.com/evergreen-ci/shrub v0.0.0-20231121224157-600e066f9de6
 	github.com/evergreen-ci/timber v0.0.0-20240509150854-9d66df03b40e

go.sum

@@ -250,8 +250,8 @@ github.com/evergreen-ci/negroni v1.0.1-0.20211028183800-67b6d7c2c035 h1:oVU/ni/s
 github.com/evergreen-ci/negroni v1.0.1-0.20211028183800-67b6d7c2c035/go.mod h1:pvK7NM0ZC+sfTLuIiJN4BgM1S9S5Oo79PJReAFFph18=
 github.com/evergreen-ci/pail v0.0.0-20211018155204-833e3187cfe7/go.mod h1:5gJ3srLW+mTEtewtmEs5qdnFiKFtwoggc/U3oFCVAdc=
 github.com/evergreen-ci/pail v0.0.0-20220405154537-920afff49d92/go.mod h1:Vne1WBTeJaI2zRTv3forHzliSSQmr1zCogZIcpjFsUo=
-github.com/evergreen-ci/pail v0.0.0-20240812165850-4ccf32c50e99 h1:E0GC2waMQngw3o49E09u/P6r/sOSqJzej02HWrVhiPo=
-github.com/evergreen-ci/pail v0.0.0-20240812165850-4ccf32c50e99/go.mod h1:ylFLTPr5wqF44aqz/RBSHZ/mfbpp7LELHkbyb+DqOME=
+github.com/evergreen-ci/pail v0.0.0-20241203190230-cd4f3226065f h1:VodG/6xAiSay/N5LgRldk7ffmzxbHvfJ7+xns2K9FLQ=
+github.com/evergreen-ci/pail v0.0.0-20241203190230-cd4f3226065f/go.mod h1:ylFLTPr5wqF44aqz/RBSHZ/mfbpp7LELHkbyb+DqOME=
 github.com/evergreen-ci/plank v0.0.0-20230207190607-5f47f8a30da1 h1:KkCHAMVyiM3/JHccjC9tAXE0KM80p19hlXJhaiggAdQ=
 github.com/evergreen-ci/plank v0.0.0-20230207190607-5f47f8a30da1/go.mod h1:v8BYoLFIhvElWTc1xtP7aHPBEwTC3dh308PgFBbROaI=
 github.com/evergreen-ci/poplar v0.0.0-20211028170046-0999224b53df/go.mod h1:xiggfkrlxlu2C2e58tvk0WAYFetgNC7U9ONqcP29xZs=

model/artifact/artifact_file.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/evergreen-ci/evergreen"
 	"github.com/evergreen-ci/pail"
+	"github.com/evergreen-ci/utility"
 	"github.com/mongodb/grip"
 	"github.com/pkg/errors"
 )
@@ -66,6 +67,21 @@ type File struct {
 	ContentType string `json:"content_type" bson:"content_type"`
 }
 
+func (f *File) validate() error {
+	catcher := grip.NewBasicCatcher()
+
+	catcher.ErrorfWhen(f.Bucket == "", "bucket is required")
+	catcher.ErrorfWhen(f.FileKey == "", "file key is required")
+
+	// Buckets that are not devprod owned require AWS credentials.
+	if !isDevProdOwnedBucket(f.Bucket) {
+		catcher.ErrorfWhen(f.AwsKey == "", "AWS key is required")
+		catcher.ErrorfWhen(f.AwsSecret == "", "AWS secret is required")
+	}
+
+	return catcher.Resolve()
+}
+
 // StripHiddenFiles is a helper for only showing users the files they are
 // allowed to see. It also pre-signs file URLs.
 func StripHiddenFiles(ctx context.Context, files []File, hasUser bool) ([]File, error) {
@@ -91,23 +107,24 @@ func StripHiddenFiles(ctx context.Context, files []File, hasUser bool) ([]File,
 }
 
 func presignFile(ctx context.Context, file File) (string, error) {
-	if file.AwsSecret == "" || file.AwsKey == "" || file.Bucket == "" || file.FileKey == "" {
-		return "", errors.New("AWS secret, AWS key, S3 bucket, or file key missing")
+	if err := file.validate(); err != nil {
+		return "", errors.Wrap(err, "file validation failed")
 	}
 
-	// TODO (DEVPROD-6193): remove this special casing once artifacts from the old
-	// AWS key have expired (after 5/20/2025).
-	// EC2Keys[0] contains static credentials that only has permissions to access the mciuploads bucket.
-	if file.Bucket == "mciuploads" {
-		file.AwsKey = evergreen.GetEnvironment().Settings().Providers.AWS.EC2Keys[0].Key
-		file.AwsSecret = evergreen.GetEnvironment().Settings().Providers.AWS.EC2Keys[0].Secret
+	// If this bucket is a devprod owned one, we sign the URL
+	// with the app's server IRSA credentials (which is used
+	// when no credentials are provided).
+	if isDevProdOwnedBucket(file.Bucket) {
+		file.AwsKey = ""
+		file.AwsSecret = ""
 	}
 
 	requestParams := pail.PreSignRequestParams{
-		Bucket:    file.Bucket,
-		FileKey:   file.FileKey,
-		AwsKey:    file.AwsKey,
-		AwsSecret: file.AwsSecret,
+		Bucket:                file.Bucket,
+		FileKey:               file.FileKey,
+		AwsKey:                file.AwsKey,
+		AwsSecret:             file.AwsSecret,
+		SignatureExpiryWindow: evergreen.PresignMinimumValidTime,
 	}
 	return pail.PreSign(ctx, requestParams)
 }
@@ -162,7 +179,7 @@ func RotateSecrets(toReplace, replacement string, dryRun bool) (map[TaskIDAndExe
 
 // EscapeFiles escapes the base of the file link to avoid issues opening links
 // with special characters in the UI.
-// For example, "url.com/something/file#1.tar.gz" will be escaped to "url.com/something/file%231.tar.gz
+// For example, "url.com/something/file#1.tar.gz" will be escaped to "url.com/something/file%231.tar.gz".
 func EscapeFiles(files []File) []File {
 	var escapedFiles []File
 	for _, file := range files {
@@ -180,3 +197,7 @@ func escapeFile(path string) string {
 	}
 	return path[:i] + strings.Replace(path[i:], base, url.QueryEscape(base), 1)
 }
+
+func isDevProdOwnedBucket(bucketName string) bool {
+	return utility.StringSliceContains(evergreen.GetEnvironment().Settings().DevProdOwnedBuckets, bucketName)
+}

rest/route/agent.go

@@ -87,14 +87,16 @@ func (h *agentSetup) Parse(ctx context.Context, r *http.Request) error {
 
 func (h *agentSetup) Run(ctx context.Context) gimlet.Responder {
 	data := apimodels.AgentSetupData{
-		SplunkServerURL:    h.settings.Splunk.SplunkConnectionInfo.ServerURL,
-		SplunkClientToken:  h.settings.Splunk.SplunkConnectionInfo.Token,
-		SplunkChannel:      h.settings.Splunk.SplunkConnectionInfo.Channel,
-		TaskOutput:         h.settings.Buckets.Credentials,
-		TaskSync:           h.settings.Providers.AWS.TaskSync,
-		EC2Keys:            h.settings.Providers.AWS.EC2Keys,
-		MaxExecTimeoutSecs: h.settings.TaskLimits.MaxExecTimeoutSecs,
+		SplunkServerURL:     h.settings.Splunk.SplunkConnectionInfo.ServerURL,
+		SplunkClientToken:   h.settings.Splunk.SplunkConnectionInfo.Token,
+		SplunkChannel:       h.settings.Splunk.SplunkConnectionInfo.Channel,
+		TaskOutput:          h.settings.Buckets.Credentials,
+		TaskSync:            h.settings.Providers.AWS.TaskSync,
+		EC2Keys:             h.settings.Providers.AWS.EC2Keys,
+		DevProdOwnedBuckets: h.settings.DevProdOwnedBuckets,
+		MaxExecTimeoutSecs:  h.settings.TaskLimits.MaxExecTimeoutSecs,
 	}
+
 	if h.settings.Tracer.Enabled {
 		data.TraceCollectorEndpoint = h.settings.Tracer.CollectorEndpoint
 	}