From 1d1a39272bcacdf7960a80994b033207d7245d89 Mon Sep 17 00:00:00 2001 From: sdess09 <37374498+sdess09@users.noreply.github.com> Date: Tue, 19 Sep 2023 11:54:21 -0400 Subject: [PATCH 01/15] Update Cluster Deletion (#3596) --- .../src/components/ProvisionerSettings.tsx | 2 +- .../dashboard/ClusterSettings.tsx | 2 +- .../main/home/modals/UpdateClusterModal.tsx | 42 ++++++++++++------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/dashboard/src/components/ProvisionerSettings.tsx b/dashboard/src/components/ProvisionerSettings.tsx index d520e2c608..367581df25 100644 --- a/dashboard/src/components/ProvisionerSettings.tsx +++ b/dashboard/src/components/ProvisionerSettings.tsx @@ -984,7 +984,7 @@ const ProvisionerSettings: React.FC = (props) => { = (props) => { ); - if (!currentCluster?.infra_id || !currentCluster?.service) { + if (!currentCluster?.infra_id && !currentProject?.capi_provisioner_enabled || !currentCluster?.service) { helperText = ( Remove this cluster from Porter. Since this cluster was not provisioned diff --git a/dashboard/src/main/home/modals/UpdateClusterModal.tsx b/dashboard/src/main/home/modals/UpdateClusterModal.tsx index dd97db5cf5..f527e2632d 100644 --- a/dashboard/src/main/home/modals/UpdateClusterModal.tsx +++ b/dashboard/src/main/home/modals/UpdateClusterModal.tsx @@ -52,7 +52,7 @@ class UpdateClusterModal extends Component { cluster_id: currentCluster.id, } ) - .then((_) => { + .then(async (_) => { if (!currentCluster?.infra_id) { // TODO: make this more declarative from the Home component this.props.setRefreshClusters(true); @@ -61,24 +61,34 @@ class UpdateClusterModal extends Component { pushFiltered(this.props, "/dashboard", ["project_id"], { tab: "overview", }); + + + // Handle destroying infra we've provisioned + api + .destroyInfra( + "", + {}, + { + project_id: currentProject.id, + infra_id: currentCluster.infra_id, + } + ) + .then(() => + console.log("destroyed provisioned infra:", currentCluster.infra_id) + ) + .catch(console.log); + + if (currentProject.simplified_view_enabled) { + await api.saveOnboardingState( + "", + { current_step: "connect_source" }, + { project_id: currentProject.id } + ); + window.location.reload(); + } return; } - // Handle destroying infra we've provisioned - api - .destroyInfra( - "", - {}, - { - project_id: currentProject.id, - infra_id: currentCluster.infra_id, - } - ) - .then(() => - console.log("destroyed provisioned infra:", currentCluster.infra_id) - ) - .catch(console.log); - this.props.setRefreshClusters(true); this.setState({ status: "successful", showDeleteOverlay: false }); this.context.setCurrentModal(null, null); From af3ef6922029822f91cf020429d4df94b7dbbb84 Mon Sep 17 00:00:00 2001 From: jose-fully-ported <141160579+jose-fully-ported@users.noreply.github.com> Date: Tue, 19 Sep 2023 12:17:43 -0400 Subject: [PATCH 02/15] POR-1756: add telemetry to pr creation endpoint (#3598) --- .../porter_app/create_secret_and_open_pr.go | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/api/server/handlers/porter_app/create_secret_and_open_pr.go b/api/server/handlers/porter_app/create_secret_and_open_pr.go index 85eaeb2633..8954be8e7b 100644 --- a/api/server/handlers/porter_app/create_secret_and_open_pr.go +++ b/api/server/handlers/porter_app/create_secret_and_open_pr.go @@ -17,6 +17,7 @@ import ( "github.com/porter-dev/porter/internal/auth/token" "github.com/porter-dev/porter/internal/integrations/ci/actions" "github.com/porter-dev/porter/internal/models" + "github.com/porter-dev/porter/internal/telemetry" ) type OpenStackPRHandler struct { @@ -34,9 +35,12 @@ func NewOpenStackPRHandler( } func (c *OpenStackPRHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - user, _ := r.Context().Value(types.UserScope).(*models.User) - project, _ := r.Context().Value(types.ProjectScope).(*models.Project) - cluster, _ := r.Context().Value(types.ClusterScope).(*models.Cluster) + ctx, span := telemetry.NewSpan(r.Context(), "serve-open-stack-pr") + defer span.End() + + user, _ := ctx.Value(types.UserScope).(*models.User) + project, _ := ctx.Value(types.ProjectScope).(*models.Project) + cluster, _ := ctx.Value(types.ClusterScope).(*models.Cluster) appName, reqErr := requestutils.GetURLParamString(r, types.URLParamPorterAppName) if reqErr != nil { c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(reqErr, http.StatusBadRequest)) @@ -45,11 +49,14 @@ func (c *OpenStackPRHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { request := &types.CreateSecretAndOpenGHPRRequest{} if ok := c.DecodeAndValidate(w, r, request); !ok { + err := telemetry.Error(ctx, span, nil, "error decoding request") + c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusBadRequest)) return } client, err := getGithubClient(c.Config(), request.GithubAppInstallationID) if err != nil { + err := telemetry.Error(ctx, span, err, "error creating github client") c.HandleAPIError(w, r, apierrors.NewErrInternal(err)) return } @@ -59,12 +66,16 @@ func (c *OpenStackPRHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // generate porter jwt token jwt, err := token.GetTokenForAPI(user.ID, project.ID) if err != nil { - c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error getting token for API: %w", err))) + err = fmt.Errorf("error getting token for API: %w", err) + err := telemetry.Error(ctx, span, err, err.Error()) + c.HandleAPIError(w, r, apierrors.NewErrInternal(err)) return } encoded, err := jwt.EncodeToken(c.Config().TokenConf) if err != nil { - c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error encoding API token: %w", err))) + err = fmt.Errorf("error encoding API token: %w", err) + err := telemetry.Error(ctx, span, err, err.Error()) + c.HandleAPIError(w, r, apierrors.NewErrInternal(err)) return } @@ -78,7 +89,9 @@ func (c *OpenStackPRHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { request.GithubRepoName, ) if err != nil { - c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error generating secret: %w", err))) + err = fmt.Errorf("error generating secret: %w", err) + err := telemetry.Error(ctx, span, err, err.Error()) + c.HandleAPIError(w, r, apierrors.NewErrInternal(err)) return } } @@ -113,12 +126,16 @@ func (c *OpenStackPRHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if unwrappedErr != nil { if errors.Is(unwrappedErr, actions.ErrProtectedBranch) { c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusConflict)) + return } else if errors.Is(unwrappedErr, actions.ErrCreatePRForProtectedBranch) { c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusPreconditionFailed)) + return } } else { - c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error setting up application in the github "+ - "repo: %w", err))) + err = fmt.Errorf("error setting up application in the github "+ + "repo: %w", err) + err := telemetry.Error(ctx, span, err, err.Error()) + c.HandleAPIError(w, r, apierrors.NewErrInternal(err)) return } } @@ -133,7 +150,9 @@ func (c *OpenStackPRHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // update DB with the PR url porterApp, err := c.Repo().PorterApp().ReadPorterAppByName(cluster.ID, appName) if err != nil { - c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("unable to get porter app db: %w", err))) + err = fmt.Errorf("unable to get porter app db: %w", err) + err := telemetry.Error(ctx, span, err, err.Error()) + c.HandleAPIError(w, r, apierrors.NewErrInternal(err)) return } @@ -141,7 +160,9 @@ func (c *OpenStackPRHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, err = c.Repo().PorterApp().UpdatePorterApp(porterApp) if err != nil { - c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("unable to write pr url to porter app db: %w", err))) + err = fmt.Errorf("unable to write pr url to porter app db: %w", err) + err := telemetry.Error(ctx, span, err, err.Error()) + c.HandleAPIError(w, r, apierrors.NewErrInternal(err)) return } } From 8282c88ce951d385945151daa8e5f124998deca2 Mon Sep 17 00:00:00 2001 From: sdess09 <37374498+sdess09@users.noreply.github.com> Date: Tue, 19 Sep 2023 14:11:12 -0400 Subject: [PATCH 03/15] Fix Provision failing when region is not available (#3600) --- dashboard/src/components/PreflightChecks.tsx | 15 +++- .../src/components/ProvisionerSettings.tsx | 81 ++++++++++--------- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/dashboard/src/components/PreflightChecks.tsx b/dashboard/src/components/PreflightChecks.tsx index 668c8457ad..c1a6d40b91 100644 --- a/dashboard/src/components/PreflightChecks.tsx +++ b/dashboard/src/components/PreflightChecks.tsx @@ -13,6 +13,7 @@ import Loading from "./Loading"; type Props = RouteComponentProps & { preflightData: any provider: 'AWS' | 'GCP' | 'DEFAULT'; + error?: string; }; @@ -97,10 +98,16 @@ const PreflightChecks: React.FC = (props) => { Porter checks that the account has the right permissions and resources to provision a cluster. - {Object.keys(currentMessageConst).map((checkKey) => ( - - ))} - + { + props.error ? + + : + Object.keys(currentMessageConst).map((checkKey) => ( + + )) + + } + ); }; diff --git a/dashboard/src/components/ProvisionerSettings.tsx b/dashboard/src/components/ProvisionerSettings.tsx index 367581df25..61bddbbbf6 100644 --- a/dashboard/src/components/ProvisionerSettings.tsx +++ b/dashboard/src/components/ProvisionerSettings.tsx @@ -133,6 +133,7 @@ const ProvisionerSettings: React.FC = (props) => { const [isLoading, setIsLoading] = useState(false); const [preflightData, setPreflightData] = useState(null) const [preflightFailed, setPreflightFailed] = useState(true) + const [preflightError, setPreflightError] = useState("") const markStepStarted = async (step: string, errMessage?: string) => { try { @@ -480,52 +481,60 @@ const ProvisionerSettings: React.FC = (props) => { useEffect(() => { if (!props.clusterId) { setStep(1) - setPreflightData(null) preflightChecks() } }, [props.selectedClusterVersion, awsRegion]); const preflightChecks = async () => { - setIsLoading(true); - setPreflightData(null); - var data = new PreflightCheckRequest({ - projectId: BigInt(currentProject.id), - cloudProvider: EnumCloudProvider.AWS, - cloudProviderCredentialsId: props.credentialId, - preflightValues: { - case: "eksPreflightValues", - value: new EKSPreflightValues({ - region: awsRegion, - }) + try { + setIsLoading(true); + setPreflightData(null); + setPreflightFailed(true) + setPreflightError(""); + + var data = new PreflightCheckRequest({ + projectId: BigInt(currentProject.id), + cloudProvider: EnumCloudProvider.AWS, + cloudProviderCredentialsId: props.credentialId, + preflightValues: { + case: "eksPreflightValues", + value: new EKSPreflightValues({ + region: awsRegion, + }) + } + }); + const preflightDataResp = await api.preflightCheck( + "", data, + { + id: currentProject.id, + } + ) + // Check if any of the preflight checks has a message + let hasMessage = false; + let errors = "Preflight Checks Failed : "; + for (let check in preflightDataResp?.data?.Msg.preflight_checks) { + if (preflightDataResp?.data?.Msg.preflight_checks[check]?.message) { + hasMessage = true; + errors = errors + check + ", " + } } - }); - const preflightDataResp = await api.preflightCheck( - "", data, - { - id: currentProject.id, + // If none of the checks have a message, set setPreflightFailed to false + if (hasMessage) { + markStepStarted("provisioning-failed", errors); } - ) - // Check if any of the preflight checks has a message - let hasMessage = false; - let errors = "Preflight Checks Failed : "; - for (let check in preflightDataResp?.data?.Msg.preflight_checks) { - if (preflightDataResp?.data?.Msg.preflight_checks[check]?.message) { - hasMessage = true; - errors = errors + check + ", " + if (!hasMessage) { + setPreflightFailed(false); + setStep(2); } + setPreflightData(preflightDataResp?.data?.Msg); + setIsLoading(false) + } catch (err) { + setPreflightError(err) + setIsLoading(false) + setPreflightFailed(true); } - // If none of the checks have a message, set setPreflightFailed to false - if (hasMessage) { - markStepStarted("provisioning-failed", errors); - } - if (!hasMessage) { - setPreflightFailed(false); - setStep(2); - } - setPreflightData(preflightDataResp?.data?.Msg); - setIsLoading(false) } const renderAdvancedSettings = () => { @@ -997,7 +1006,7 @@ const ProvisionerSettings: React.FC = (props) => { , <> - + {(preflightFailed && preflightData) && <> From b99a90e5e8b7d6a7cf17a33a48cc1c7e6489b2ec Mon Sep 17 00:00:00 2001 From: d-g-town <66391417+d-g-town@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:45:24 -0400 Subject: [PATCH 04/15] fix env group bugs with excluding default app env (#3601) Co-authored-by: David Townley --- .../handlers/environment_groups/create.go | 7 +----- .../handlers/environment_groups/list.go | 2 +- api/server/handlers/porter_app/get_app_env.go | 2 +- .../update_app_environment_group.go | 4 ++-- .../kubernetes/environment_groups/create.go | 13 +++++++--- internal/kubernetes/environment_groups/get.go | 19 +++++++++++---- .../kubernetes/environment_groups/list.go | 23 +++++++++++------- internal/porter_app/environment.go | 24 ++++++++++++++----- 8 files changed, 62 insertions(+), 32 deletions(-) diff --git a/api/server/handlers/environment_groups/create.go b/api/server/handlers/environment_groups/create.go index ad9114e8ed..4c4502447e 100644 --- a/api/server/handlers/environment_groups/create.go +++ b/api/server/handlers/environment_groups/create.go @@ -75,15 +75,10 @@ func (c *UpdateEnvironmentGroupHandler) ServeHTTP(w http.ResponseWriter, r *http return } - secrets := make(map[string][]byte) - for k, v := range request.SecretVariables { - secrets[k] = []byte(v) - } - envGroup := environment_groups.EnvironmentGroup{ Name: request.Name, Variables: request.Variables, - SecretVariables: secrets, + SecretVariables: request.SecretVariables, CreatedAtUTC: time.Now().UTC(), } diff --git a/api/server/handlers/environment_groups/list.go b/api/server/handlers/environment_groups/list.go index db5f8090ff..e2556d3566 100644 --- a/api/server/handlers/environment_groups/list.go +++ b/api/server/handlers/environment_groups/list.go @@ -58,7 +58,7 @@ func (c *ListEnvironmentGroupsHandler) ServeHTTP(w http.ResponseWriter, r *http. return } - allEnvGroupVersions, err := environmentgroups.ListEnvironmentGroups(ctx, agent, environmentgroups.WithNamespace(environmentgroups.Namespace_EnvironmentGroups)) + allEnvGroupVersions, err := environmentgroups.ListEnvironmentGroups(ctx, agent, environmentgroups.WithNamespace(environmentgroups.Namespace_EnvironmentGroups), environmentgroups.WithoutDefaultAppEnvironmentGroups()) if err != nil { err = telemetry.Error(ctx, span, err, "unable to list all environment groups") c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) diff --git a/api/server/handlers/porter_app/get_app_env.go b/api/server/handlers/porter_app/get_app_env.go index 0ae1532115..01e5cc5a8f 100644 --- a/api/server/handlers/porter_app/get_app_env.go +++ b/api/server/handlers/porter_app/get_app_env.go @@ -131,7 +131,7 @@ func (c *GetAppEnvHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { DeploymentTargetRepository: c.Repo().DeploymentTarget(), } - envGroups, err := porter_app.AppEnvironmentFromProto(ctx, envFromProtoInp, porter_app.WithEnvGroupFilter(request.EnvGroups), porter_app.WithSecrets()) + envGroups, err := porter_app.AppEnvironmentFromProto(ctx, envFromProtoInp, porter_app.WithEnvGroupFilter(request.EnvGroups), porter_app.WithSecrets(), porter_app.WithoutDefaultAppEnvGroups()) if err != nil { err := telemetry.Error(ctx, span, err, "error getting app environment from revision") c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) diff --git a/api/server/handlers/porter_app/update_app_environment_group.go b/api/server/handlers/porter_app/update_app_environment_group.go index d76b54a6b9..021a850713 100644 --- a/api/server/handlers/porter_app/update_app_environment_group.go +++ b/api/server/handlers/porter_app/update_app_environment_group.go @@ -201,7 +201,7 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R } variables := make(map[string]string) - secrets := make(map[string][]byte) + secrets := make(map[string]string) if !request.HardUpdate { for key, value := range latestEnvironmentGroup.Variables { @@ -216,7 +216,7 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R variables[key] = value } for key, value := range request.Secrets { - secrets[key] = []byte(value) + secrets[key] = value } envGroup := environment_groups.EnvironmentGroup{ diff --git a/internal/kubernetes/environment_groups/create.go b/internal/kubernetes/environment_groups/create.go index 6fc5462c23..7714f8fad0 100644 --- a/internal/kubernetes/environment_groups/create.go +++ b/internal/kubernetes/environment_groups/create.go @@ -1,7 +1,6 @@ package environment_groups import ( - "bytes" "context" "fmt" "strconv" @@ -46,7 +45,7 @@ func CreateOrUpdateBaseEnvironmentGroup(ctx context.Context, a *kubernetes.Agent // If any of the secret variables are set to the dummy value (i.e. are unchanged), replace them with the existing value. for k, v := range environmentGroup.SecretVariables { - if bytes.Equal(v, []byte(EnvGroupSecretDummyValue)) { + if v == EnvGroupSecretDummyValue { existingValue, ok := latestEnvironmentGroup.SecretVariables[k] if !ok { return telemetry.Error(ctx, span, nil, "secret variable does not exist in latest environment group") @@ -144,6 +143,11 @@ func createVersionedEnvironmentGroupInNamespace(ctx context.Context, a *kubernet return telemetry.Error(ctx, span, err, "unable to create new environment group variables version") } + secretData := make(map[string][]byte) + for k, v := range environmentGroup.SecretVariables { + secretData[k] = []byte(v) + } + secret := v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s.%d", environmentGroup.Name, environmentGroup.Version), @@ -153,7 +157,10 @@ func createVersionedEnvironmentGroupInNamespace(ctx context.Context, a *kubernet LabelKey_EnvironmentGroupVersion: strconv.Itoa(environmentGroup.Version), }, }, - Data: environmentGroup.SecretVariables, + Data: secretData, + } + for k, v := range additionalLabels { + secret.Labels[k] = v } err = createSecretWithVersion(ctx, a, secret, environmentGroup.Version) diff --git a/internal/kubernetes/environment_groups/get.go b/internal/kubernetes/environment_groups/get.go index 71a1ec500b..ea30fcc6bb 100644 --- a/internal/kubernetes/environment_groups/get.go +++ b/internal/kubernetes/environment_groups/get.go @@ -71,9 +71,10 @@ func latestBaseEnvironmentGroup(ctx context.Context, a *kubernetes.Agent, enviro // If you are looking for envrionment groups in the base namespace, consider using LatestBaseEnvironmentGroup or ListBaseEnvironmentGroups instead type EnvironmentGroupInTargetNamespaceInput struct { // Name is the environment group name which can be found on the configmap label - Name string - Version int - Namespace string + Name string + Version int + Namespace string + ExcludeDefaultAppEnvironmentGroup bool } // EnvironmentGroupInTargetNamespace checks if an environment group of a specific name and version exists in a target namespace. @@ -99,7 +100,17 @@ func EnvironmentGroupInTargetNamespace(ctx context.Context, a *kubernetes.Agent, telemetry.AttributeKV{Key: "namespace", Value: inp.Namespace}, ) - environmentGroups, err := ListEnvironmentGroups(ctx, a, WithEnvironmentGroupName(inp.Name), WithEnvironmentGroupVersion(inp.Version), WithNamespace(inp.Namespace)) + opts := []EnvironmentGroupOption{ + WithEnvironmentGroupName(inp.Name), + WithEnvironmentGroupVersion(inp.Version), + WithNamespace(inp.Namespace), + } + + if inp.ExcludeDefaultAppEnvironmentGroup { + opts = append(opts, WithoutDefaultAppEnvironmentGroups()) + } + + environmentGroups, err := ListEnvironmentGroups(ctx, a, opts...) if err != nil { return eg, telemetry.Error(ctx, span, err, "unable to list environment groups in target namespace") } diff --git a/internal/kubernetes/environment_groups/list.go b/internal/kubernetes/environment_groups/list.go index c79ab2926d..43cc222f62 100644 --- a/internal/kubernetes/environment_groups/list.go +++ b/internal/kubernetes/environment_groups/list.go @@ -33,7 +33,7 @@ type EnvironmentGroup struct { // Variables are non-secret values for the EnvironmentGroup. This usually will be a configmap Variables map[string]string `json:"variables,omitempty"` // SecretVariables are secret values for the EnvironmentGroup. This usually will be a Secret on the kubernetes cluster - SecretVariables map[string][]byte `json:"secret_variables,omitempty"` + SecretVariables map[string]string `json:"secret_variables,omitempty"` // CreatedAt is only used for display purposes and is in UTC Unix time CreatedAtUTC time.Time `json:"created_at"` } @@ -42,7 +42,7 @@ type environmentGroupOptions struct { namespace string environmentGroupLabelName string environmentGroupLabelVersion int - includeDefaultAppEnvironmentGroups bool + excludeDefaultAppEnvironmentGroups bool } // EnvironmentGroupOption is a function that modifies ListEnvironmentGroups @@ -69,10 +69,10 @@ func WithEnvironmentGroupVersion(version int) EnvironmentGroupOption { } } -// WithDefaultAppEnvironmentGroup includes default app environment groups in the list -func WithDefaultAppEnvironmentGroup() EnvironmentGroupOption { +// WithoutDefaultAppEnvironmentGroups includes default app environment groups in the list +func WithoutDefaultAppEnvironmentGroups() EnvironmentGroupOption { return func(opts *environmentGroupOptions) { - opts.includeDefaultAppEnvironmentGroups = true + opts.excludeDefaultAppEnvironmentGroups = true } } @@ -133,7 +133,7 @@ func listEnvironmentGroups(ctx context.Context, a *kubernetes.Agent, listOpts .. continue // invalid version label as it should be an int, not an environment group } - if !opts.includeDefaultAppEnvironmentGroups { + if opts.excludeDefaultAppEnvironmentGroups { value := cm.Labels[LabelKey_DefaultAppEnvironment] if value == "true" { continue // do not include default app environment groups @@ -153,6 +153,11 @@ func listEnvironmentGroups(ctx context.Context, a *kubernetes.Agent, listOpts .. } for _, secret := range secretListResp.Items { + stringSecret := make(map[string]string) + for k, v := range secret.Data { + stringSecret[k] = string(v) + } + name, ok := secret.Labels[LabelKey_EnvironmentGroupName] if !ok { continue // missing name label, not an environment group @@ -166,7 +171,7 @@ func listEnvironmentGroups(ctx context.Context, a *kubernetes.Agent, listOpts .. continue // invalid version label as it should be an int, not an environment group } - if !opts.includeDefaultAppEnvironmentGroups { + if opts.excludeDefaultAppEnvironmentGroups { value, ok := secret.Labels[LabelKey_DefaultAppEnvironment] if ok && value == "true" { continue // do not include default app environment groups @@ -179,7 +184,7 @@ func listEnvironmentGroups(ctx context.Context, a *kubernetes.Agent, listOpts .. envGroupSet[secret.Name] = EnvironmentGroup{ Name: name, Version: version, - SecretVariables: secret.Data, + SecretVariables: stringSecret, Variables: envGroupSet[secret.Name].Variables, CreatedAtUTC: secret.CreationTimestamp.Time.UTC(), } @@ -210,7 +215,7 @@ func ListEnvironmentGroups(ctx context.Context, a *kubernetes.Agent, listOpts .. for _, envGroup := range envGroups { for k := range envGroup.SecretVariables { - envGroup.SecretVariables[k] = []byte(EnvGroupSecretDummyValue) + envGroup.SecretVariables[k] = EnvGroupSecretDummyValue } } diff --git a/internal/porter_app/environment.go b/internal/porter_app/environment.go index c5227bdb64..8cd7b06951 100644 --- a/internal/porter_app/environment.go +++ b/internal/porter_app/environment.go @@ -13,8 +13,9 @@ import ( ) type envVariarableOptions struct { - includeSecrets bool - envGroups []string + includeSecrets bool + envGroups []string + excludeDefaultAppEnvGroups bool } // EnvVariableOption is a function that modifies AppEnvironmentFromProto @@ -34,6 +35,13 @@ func WithEnvGroupFilter(envGroups []string) EnvVariableOption { } } +// WithoutDefaultAppEnvGroups filters out the default app environment groups from the returned list +func WithoutDefaultAppEnvGroups() EnvVariableOption { + return func(opts *envVariarableOptions) { + opts.excludeDefaultAppEnvGroups = true + } +} + // AppEnvironmentFromProtoInput is the input struct for AppEnvironmentFromProto type AppEnvironmentFromProtoInput struct { ProjectID uint @@ -107,9 +115,10 @@ func AppEnvironmentFromProto(ctx context.Context, inp AppEnvironmentFromProtoInp for _, envGroupRef := range filteredEnvGroups { envGroup, err := environment_groups.EnvironmentGroupInTargetNamespace(ctx, inp.K8SAgent, environment_groups.EnvironmentGroupInTargetNamespaceInput{ - Name: envGroupRef.GetName(), - Version: int(envGroupRef.GetVersion()), - Namespace: namespace, + Name: envGroupRef.GetName(), + Version: int(envGroupRef.GetVersion()), + Namespace: namespace, + ExcludeDefaultAppEnvironmentGroup: opts.excludeDefaultAppEnvGroups, }) if err != nil { return nil, telemetry.Error(ctx, span, err, "error getting environment group in target namespace") @@ -119,7 +128,10 @@ func AppEnvironmentFromProto(ctx context.Context, inp AppEnvironmentFromProtoInp envGroup.SecretVariables = nil } - envGroups = append(envGroups, envGroup) + // if envGroup.Name is empty, it means the environment group was a default app environment group and was filtered out + if envGroup.Name != "" { + envGroups = append(envGroups, envGroup) + } } return envGroups, nil From c3420cc9c5578b1fe5e14f6117b557e547c75f20 Mon Sep 17 00:00:00 2001 From: jose-fully-ported <141160579+jose-fully-ported@users.noreply.github.com> Date: Tue, 19 Sep 2023 18:03:45 -0400 Subject: [PATCH 05/15] Set default builders to be Ubuntu 22.04 based (#3602) --- api/server/handlers/gitinstallation/get_buildpack.go | 2 ++ .../handlers/project_integration/get_gitlab_repo_buildpack.go | 2 ++ dashboard/src/components/repo-selector/BuildpackSelection.tsx | 4 ++-- dashboard/src/main/home/app-dashboard/types/buildpack.ts | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/server/handlers/gitinstallation/get_buildpack.go b/api/server/handlers/gitinstallation/get_buildpack.go index 183e90e918..d03f1d4616 100644 --- a/api/server/handlers/gitinstallation/get_buildpack.go +++ b/api/server/handlers/gitinstallation/get_buildpack.go @@ -22,12 +22,14 @@ func initBuilderInfo() map[string]*buildpacks.BuilderInfo { builders[buildpacks.PaketoBuilder] = &buildpacks.BuilderInfo{ Name: "Paketo", Builders: []string{ + "paketobuildpacks/builder-jammy-full:latest", "paketobuildpacks/builder:full", }, } builders[buildpacks.HerokuBuilder] = &buildpacks.BuilderInfo{ Name: "Heroku", Builders: []string{ + "heroku/buildpacks:22", "heroku/buildpacks:20", "heroku/buildpacks:18", }, diff --git a/api/server/handlers/project_integration/get_gitlab_repo_buildpack.go b/api/server/handlers/project_integration/get_gitlab_repo_buildpack.go index 6fdb3a8292..33ef8fb3fa 100644 --- a/api/server/handlers/project_integration/get_gitlab_repo_buildpack.go +++ b/api/server/handlers/project_integration/get_gitlab_repo_buildpack.go @@ -130,12 +130,14 @@ func initBuilderInfo() map[string]*buildpacks.BuilderInfo { builders[buildpacks.PaketoBuilder] = &buildpacks.BuilderInfo{ Name: "Paketo", Builders: []string{ + "paketobuildpacks/builder-jammy-full:latest", "paketobuildpacks/builder:full", }, } builders[buildpacks.HerokuBuilder] = &buildpacks.BuilderInfo{ Name: "Heroku", Builders: []string{ + "heroku/buildpacks:22", "heroku/buildpacks:20", "heroku/buildpacks:18", }, diff --git a/dashboard/src/components/repo-selector/BuildpackSelection.tsx b/dashboard/src/components/repo-selector/BuildpackSelection.tsx index 8431500170..86c75d4010 100644 --- a/dashboard/src/components/repo-selector/BuildpackSelection.tsx +++ b/dashboard/src/components/repo-selector/BuildpackSelection.tsx @@ -10,8 +10,8 @@ import { ActionConfigType } from "shared/types"; import styled, { keyframes } from "styled-components"; const DEFAULT_BUILDER_NAME = "heroku"; -const DEFAULT_PAKETO_STACK = "paketobuildpacks/builder:full"; -const DEFAULT_HEROKU_STACK = "heroku/buildpacks:20"; +const DEFAULT_PAKETO_STACK = "paketobuildpacks/builder-jammy-full:latest"; +const DEFAULT_HEROKU_STACK = "heroku/buildpacks:22"; type BuildConfig = { builder: string; diff --git a/dashboard/src/main/home/app-dashboard/types/buildpack.ts b/dashboard/src/main/home/app-dashboard/types/buildpack.ts index 1ec05dd0b9..0704017db8 100644 --- a/dashboard/src/main/home/app-dashboard/types/buildpack.ts +++ b/dashboard/src/main/home/app-dashboard/types/buildpack.ts @@ -24,8 +24,8 @@ export const detectedBuildpackSchema = z.object({ export type DetectedBuildpack = z.infer; export const DEFAULT_BUILDER_NAME = "heroku"; -export const DEFAULT_PAKETO_STACK = "paketobuildpacks/builder:full"; -export const DEFAULT_HEROKU_STACK = "heroku/buildpacks:20"; +export const DEFAULT_PAKETO_STACK = "paketobuildpacks/builder-jammy-full:latest"; +export const DEFAULT_HEROKU_STACK = "heroku/buildpacks:22"; export const BUILDPACK_TO_NAME: { [key: string]: string } = { "heroku/nodejs": "NodeJS", From e101a54d8a64e5343790ce01d5acfeedcb345cb3 Mon Sep 17 00:00:00 2001 From: d-g-town <66391417+d-g-town@users.noreply.github.com> Date: Tue, 19 Sep 2023 18:14:39 -0400 Subject: [PATCH 06/15] add app name to front end parse yaml api call (#3593) Co-authored-by: David Townley Co-authored-by: Feroze Mohideen --- dashboard/src/lib/hooks/usePorterYaml.ts | 7 ++++++- .../home/app-dashboard/app-view/LatestRevisionContext.tsx | 1 + .../src/main/home/app-dashboard/create-app/CreateApp.tsx | 2 +- dashboard/src/shared/api.tsx | 1 + internal/porter_app/parse_test.go | 2 +- internal/porter_app/testdata/v2_input_nobuild.yaml | 2 +- internal/porter_app/v2/yaml.go | 8 ++++++++ 7 files changed, 19 insertions(+), 4 deletions(-) diff --git a/dashboard/src/lib/hooks/usePorterYaml.ts b/dashboard/src/lib/hooks/usePorterYaml.ts index 8550f68f8f..d3f83d28c8 100644 --- a/dashboard/src/lib/hooks/usePorterYaml.ts +++ b/dashboard/src/lib/hooks/usePorterYaml.ts @@ -30,9 +30,11 @@ type PorterYamlStatus = */ export const usePorterYaml = ({ source, + appName = "", useDefaults = true, }: { source: (SourceOptions & { type: "github" }) | null; + appName?: string; useDefaults?: boolean; }): PorterYamlStatus => { const { currentProject, currentCluster } = useContext(Context); @@ -93,17 +95,19 @@ export const usePorterYaml = ({ const detectServices = useCallback( async ({ b64Yaml, + appName, projectId, clusterId, }: { b64Yaml: string; + appName: string; projectId: number; clusterId: number; }) => { try { const res = await api.parsePorterYaml( "", - { b64_yaml: b64Yaml }, + { b64_yaml: b64Yaml, app_name: appName}, { project_id: projectId, cluster_id: clusterId, @@ -147,6 +151,7 @@ export const usePorterYaml = ({ if (data) { detectServices({ b64Yaml: data, + appName: appName, projectId: currentProject.id, clusterId: currentCluster.id, }); diff --git a/dashboard/src/main/home/app-dashboard/app-view/LatestRevisionContext.tsx b/dashboard/src/main/home/app-dashboard/app-view/LatestRevisionContext.tsx index 8dacfe1d9e..d2b0bca355 100644 --- a/dashboard/src/main/home/app-dashboard/app-view/LatestRevisionContext.tsx +++ b/dashboard/src/main/home/app-dashboard/app-view/LatestRevisionContext.tsx @@ -145,6 +145,7 @@ export const LatestRevisionProvider = ({ const { loading: porterYamlLoading, detectedServices } = usePorterYaml({ source: latestSource?.type === "github" ? latestSource : null, + appName: appName, useDefaults: false, }); diff --git a/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx b/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx index cb20efedbb..370d535333 100644 --- a/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx +++ b/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx @@ -183,7 +183,7 @@ const CreateApp: React.FC = ({ history }) => { porterYamlFound, detectedName, loading: isLoadingPorterYaml, - } = usePorterYaml({ source: source?.type === "github" ? source : null }); + } = usePorterYaml({ source: source?.type === "github" ? source : null, appName: name.value }); const deploymentTarget = useDefaultDeploymentTarget(); const { updateAppStep } = useAppAnalytics(name.value); const { validateApp } = useAppValidation({ diff --git a/dashboard/src/shared/api.tsx b/dashboard/src/shared/api.tsx index 0f2c83b960..0b45043389 100644 --- a/dashboard/src/shared/api.tsx +++ b/dashboard/src/shared/api.tsx @@ -832,6 +832,7 @@ const getPorterYamlContents = baseApi< const parsePorterYaml = baseApi< { b64_yaml: string; + app_name?: string; }, { project_id: number; diff --git a/internal/porter_app/parse_test.go b/internal/porter_app/parse_test.go index f8de16e9c9..9b8a35966a 100644 --- a/internal/porter_app/parse_test.go +++ b/internal/porter_app/parse_test.go @@ -44,7 +44,7 @@ func TestParseYAML(t *testing.T) { } var result_nobuild = &porterv1.PorterApp{ - Name: "js-test-app", + Name: "test-app", Services: map[string]*porterv1.Service{ "example-job": { Run: "echo 'hello world'", diff --git a/internal/porter_app/testdata/v2_input_nobuild.yaml b/internal/porter_app/testdata/v2_input_nobuild.yaml index de014a4fec..63aff6f8eb 100644 --- a/internal/porter_app/testdata/v2_input_nobuild.yaml +++ b/internal/porter_app/testdata/v2_input_nobuild.yaml @@ -1,5 +1,5 @@ version: v2 -name: "js-test-app" +name: "test-app" image: repository: nginx tag: latest diff --git a/internal/porter_app/v2/yaml.go b/internal/porter_app/v2/yaml.go index 23ac6accb9..57d73533fe 100644 --- a/internal/porter_app/v2/yaml.go +++ b/internal/porter_app/v2/yaml.go @@ -31,10 +31,18 @@ func AppProtoFromYaml(ctx context.Context, porterYamlBytes []byte, appName strin porterYaml.Name = appName } + if porterYaml.Name != "" && appName != "" && porterYaml.Name != appName { + return nil, nil, telemetry.Error(ctx, span, nil, "name specified in porter.yaml does not match app name") + } + appProto := &porterv1.PorterApp{ Name: porterYaml.Name, } + if appProto.Name == "" { + return nil, nil, telemetry.Error(ctx, span, nil, "app name is empty") + } + if porterYaml.Build != nil { appProto.Build = &porterv1.Build{ Context: porterYaml.Build.Context, From 75753d0e58968a622d157899ecedea12991f6e50 Mon Sep 17 00:00:00 2001 From: Feroze Mohideen Date: Wed, 20 Sep 2023 05:04:09 -0400 Subject: [PATCH 07/15] env group parsing hotfix (#3605) Co-authored-by: Feroze Mohideen --- .../cluster-dashboard/env-groups/EnvGroupArrayStacks.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dashboard/src/main/home/cluster-dashboard/env-groups/EnvGroupArrayStacks.tsx b/dashboard/src/main/home/cluster-dashboard/env-groups/EnvGroupArrayStacks.tsx index 05a994523f..aa4c58c563 100644 --- a/dashboard/src/main/home/cluster-dashboard/env-groups/EnvGroupArrayStacks.tsx +++ b/dashboard/src/main/home/cluster-dashboard/env-groups/EnvGroupArrayStacks.tsx @@ -45,9 +45,10 @@ const EnvGroupArray = ({ }, [values]); const isKeyOverriding = (key: string) => { if (!syncedEnvGroups || !values) return false; - return syncedEnvGroups?.some(envGroup => - (envGroup?.variables) && key in envGroup?.variables || key in envGroup?.secret_variables - ); + return syncedEnvGroups?.some(envGroup => { + if (!envGroup || !envGroup.variables) return false; + return key in envGroup.variables || (envGroup.secret_variables && key in envGroup.secret_variables); + }); }; const readFile = (env: string) => { From 11eeb5340574f56d5a5abb000b0beeabde289674 Mon Sep 17 00:00:00 2001 From: ianedwards Date: Wed, 20 Sep 2023 10:05:49 -0400 Subject: [PATCH 08/15] sync latest version of env groups when updating app (#3599) --- api/client/porter_app.go | 2 + .../handlers/porter_app/get_build_env.go | 9 +- .../update_app_environment_group.go | 137 +++++++++++++++++- cli/cmd/v2/apply.go | 2 +- .../app-view/AppDataContainer.tsx | 5 +- .../app-dashboard/create-app/CreateApp.tsx | 1 + dashboard/src/shared/api.tsx | 18 +-- internal/porter_app/v2/yaml.go | 1 + 8 files changed, 147 insertions(+), 28 deletions(-) diff --git a/api/client/porter_app.go b/api/client/porter_app.go index e5e7f879fb..081b8c55d8 100644 --- a/api/client/porter_app.go +++ b/api/client/porter_app.go @@ -442,6 +442,7 @@ func (c *Client) CreateOrUpdateAppEnvironment( deploymentTargetID string, variables map[string]string, secrets map[string]string, + Base64AppProto string, ) (*porter_app.UpdateAppEnvironmentResponse, error) { resp := &porter_app.UpdateAppEnvironmentResponse{} @@ -450,6 +451,7 @@ func (c *Client) CreateOrUpdateAppEnvironment( Variables: variables, Secrets: secrets, HardUpdate: false, + Base64AppProto: Base64AppProto, } err := c.postRequest( diff --git a/api/server/handlers/porter_app/get_build_env.go b/api/server/handlers/porter_app/get_build_env.go index d60b82c802..6dec280c6e 100644 --- a/api/server/handlers/porter_app/get_build_env.go +++ b/api/server/handlers/porter_app/get_build_env.go @@ -115,10 +115,11 @@ func (c *GetBuildEnvHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } envFromProtoInp := porter_app.AppEnvironmentFromProtoInput{ - ProjectID: project.ID, - ClusterID: int(cluster.ID), - App: appProto, - K8SAgent: agent, + ProjectID: project.ID, + ClusterID: int(cluster.ID), + App: appProto, + K8SAgent: agent, + DeploymentTargetRepository: c.Repo().DeploymentTarget(), } envGroups, err := porter_app.AppEnvironmentFromProto(ctx, envFromProtoInp) if err != nil { diff --git a/api/server/handlers/porter_app/update_app_environment_group.go b/api/server/handlers/porter_app/update_app_environment_group.go index 021a850713..22ed7b1655 100644 --- a/api/server/handlers/porter_app/update_app_environment_group.go +++ b/api/server/handlers/porter_app/update_app_environment_group.go @@ -1,11 +1,14 @@ package porter_app import ( + "context" + "encoding/base64" "net/http" "strconv" "strings" "time" + "github.com/porter-dev/porter/internal/kubernetes" "github.com/porter-dev/porter/internal/porter_app" "github.com/porter-dev/porter/api/server/shared/requestutils" @@ -13,6 +16,7 @@ import ( "connectrpc.com/connect" + "github.com/porter-dev/api-contracts/generated/go/helpers" porterv1 "github.com/porter-dev/api-contracts/generated/go/porter/v1" "github.com/porter-dev/porter/api/server/authz" @@ -54,6 +58,7 @@ const ( // UpdateAppEnvironmentRequest represents the accepted fields on a request to the /apps/{porter_app_name}/environment-group endpoint type UpdateAppEnvironmentRequest struct { + Base64AppProto string `json:"b64_app_proto"` DeploymentTargetID string `json:"deployment_target_id"` Variables map[string]string `json:"variables"` Secrets map[string]string `json:"secrets"` @@ -110,6 +115,27 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R } telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "deployment-target-id", Value: request.DeploymentTargetID}) + if request.Base64AppProto == "" { + err := telemetry.Error(ctx, span, nil, "b64 yaml is empty") + c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusBadRequest)) + return + } + + decoded, err := base64.StdEncoding.DecodeString(request.Base64AppProto) + if err != nil { + err := telemetry.Error(ctx, span, err, "error decoding base yaml") + c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) + return + } + + appProto := &porterv1.PorterApp{} + err = helpers.UnmarshalContractObject(decoded, appProto) + if err != nil { + err := telemetry.Error(ctx, span, err, "error unmarshalling app proto") + c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) + return + } + deploymentTargetDetailsReq := connect.NewRequest(&porterv1.DeploymentTargetDetailsRequest{ ProjectId: int64(project.ID), DeploymentTargetId: request.DeploymentTargetID, @@ -139,7 +165,7 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "hard-update", Value: request.HardUpdate}) - envGroupName, err := porter_app.AppEnvGroupName(ctx, appName, request.DeploymentTargetID, cluster.ID, c.Repo().PorterApp()) + appEnvGroupName, err := porter_app.AppEnvGroupName(ctx, appName, request.DeploymentTargetID, cluster.ID, c.Repo().PorterApp()) if err != nil { err := telemetry.Error(ctx, span, err, "error getting app env group name") c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) @@ -153,7 +179,7 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R return } - latestEnvironmentGroup, err := environment_groups.LatestBaseEnvironmentGroup(ctx, agent, envGroupName) + latestEnvironmentGroup, err := environment_groups.LatestBaseEnvironmentGroup(ctx, agent, appEnvGroupName) if err != nil { err := telemetry.Error(ctx, span, err, "unable to get latest base environment group") c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) @@ -190,6 +216,23 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "same-env-group", Value: sameEnvGroup}) if sameEnvGroup { + // even if the env group is the same, we still need to sync the latest versions of the other env groups + syncInp := syncLatestEnvGroupVersionsInput{ + envGroups: appProto.EnvGroups, + appName: appName, + appEnvName: appEnvGroupName, + sameAppEnv: true, + namespace: namespace, + deploymentTargetID: request.DeploymentTargetID, + k8sAgent: agent, + } + err = syncLatestEnvGroupVersions(ctx, syncInp) + if err != nil { + err := telemetry.Error(ctx, span, err, "error syncing latest env group versions") + c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) + return + } + res := &UpdateAppEnvironmentResponse{ EnvGroupName: latestEnvironmentGroup.Name, EnvGroupVersion: latestEnvironmentGroup.Version, @@ -220,7 +263,7 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R } envGroup := environment_groups.EnvironmentGroup{ - Name: envGroupName, + Name: appEnvGroupName, Variables: variables, SecretVariables: secrets, CreatedAtUTC: time.Now().UTC(), @@ -241,7 +284,7 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R } inp := environment_groups.SyncLatestVersionToNamespaceInput{ - BaseEnvironmentGroupName: envGroupName, + BaseEnvironmentGroupName: appEnvGroupName, TargetNamespace: namespace, } @@ -267,6 +310,22 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R return } + syncInp := syncLatestEnvGroupVersionsInput{ + envGroups: appProto.EnvGroups, + appName: appName, + appEnvName: appEnvGroupName, + sameAppEnv: false, + namespace: namespace, + deploymentTargetID: request.DeploymentTargetID, + k8sAgent: agent, + } + err = syncLatestEnvGroupVersions(ctx, syncInp) + if err != nil { + err := telemetry.Error(ctx, span, err, "error syncing latest env group versions") + c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) + return + } + res := &UpdateAppEnvironmentResponse{ EnvGroupName: split[0], EnvGroupVersion: version, @@ -274,3 +333,73 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R c.WriteResult(w, r, res) } + +type syncLatestEnvGroupVersionsInput struct { + // envGroups is the list of env groups to sync. We only need the names and will get the latest version of each from the porter-env-group ns + envGroups []*porterv1.EnvGroup + // appName is the name of the app + appName string + // appEnvName is the name of the app env. This is the env group created when the app is created for storing app-specific variables + appEnvName string + // sameAppEnv is true if the app env group variables are unchanged. If true, we do not need to sync the latest version of the app env group + sameAppEnv bool + // namespace is the namespace to sync the latest versions to + namespace string + // deploymentTargetID is the id of the deployment target + deploymentTargetID string + // k8sAgent is the kubernetes agent + k8sAgent *kubernetes.Agent +} + +// syncLatestEnvGroupVersions syncs the latest versions of the env groups to the namespace where an app is deployed +func syncLatestEnvGroupVersions(ctx context.Context, inp syncLatestEnvGroupVersionsInput) error { + ctx, span := telemetry.NewSpan(ctx, "sync-latest-env-group-versions") + defer span.End() + + if inp.deploymentTargetID == "" { + return telemetry.Error(ctx, span, nil, "deployment target id is empty") + } + if inp.appName == "" { + return telemetry.Error(ctx, span, nil, "app name is empty") + } + if inp.appEnvName == "" { + return telemetry.Error(ctx, span, nil, "app env name is empty") + } + if inp.namespace == "" { + return telemetry.Error(ctx, span, nil, "namespace is empty") + } + if inp.k8sAgent == nil { + return telemetry.Error(ctx, span, nil, "k8s agent is nil") + } + + for _, envGroup := range inp.envGroups { + if envGroup == nil { + continue + } + + additionalEnvGroupLabels := map[string]string{ + LabelKey_AppName: inp.appName, + LabelKey_DeploymentTargetID: inp.deploymentTargetID, + LabelKey_PorterManaged: "true", + } + + if envGroup.GetName() == inp.appEnvName { + if inp.sameAppEnv { + continue + } + + additionalEnvGroupLabels[environment_groups.LabelKey_DefaultAppEnvironment] = "true" + } + + _, err := environment_groups.SyncLatestVersionToNamespace(ctx, inp.k8sAgent, environment_groups.SyncLatestVersionToNamespaceInput{ + TargetNamespace: inp.namespace, + BaseEnvironmentGroupName: envGroup.GetName(), + }, additionalEnvGroupLabels) + if err != nil { + telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "env-group-name", Value: envGroup.GetName()}) + return telemetry.Error(ctx, span, err, "error syncing latest version to namespace") + } + } + + return nil +} diff --git a/cli/cmd/v2/apply.go b/cli/cmd/v2/apply.go index 629a4cd789..1d74a596d0 100644 --- a/cli/cmd/v2/apply.go +++ b/cli/cmd/v2/apply.go @@ -73,7 +73,7 @@ func Apply(ctx context.Context, cliConf config.CLIConfig, client api.Client, por return fmt.Errorf("error getting app name from b64 app proto: %w", err) } - envGroupResp, err := client.CreateOrUpdateAppEnvironment(ctx, cliConf.Project, cliConf.Cluster, appName, targetResp.DeploymentTargetID, parseResp.EnvVariables, parseResp.EnvSecrets) + envGroupResp, err := client.CreateOrUpdateAppEnvironment(ctx, cliConf.Project, cliConf.Cluster, appName, targetResp.DeploymentTargetID, parseResp.EnvVariables, parseResp.EnvSecrets, parseResp.B64AppProto) if err != nil { return fmt.Errorf("error calling create or update app environment group endpoint: %w", err) } diff --git a/dashboard/src/main/home/app-dashboard/app-view/AppDataContainer.tsx b/dashboard/src/main/home/app-dashboard/app-view/AppDataContainer.tsx index 045f5d83ad..1f7e28021a 100644 --- a/dashboard/src/main/home/app-dashboard/app-view/AppDataContainer.tsx +++ b/dashboard/src/main/home/app-dashboard/app-view/AppDataContainer.tsx @@ -165,16 +165,17 @@ const AppDataContainer: React.FC = ({ tabParam }) => { ); // updates the default env group associated with this app to store app specific env vars - const res = await api.updateAppEnvironmentGroup( + const res = await api.updateEnvironmentGroupV2( "", { deployment_target_id: deploymentTargetId, variables, secrets, + b64_app_proto: btoa(validatedAppProto.toJsonString()), remove_missing: true, }, { - project_id: projectId, + id: projectId, cluster_id: clusterId, app_name: porterApp.name, } diff --git a/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx b/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx index 370d535333..96d7bc7408 100644 --- a/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx +++ b/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx @@ -262,6 +262,7 @@ const CreateApp: React.FC = ({ history }) => { { deployment_target_id: deploymentTarget.deployment_target_id, variables: variables, + b64_app_proto: btoa(app.toJsonString()), secrets: secrets, }, { diff --git a/dashboard/src/shared/api.tsx b/dashboard/src/shared/api.tsx index 0b45043389..7ddbeae691 100644 --- a/dashboard/src/shared/api.tsx +++ b/dashboard/src/shared/api.tsx @@ -914,22 +914,6 @@ const createApp = baseApi< return `/api/projects/${pathParams.project_id}/clusters/${pathParams.cluster_id}/apps/create`; }); -const updateAppEnvironmentGroup = baseApi< - { - deployment_target_id: string; - variables: Record; - secrets: Record; - remove_missing: boolean; - }, - { - project_id: number; - cluster_id: number; - app_name: string; - } ->("POST", (pathParams) => { - return `/api/projects/${pathParams.project_id}/clusters/${pathParams.cluster_id}/apps/${pathParams.app_name}/update-environment`; -}); - const applyApp = baseApi< { deployment_target_id: string; @@ -1852,6 +1836,7 @@ const updateEnvironmentGroupV2 = baseApi< deployment_target_id: string; variables: Record; secrets: Record; + b64_app_proto: string; remove_missing?: boolean; }, { @@ -3108,7 +3093,6 @@ export default { getBranchHead, validatePorterApp, createApp, - updateAppEnvironmentGroup, applyApp, getAttachedEnvGroups, getLatestRevision, diff --git a/internal/porter_app/v2/yaml.go b/internal/porter_app/v2/yaml.go index 57d73533fe..0cb2e0c044 100644 --- a/internal/porter_app/v2/yaml.go +++ b/internal/porter_app/v2/yaml.go @@ -100,6 +100,7 @@ type PorterYAML struct { Env map[string]string `yaml:"env"` Predeploy *Service `yaml:"predeploy"` + EnvGroups []string `yaml:"env_groups,omitempty"` } // Build represents the build settings for a Porter app From a4c9ae7500f91cdd526f899725eb8c3eff83dfd7 Mon Sep 17 00:00:00 2001 From: jose-fully-ported <141160579+jose-fully-ported@users.noreply.github.com> Date: Wed, 20 Sep 2023 10:57:46 -0400 Subject: [PATCH 09/15] fix: use correct builder for heroku-22 (#3606) --- api/server/handlers/gitinstallation/get_buildpack.go | 2 +- .../handlers/project_integration/get_gitlab_repo_buildpack.go | 2 +- dashboard/src/components/repo-selector/BuildpackSelection.tsx | 2 +- dashboard/src/main/home/app-dashboard/types/buildpack.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server/handlers/gitinstallation/get_buildpack.go b/api/server/handlers/gitinstallation/get_buildpack.go index d03f1d4616..4693da1fa7 100644 --- a/api/server/handlers/gitinstallation/get_buildpack.go +++ b/api/server/handlers/gitinstallation/get_buildpack.go @@ -29,7 +29,7 @@ func initBuilderInfo() map[string]*buildpacks.BuilderInfo { builders[buildpacks.HerokuBuilder] = &buildpacks.BuilderInfo{ Name: "Heroku", Builders: []string{ - "heroku/buildpacks:22", + "heroku/builder:22", "heroku/buildpacks:20", "heroku/buildpacks:18", }, diff --git a/api/server/handlers/project_integration/get_gitlab_repo_buildpack.go b/api/server/handlers/project_integration/get_gitlab_repo_buildpack.go index 33ef8fb3fa..f4e21b6f32 100644 --- a/api/server/handlers/project_integration/get_gitlab_repo_buildpack.go +++ b/api/server/handlers/project_integration/get_gitlab_repo_buildpack.go @@ -137,7 +137,7 @@ func initBuilderInfo() map[string]*buildpacks.BuilderInfo { builders[buildpacks.HerokuBuilder] = &buildpacks.BuilderInfo{ Name: "Heroku", Builders: []string{ - "heroku/buildpacks:22", + "heroku/builder:22", "heroku/buildpacks:20", "heroku/buildpacks:18", }, diff --git a/dashboard/src/components/repo-selector/BuildpackSelection.tsx b/dashboard/src/components/repo-selector/BuildpackSelection.tsx index 86c75d4010..ad9f0e116e 100644 --- a/dashboard/src/components/repo-selector/BuildpackSelection.tsx +++ b/dashboard/src/components/repo-selector/BuildpackSelection.tsx @@ -11,7 +11,7 @@ import styled, { keyframes } from "styled-components"; const DEFAULT_BUILDER_NAME = "heroku"; const DEFAULT_PAKETO_STACK = "paketobuildpacks/builder-jammy-full:latest"; -const DEFAULT_HEROKU_STACK = "heroku/buildpacks:22"; +const DEFAULT_HEROKU_STACK = "heroku/builder:22"; type BuildConfig = { builder: string; diff --git a/dashboard/src/main/home/app-dashboard/types/buildpack.ts b/dashboard/src/main/home/app-dashboard/types/buildpack.ts index 0704017db8..0ec2a15af8 100644 --- a/dashboard/src/main/home/app-dashboard/types/buildpack.ts +++ b/dashboard/src/main/home/app-dashboard/types/buildpack.ts @@ -25,7 +25,7 @@ export type DetectedBuildpack = z.infer; export const DEFAULT_BUILDER_NAME = "heroku"; export const DEFAULT_PAKETO_STACK = "paketobuildpacks/builder-jammy-full:latest"; -export const DEFAULT_HEROKU_STACK = "heroku/buildpacks:22"; +export const DEFAULT_HEROKU_STACK = "heroku/builder:22"; export const BUILDPACK_TO_NAME: { [key: string]: string } = { "heroku/nodejs": "NodeJS", From 0091d6bb2fa9bd8db151300174da370f25601cda Mon Sep 17 00:00:00 2001 From: d-g-town <66391417+d-g-town@users.noreply.github.com> Date: Wed, 20 Sep 2023 12:18:08 -0400 Subject: [PATCH 10/15] sort tags alphabetically (#3608) Co-authored-by: David Townley --- internal/repository/gorm/tag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repository/gorm/tag.go b/internal/repository/gorm/tag.go index fabed57b6e..d6812272f1 100644 --- a/internal/repository/gorm/tag.go +++ b/internal/repository/gorm/tag.go @@ -80,7 +80,7 @@ func (repo *TagRepository) ReadTagByNameAndProjectId(tagName string, projectId u func (repo *TagRepository) ListTagsByProjectId(projectId uint) ([]*models.Tag, error) { tags := make([]*models.Tag, 0) - err := repo.db.Preload("Releases").Where("project_id = ?", projectId).Find(&tags).Error + err := repo.db.Preload("Releases").Where("project_id = ?", projectId).Order("name asc").Find(&tags).Error if err != nil { return nil, err } From 40daca35d846ee9d46c00b258b90bbf6cd5b43bf Mon Sep 17 00:00:00 2001 From: ianedwards Date: Wed, 20 Sep 2023 12:51:29 -0400 Subject: [PATCH 11/15] POR-1781 associated attached env versions with revision (#3609) --- .../update_app_environment_group.go | 105 ++++++++++-------- cli/cmd/v2/apply.go | 23 ++-- .../app-view/AppDataContainer.tsx | 24 ++-- .../app-dashboard/create-app/CreateApp.tsx | 37 +++--- .../kubernetes/environment_groups/list.go | 2 +- 5 files changed, 100 insertions(+), 91 deletions(-) diff --git a/api/server/handlers/porter_app/update_app_environment_group.go b/api/server/handlers/porter_app/update_app_environment_group.go index 22ed7b1655..1e695e9e61 100644 --- a/api/server/handlers/porter_app/update_app_environment_group.go +++ b/api/server/handlers/porter_app/update_app_environment_group.go @@ -69,8 +69,7 @@ type UpdateAppEnvironmentRequest struct { // UpdateAppEnvironmentResponse represents the fields on the response object from the /apps/{porter_app_name}/environment-group endpoint type UpdateAppEnvironmentResponse struct { - EnvGroupName string `json:"env_group_name"` - EnvGroupVersion int `json:"env_group_version"` + EnvGroups []environment_groups.EnvironmentGroup `json:"env_groups"` } // ServeHTTP updates or creates the environment group for an app @@ -221,21 +220,24 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R envGroups: appProto.EnvGroups, appName: appName, appEnvName: appEnvGroupName, - sameAppEnv: true, namespace: namespace, deploymentTargetID: request.DeploymentTargetID, k8sAgent: agent, } - err = syncLatestEnvGroupVersions(ctx, syncInp) + latestEnvGroups, err := syncLatestEnvGroupVersions(ctx, syncInp) if err != nil { err := telemetry.Error(ctx, span, err, "error syncing latest env group versions") c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) return } + latestEnvGroups = append(latestEnvGroups, environment_groups.EnvironmentGroup{ + Name: latestEnvironmentGroup.Name, + Version: latestEnvironmentGroup.Version, + }) + res := &UpdateAppEnvironmentResponse{ - EnvGroupName: latestEnvironmentGroup.Name, - EnvGroupVersion: latestEnvironmentGroup.Version, + EnvGroups: latestEnvGroups, } c.WriteResult(w, r, res) @@ -288,47 +290,50 @@ func (c *UpdateAppEnvironmentHandler) ServeHTTP(w http.ResponseWriter, r *http.R TargetNamespace: namespace, } - syncedEnvironment, err := environment_groups.SyncLatestVersionToNamespace(ctx, agent, inp, additionalEnvGroupLabels) + syncedAppEnvironment, err := environment_groups.SyncLatestVersionToNamespace(ctx, agent, inp, additionalEnvGroupLabels) if err != nil { err := telemetry.Error(ctx, span, err, "unable to create or update synced environment group") c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) return } - telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "env-group-versioned-name", Value: syncedEnvironment.EnvironmentGroupVersionedName}) - - split := strings.Split(syncedEnvironment.EnvironmentGroupVersionedName, ".") - if len(split) != 2 { - err := telemetry.Error(ctx, span, err, "unexpected environment group versioned name") - c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) - return - } - - version, err := strconv.Atoi(split[1]) - if err != nil { - err := telemetry.Error(ctx, span, err, "error converting environment group version to int") - c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) - return - } + telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "env-group-versioned-name", Value: syncedAppEnvironment.EnvironmentGroupVersionedName}) syncInp := syncLatestEnvGroupVersionsInput{ envGroups: appProto.EnvGroups, appName: appName, appEnvName: appEnvGroupName, - sameAppEnv: false, namespace: namespace, deploymentTargetID: request.DeploymentTargetID, k8sAgent: agent, } - err = syncLatestEnvGroupVersions(ctx, syncInp) + latestEnvGroups, err := syncLatestEnvGroupVersions(ctx, syncInp) if err != nil { err := telemetry.Error(ctx, span, err, "error syncing latest env group versions") c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) return } + split := strings.Split(syncedAppEnvironment.EnvironmentGroupVersionedName, ".") + if len(split) != 2 { + err := telemetry.Error(ctx, span, err, "unexpected environment group versioned name") + c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) + return + } + + version, err := strconv.Atoi(split[1]) + if err != nil { + err := telemetry.Error(ctx, span, err, "error converting environment group version to int") + c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError)) + return + } + + latestEnvGroups = append(latestEnvGroups, environment_groups.EnvironmentGroup{ + Name: split[0], + Version: version, + }) + res := &UpdateAppEnvironmentResponse{ - EnvGroupName: split[0], - EnvGroupVersion: version, + EnvGroups: latestEnvGroups, } c.WriteResult(w, r, res) @@ -341,8 +346,6 @@ type syncLatestEnvGroupVersionsInput struct { appName string // appEnvName is the name of the app env. This is the env group created when the app is created for storing app-specific variables appEnvName string - // sameAppEnv is true if the app env group variables are unchanged. If true, we do not need to sync the latest version of the app env group - sameAppEnv bool // namespace is the namespace to sync the latest versions to namespace string // deploymentTargetID is the id of the deployment target @@ -352,30 +355,35 @@ type syncLatestEnvGroupVersionsInput struct { } // syncLatestEnvGroupVersions syncs the latest versions of the env groups to the namespace where an app is deployed -func syncLatestEnvGroupVersions(ctx context.Context, inp syncLatestEnvGroupVersionsInput) error { +func syncLatestEnvGroupVersions(ctx context.Context, inp syncLatestEnvGroupVersionsInput) ([]environment_groups.EnvironmentGroup, error) { ctx, span := telemetry.NewSpan(ctx, "sync-latest-env-group-versions") defer span.End() + var envGroups []environment_groups.EnvironmentGroup + if inp.deploymentTargetID == "" { - return telemetry.Error(ctx, span, nil, "deployment target id is empty") + return envGroups, telemetry.Error(ctx, span, nil, "deployment target id is empty") } if inp.appName == "" { - return telemetry.Error(ctx, span, nil, "app name is empty") + return envGroups, telemetry.Error(ctx, span, nil, "app name is empty") } if inp.appEnvName == "" { - return telemetry.Error(ctx, span, nil, "app env name is empty") + return envGroups, telemetry.Error(ctx, span, nil, "app env name is empty") } if inp.namespace == "" { - return telemetry.Error(ctx, span, nil, "namespace is empty") + return envGroups, telemetry.Error(ctx, span, nil, "namespace is empty") } if inp.k8sAgent == nil { - return telemetry.Error(ctx, span, nil, "k8s agent is nil") + return envGroups, telemetry.Error(ctx, span, nil, "k8s agent is nil") } for _, envGroup := range inp.envGroups { if envGroup == nil { continue } + if envGroup.GetName() == inp.appEnvName { + continue + } additionalEnvGroupLabels := map[string]string{ LabelKey_AppName: inp.appName, @@ -383,23 +391,30 @@ func syncLatestEnvGroupVersions(ctx context.Context, inp syncLatestEnvGroupVersi LabelKey_PorterManaged: "true", } - if envGroup.GetName() == inp.appEnvName { - if inp.sameAppEnv { - continue - } - - additionalEnvGroupLabels[environment_groups.LabelKey_DefaultAppEnvironment] = "true" - } - - _, err := environment_groups.SyncLatestVersionToNamespace(ctx, inp.k8sAgent, environment_groups.SyncLatestVersionToNamespaceInput{ + syncedEnvironment, err := environment_groups.SyncLatestVersionToNamespace(ctx, inp.k8sAgent, environment_groups.SyncLatestVersionToNamespaceInput{ TargetNamespace: inp.namespace, BaseEnvironmentGroupName: envGroup.GetName(), }, additionalEnvGroupLabels) if err != nil { telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "env-group-name", Value: envGroup.GetName()}) - return telemetry.Error(ctx, span, err, "error syncing latest version to namespace") + return envGroups, telemetry.Error(ctx, span, err, "error syncing latest version to namespace") } + + split := strings.Split(syncedEnvironment.EnvironmentGroupVersionedName, ".") + if len(split) != 2 { + return envGroups, telemetry.Error(ctx, span, err, "unexpected environment group versioned name") + } + + version, err := strconv.Atoi(split[1]) + if err != nil { + return envGroups, telemetry.Error(ctx, span, err, "error converting environment group version to int") + } + + envGroups = append(envGroups, environment_groups.EnvironmentGroup{ + Name: split[0], + Version: version, + }) } - return nil + return envGroups, nil } diff --git a/cli/cmd/v2/apply.go b/cli/cmd/v2/apply.go index 1d74a596d0..cf0a43ae1a 100644 --- a/cli/cmd/v2/apply.go +++ b/cli/cmd/v2/apply.go @@ -12,6 +12,7 @@ import ( "github.com/porter-dev/porter/api/server/handlers/porter_app" "github.com/porter-dev/porter/api/types" + "github.com/porter-dev/porter/internal/kubernetes/environment_groups" "github.com/porter-dev/porter/internal/models" "github.com/cli/cli/git" @@ -78,7 +79,7 @@ func Apply(ctx context.Context, cliConf config.CLIConfig, client api.Client, por return fmt.Errorf("error calling create or update app environment group endpoint: %w", err) } - b64AppProto, err = updateAppEnvGroupInProto(ctx, b64AppProto, envGroupResp.EnvGroupName, envGroupResp.EnvGroupVersion) + b64AppProto, err = updateEnvGroupsInProto(ctx, b64AppProto, envGroupResp.EnvGroups) if err != nil { return fmt.Errorf("error updating app env group in proto: %w", err) } @@ -355,7 +356,7 @@ func imageTagFromBase64AppProto(base64AppProto string) (string, error) { return app.Image.Tag, nil } -func updateAppEnvGroupInProto(ctx context.Context, base64AppProto string, envGroupName string, envGroupVersion int) (string, error) { +func updateEnvGroupsInProto(ctx context.Context, base64AppProto string, envGroups []environment_groups.EnvironmentGroup) (string, error) { var editedB64AppProto string decoded, err := base64.StdEncoding.DecodeString(base64AppProto) @@ -369,20 +370,14 @@ func updateAppEnvGroupInProto(ctx context.Context, base64AppProto string, envGro return editedB64AppProto, fmt.Errorf("unable to unmarshal app for revision: %w", err) } - envGroupExists := false - for _, envGroup := range app.EnvGroups { - if envGroup.Name == envGroupName { - envGroup.Version = int64(envGroupVersion) - envGroupExists = true - break - } - } - if !envGroupExists { - app.EnvGroups = append(app.EnvGroups, &porterv1.EnvGroup{ - Name: envGroupName, - Version: int64(envGroupVersion), + egs := make([]*porterv1.EnvGroup, 0) + for _, envGroup := range envGroups { + egs = append(egs, &porterv1.EnvGroup{ + Name: envGroup.Name, + Version: int64(envGroup.Version), }) } + app.EnvGroups = egs marshalled, err := helpers.MarshalContractObject(ctx, app) if err != nil { diff --git a/dashboard/src/main/home/app-dashboard/app-view/AppDataContainer.tsx b/dashboard/src/main/home/app-dashboard/app-view/AppDataContainer.tsx index 1f7e28021a..74ea6257a3 100644 --- a/dashboard/src/main/home/app-dashboard/app-view/AppDataContainer.tsx +++ b/dashboard/src/main/home/app-dashboard/app-view/AppDataContainer.tsx @@ -181,25 +181,23 @@ const AppDataContainer: React.FC = ({ tabParam }) => { } ); - const updatedEnvGroup = z + const updatedEnvGroups = z .object({ - env_group_name: z.string(), - env_group_version: z.coerce.bigint(), + env_groups: z + .object({ + name: z.string(), + latest_version: z.coerce.bigint(), + }) + .array(), }) .parse(res.data); const protoWithUpdatedEnv = new PorterApp({ ...validatedAppProto, - envGroups: validatedAppProto.envGroups.map((envGroup) => { - if (envGroup.name === updatedEnvGroup.env_group_name) { - return { - ...envGroup, - version: updatedEnvGroup.env_group_version, - }; - } - - return envGroup; - }), + envGroups: updatedEnvGroups.env_groups.map((eg) => ({ + name: eg.name, + version: eg.latest_version, + })), }); await api.applyApp( diff --git a/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx b/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx index 96d7bc7408..3c22e1a234 100644 --- a/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx +++ b/dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx @@ -183,7 +183,10 @@ const CreateApp: React.FC = ({ history }) => { porterYamlFound, detectedName, loading: isLoadingPorterYaml, - } = usePorterYaml({ source: source?.type === "github" ? source : null, appName: name.value }); + } = usePorterYaml({ + source: source?.type === "github" ? source : null, + appName: name.value, + }); const deploymentTarget = useDefaultDeploymentTarget(); const { updateAppStep } = useAppAnalytics(name.value); const { validateApp } = useAppValidation({ @@ -257,7 +260,7 @@ const CreateApp: React.FC = ({ history }) => { } ); - const envGroupResponse = await api.updateEnvironmentGroupV2( + const res = await api.updateEnvironmentGroupV2( "", { deployment_target_id: deploymentTarget.deployment_target_id, @@ -272,31 +275,29 @@ const CreateApp: React.FC = ({ history }) => { } ); - const addedEnvGroup = await z + const updatedEnvGroups = z .object({ - env_group_name: z.string(), - env_group_version: z.coerce.bigint(), + env_groups: z + .object({ + name: z.string(), + latest_version: z.coerce.bigint(), + }) + .array(), }) - .parseAsync(envGroupResponse.data); + .parse(res.data); - const envGroups = [ - ...app.envGroups.filter( - (group) => group.name !== addedEnvGroup.env_group_name - ), - { - name: addedEnvGroup.env_group_name, - version: addedEnvGroup.env_group_version, - }, - ]; - const appWithSeededEnv = new PorterApp({ + const protoWithUpdatedEnv = new PorterApp({ ...app, - envGroups, + envGroups: updatedEnvGroups.env_groups.map((eg) => ({ + name: eg.name, + version: eg.latest_version, + })), }); await api.applyApp( "", { - b64_app_proto: btoa(appWithSeededEnv.toJsonString()), + b64_app_proto: btoa(protoWithUpdatedEnv.toJsonString()), deployment_target_id: deploymentTarget.deployment_target_id, }, { diff --git a/internal/kubernetes/environment_groups/list.go b/internal/kubernetes/environment_groups/list.go index 43cc222f62..cfe3eb3e54 100644 --- a/internal/kubernetes/environment_groups/list.go +++ b/internal/kubernetes/environment_groups/list.go @@ -35,7 +35,7 @@ type EnvironmentGroup struct { // SecretVariables are secret values for the EnvironmentGroup. This usually will be a Secret on the kubernetes cluster SecretVariables map[string]string `json:"secret_variables,omitempty"` // CreatedAt is only used for display purposes and is in UTC Unix time - CreatedAtUTC time.Time `json:"created_at"` + CreatedAtUTC time.Time `json:"created_at,omitempty"` } type environmentGroupOptions struct { From 67c6965e8fcf182f871e90cb3e873409c1bc9bce Mon Sep 17 00:00:00 2001 From: jose-fully-ported <141160579+jose-fully-ported@users.noreply.github.com> Date: Wed, 20 Sep 2023 13:09:26 -0400 Subject: [PATCH 12/15] fix: switch the default heroku builder to heroku-20 (#3611) --- dashboard/src/components/repo-selector/BuildpackSelection.tsx | 2 +- dashboard/src/main/home/app-dashboard/types/buildpack.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/src/components/repo-selector/BuildpackSelection.tsx b/dashboard/src/components/repo-selector/BuildpackSelection.tsx index ad9f0e116e..57c0a02081 100644 --- a/dashboard/src/components/repo-selector/BuildpackSelection.tsx +++ b/dashboard/src/components/repo-selector/BuildpackSelection.tsx @@ -11,7 +11,7 @@ import styled, { keyframes } from "styled-components"; const DEFAULT_BUILDER_NAME = "heroku"; const DEFAULT_PAKETO_STACK = "paketobuildpacks/builder-jammy-full:latest"; -const DEFAULT_HEROKU_STACK = "heroku/builder:22"; +const DEFAULT_HEROKU_STACK = "heroku/buildpacks:20"; type BuildConfig = { builder: string; diff --git a/dashboard/src/main/home/app-dashboard/types/buildpack.ts b/dashboard/src/main/home/app-dashboard/types/buildpack.ts index 0ec2a15af8..198c436a7b 100644 --- a/dashboard/src/main/home/app-dashboard/types/buildpack.ts +++ b/dashboard/src/main/home/app-dashboard/types/buildpack.ts @@ -25,7 +25,7 @@ export type DetectedBuildpack = z.infer; export const DEFAULT_BUILDER_NAME = "heroku"; export const DEFAULT_PAKETO_STACK = "paketobuildpacks/builder-jammy-full:latest"; -export const DEFAULT_HEROKU_STACK = "heroku/builder:22"; +export const DEFAULT_HEROKU_STACK = "heroku/buildpacks:20"; export const BUILDPACK_TO_NAME: { [key: string]: string } = { "heroku/nodejs": "NodeJS", From 5830575c02d1122de7b183ab7e80a694f1d463c8 Mon Sep 17 00:00:00 2001 From: d-g-town <66391417+d-g-town@users.noreply.github.com> Date: Wed, 20 Sep 2023 13:14:39 -0400 Subject: [PATCH 13/15] support ccp image call (#3607) Co-authored-by: David Townley --- .../src/components/image-selector/TagList.tsx | 2 +- go.mod | 2 +- go.sum | 4 +- internal/registry/registry.go | 44 +++++++++++++++++++ 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/dashboard/src/components/image-selector/TagList.tsx b/dashboard/src/components/image-selector/TagList.tsx index bad4a6b502..64d1e91596 100644 --- a/dashboard/src/components/image-selector/TagList.tsx +++ b/dashboard/src/components/image-selector/TagList.tsx @@ -86,7 +86,7 @@ export default class TagList extends Component { const [latestImage] = tags.splice(latestImageIndex, 1); tags.unshift(latestImage); } - this.setState({ tags: tags.map((tag) => tag.tag), loading: false }); + this.setState({ tags: tags.map((tag) => tag.tag), loading: false, error: false }); }) .catch((err) => { console.log(err); diff --git a/go.mod b/go.mod index b24c1502b0..c0873894d9 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/matryer/is v1.4.0 github.com/nats-io/nats.go v1.24.0 github.com/open-policy-agent/opa v0.44.0 - github.com/porter-dev/api-contracts v0.1.5 + github.com/porter-dev/api-contracts v0.1.6 github.com/riandyrn/otelchi v0.5.1 github.com/santhosh-tekuri/jsonschema/v5 v5.0.1 github.com/stefanmcshane/helm v0.0.0-20221213002717-88a4a2c6e77d diff --git a/go.sum b/go.sum index 2406e2826e..59575460fe 100644 --- a/go.sum +++ b/go.sum @@ -1516,8 +1516,8 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polyfloyd/go-errorlint v0.0.0-20210722154253-910bb7978349/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw= -github.com/porter-dev/api-contracts v0.1.5 h1:Nz0bEIXedKHYfC0YzOj579ABXHxDaH4DXjKcstvWR8A= -github.com/porter-dev/api-contracts v0.1.5/go.mod h1:fX6JmP5QuzxDLvqP3evFOTXjI4dHxsG0+VKNTjImZU8= +github.com/porter-dev/api-contracts v0.1.6 h1:nMP/+M53Mohwe/1mNq/HYEnfIKEiDHEFItZwjLmPZ+8= +github.com/porter-dev/api-contracts v0.1.6/go.mod h1:fX6JmP5QuzxDLvqP3evFOTXjI4dHxsG0+VKNTjImZU8= github.com/porter-dev/switchboard v0.0.3 h1:dBuYkiVLa5Ce7059d6qTe9a1C2XEORFEanhbtV92R+M= github.com/porter-dev/switchboard v0.0.3/go.mod h1:xSPzqSFMQ6OSbp42fhCi4AbGbQbsm6nRvOkrblFeXU4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= diff --git a/internal/registry/registry.go b/internal/registry/registry.go index f35f55407d..23fc3031cc 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -1026,6 +1026,17 @@ func (r *Registry) ListImages( repo repository.Repository, conf *config.Config, ) ([]*ptypes.Image, error) { + ctx, span := telemetry.NewSpan(ctx, "list-repositories") + defer span.End() + + telemetry.WithAttributes(span, + telemetry.AttributeKV{Key: "registry-name", Value: r.Name}, + telemetry.AttributeKV{Key: "registry-id", Value: r.ID}, + telemetry.AttributeKV{Key: "registry-url", Value: r.URL}, + telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID}, + telemetry.AttributeKV{Key: "repo-name", Value: repoName}, + ) + // switch on the auth mechanism to get a token if r.AWSIntegrationID != 0 { aws, err := repo.AWSIntegration().ReadAWSIntegration( @@ -1064,6 +1075,39 @@ func (r *Registry) ListImages( } if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) { + + if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") { + req := connect.NewRequest(&porterv1.ListImagesForRepositoryRequest{ + ProjectId: int64(r.ProjectID), + RegistryUri: r.URL, + RepoName: repoName, + }) + + resp, err := conf.ClusterControlPlaneClient.ListImagesForRepository(ctx, req) + if err != nil { + return nil, telemetry.Error(ctx, span, err, "error calling ccp list images") + } + + res := make([]*ptypes.Image, 0) + + for _, image := range resp.Msg.Images { + if image.UpdatedAt == nil { + continue + } + lastUpdateTime := image.UpdatedAt.AsTime() + + res = append(res, &ptypes.Image{ + Digest: image.Digest, + Tag: image.Tag, + Manifest: "", + RepositoryName: image.RepositoryName, + PushedAt: &lastUpdateTime, + }) + } + + return res, nil + } + uri := strings.TrimPrefix(r.URL, "https://") splits := strings.Split(uri, ".") accountID := splits[0] From 078792cff0259754a3bb0b9067ff532333809599 Mon Sep 17 00:00:00 2001 From: jose-fully-ported <141160579+jose-fully-ported@users.noreply.github.com> Date: Wed, 20 Sep 2023 13:24:44 -0400 Subject: [PATCH 14/15] feat: add telemetry for the cluster status endpoint (#3610) --- api/server/handlers/cluster/cluster_status.go | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/api/server/handlers/cluster/cluster_status.go b/api/server/handlers/cluster/cluster_status.go index cc4b013e01..9dc44c87ba 100644 --- a/api/server/handlers/cluster/cluster_status.go +++ b/api/server/handlers/cluster/cluster_status.go @@ -13,6 +13,7 @@ import ( "github.com/porter-dev/porter/api/server/shared/config" "github.com/porter-dev/porter/api/types" "github.com/porter-dev/porter/internal/models" + "github.com/porter-dev/porter/internal/telemetry" ) type ClusterStatusHandler struct { @@ -40,22 +41,25 @@ type ClusterStatusResponse struct { } func (c *ClusterStatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - cluster, _ := ctx.Value(types.ClusterScope).(*models.Cluster) + ctx, span := telemetry.NewSpan(r.Context(), "serve-cluster-status") + defer span.End() + cluster, _ := ctx.Value(types.ClusterScope).(*models.Cluster) req := connect.NewRequest(&porterv1.ClusterStatusRequest{ ProjectId: int64(cluster.ProjectID), ClusterId: int64(cluster.ID), }) status, err := c.Config().ClusterControlPlaneClient.ClusterStatus(ctx, req) if err != nil { - e := fmt.Errorf("unable to retrieve status for cluster: %w", err) - c.HandleAPIError(w, r, apierrors.NewErrInternal(e)) + err := fmt.Errorf("unable to retrieve status for cluster: %w", err) + err = telemetry.Error(ctx, span, err, err.Error()) + c.HandleAPIError(w, r, apierrors.NewErrInternal(err)) return } if status.Msg == nil { - e := fmt.Errorf("unable to parse status for cluster: %w", err) - c.HandleAPIError(w, r, apierrors.NewErrInternal(e)) + err := fmt.Errorf("unable to parse status for cluster: %w", err) + err = telemetry.Error(ctx, span, err, err.Error()) + c.HandleAPIError(w, r, apierrors.NewErrInternal(err)) return } statusResp := status.Msg @@ -68,6 +72,12 @@ func (c *ClusterStatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) IsControlPlaneReady: statusResp.ControlPlaneStatus, } + telemetry.WithAttributes(span, + telemetry.AttributeKV{Key: "cluster-phase", Value: statusResp.Phase}, + telemetry.AttributeKV{Key: "cluster-infra-status", Value: statusResp.InfrastructureStatus}, + telemetry.AttributeKV{Key: "cluster-control-plane-status", Value: statusResp.ControlPlaneStatus}, + ) + c.WriteResult(w, r, resp) w.WriteHeader(http.StatusOK) } From 73742e9e020d926fa68873a3385ab5ef81f40a30 Mon Sep 17 00:00:00 2001 From: Feroze Mohideen Date: Wed, 20 Sep 2023 13:28:32 -0400 Subject: [PATCH 15/15] filter out pre-deploy logs from 'all' logs view (#3597) --- .../app-dashboard/app-view/tabs/LogsTab.tsx | 1 + .../validate-apply/logs/Logs.tsx | 21 ++++++++-------- .../validate-apply/logs/utils.ts | 25 +++++++++++++++++-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/dashboard/src/main/home/app-dashboard/app-view/tabs/LogsTab.tsx b/dashboard/src/main/home/app-dashboard/app-view/tabs/LogsTab.tsx index 9cca803d1d..bea5a6ec2d 100644 --- a/dashboard/src/main/home/app-dashboard/app-view/tabs/LogsTab.tsx +++ b/dashboard/src/main/home/app-dashboard/app-view/tabs/LogsTab.tsx @@ -15,6 +15,7 @@ const LogsTab: React.FC = () => { appName={appName} serviceNames={serviceNames} deploymentTargetId={deploymentTargetId} + filterPredeploy={true} /> ); }; diff --git a/dashboard/src/main/home/app-dashboard/validate-apply/logs/Logs.tsx b/dashboard/src/main/home/app-dashboard/validate-apply/logs/Logs.tsx index ec522ff4ea..85fea490e1 100644 --- a/dashboard/src/main/home/app-dashboard/validate-apply/logs/Logs.tsx +++ b/dashboard/src/main/home/app-dashboard/validate-apply/logs/Logs.tsx @@ -25,10 +25,8 @@ import Container from "components/porter/Container"; import Button from "components/porter/Button"; import LogFilterContainer from "../../expanded-app/logs/LogFilterContainer"; import StyledLogs from "../../expanded-app/logs/StyledLogs"; -import { AppRevision } from "lib/revisions/types"; import { useLatestRevisionNumber, useRevisionIdToNumber } from "lib/hooks/useRevisionList"; import { useLocation } from "react-router"; -import { valueExists } from "shared/util"; type Props = { projectId: number; @@ -38,6 +36,7 @@ type Props = { deploymentTargetId: string; appRevisionId?: string; logFilterNames?: LogFilterName[]; + filterPredeploy?: boolean; }; const Logs: React.FC = ({ @@ -48,6 +47,7 @@ const Logs: React.FC = ({ deploymentTargetId, appRevisionId, logFilterNames = ["service_name", "revision", "output_stream"], + filterPredeploy = false, }) => { const { search } = useLocation(); const queryParams = new URLSearchParams(search); @@ -173,20 +173,21 @@ const Logs: React.FC = ({ }, 5000); }; - const { logs, refresh, moveCursor, paginationInfo } = useLogs( - projectId, - clusterId, + const { logs, refresh, moveCursor, paginationInfo } = useLogs({ + projectID: projectId, + clusterID: clusterId, selectedFilterValues, appName, - selectedFilterValues.service_name, + serviceName: selectedFilterValues.service_name, deploymentTargetId, - enteredSearchText, + searchParam: enteredSearchText, notify, - setIsLoading, + setLoading: setIsLoading, revisionIdToNumber, - selectedDate, + setDate: selectedDate, appRevisionId, - ); + filterPredeploy, + }); useEffect(() => { setFilters([ diff --git a/dashboard/src/main/home/app-dashboard/validate-apply/logs/utils.ts b/dashboard/src/main/home/app-dashboard/validate-apply/logs/utils.ts index 4822c9afee..df4e846398 100644 --- a/dashboard/src/main/home/app-dashboard/validate-apply/logs/utils.ts +++ b/dashboard/src/main/home/app-dashboard/validate-apply/logs/utils.ts @@ -41,7 +41,22 @@ export const parseLogs = (logs: any[] = []): PorterLog[] => { }); }; -export const useLogs = ( +export const useLogs = ({ + projectID, + clusterID, + selectedFilterValues, + appName, + serviceName, + deploymentTargetId, + searchParam, + notify, + setLoading, + revisionIdToNumber, + setDate, + appRevisionId = "", + timeRange, + filterPredeploy, +}: { projectID: number, clusterID: number, selectedFilterValues: Record, @@ -54,11 +69,13 @@ export const useLogs = ( revisionIdToNumber: Record, // if setDate is set, results are not live setDate?: Date, - appRevisionId: string = "", + appRevisionId?: string, timeRange?: { startTime?: Dayjs, endTime?: Dayjs, }, + filterPredeploy: boolean, +} ) => { const isLive = !setDate; const logsBufferRef = useRef([]); @@ -218,6 +235,10 @@ export const useLogs = ( return false; } + if (filterPredeploy && (log.metadata.raw_labels?.porter_run_service_name ?? "").endsWith("predeploy")) { + return false; + } + if (selectedFilterValues.revision !== GenericLogFilter.getDefaultOption("revision").value && log.metadata.revision !== selectedFilterValues.revision) { return false;