Skip to content

Commit

Permalink
Breaking change: Rework taint model in GKE (#9011)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rileykarson
rileykarson authored Sep 22, 2023
1 parent fe50764 commit c896b46
Show file tree
Hide file tree
Showing 6 changed files with 106 additions and 200 deletions.
182 changes: 84 additions & 98 deletions mmv1/third_party/terraform/services/container/node_config.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,18 +411,10 @@ func schemaNodeConfig() *schema.Schema {
},

"taint": {
Type: schema.TypeList,
Optional: true,
// Computed=true because GKE Sandbox will automatically add taints to nodes that can/cannot run sandboxed pods.
Computed: true,
ForceNew: true,
// Legacy config mode allows explicitly defining an empty taint.
// See https://www.terraform.io/docs/configuration/attr-as-blocks.html
ConfigMode: schema.SchemaConfigModeAttr,
Type: schema.TypeList,
Optional: true,
ForceNew: true,
Description: `List of Kubernetes taints to be applied to each node.`,
<% unless version.nil? || version == 'ga' -%>
DiffSuppressFunc: containerNodePoolTaintSuppress,
<% end -%>
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"key": {
Expand All @@ -448,6 +440,31 @@ func schemaNodeConfig() *schema.Schema {
},
},

"effective_taints": {
Type: schema.TypeList,
Computed: true,
Description: `List of kubernetes taints applied to each node.`,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"key": {
Type: schema.TypeString,
Computed: true,
Description: `Key for taint.`,
},
"value": {
Type: schema.TypeString,
Computed: true,
Description: `Value for taint.`,
},
"effect": {
Type: schema.TypeString,
Computed: true,
Description: `Effect for taint.`,
},
},
},
},

"workload_metadata_config": {
Computed: true,
Type: schema.TypeList,
Expand Down Expand Up @@ -897,12 +914,13 @@ func expandNodeConfig(v interface{}) *container.NodeConfig {
Value: data["value"].(string),
Effect: data["effect"].(string),
}

nodeTaints = append(nodeTaints, taint)
}

nc.Taints = nodeTaints
}


if v, ok := nodeConfig["workload_metadata_config"]; ok {
nc.WorkloadMetadataConfig = expandWorkloadMetadataConfig(v)
}
Expand Down Expand Up @@ -1081,11 +1099,11 @@ func expandConfidentialNodes(configured interface{}) *container.ConfidentialNode
}

func flattenNodeConfigDefaults(c *container.NodeConfigDefaults) []map[string]interface{} {
result := make([]map[string]interface{}, 0, 1)
result := make([]map[string]interface{}, 0, 1)

if c == nil {
return result
}
if c == nil {
return result
}

result = append(result, map[string]interface{}{})

Expand All @@ -1094,16 +1112,27 @@ func flattenNodeConfigDefaults(c *container.NodeConfigDefaults) []map[string]int
<% unless version == 'ga' -%>
result[0]["gcfs_config"] = flattenGcfsConfig(c.GcfsConfig)
<% end -%>
return result
return result
}

func flattenNodeConfig(c *container.NodeConfig) []map[string]interface{} {
// v == old state of `node_config`
func flattenNodeConfig(c *container.NodeConfig, v interface{}) []map[string]interface{} {
config := make([]map[string]interface{}, 0, 1)

if c == nil {
return config
}

// default to no prior taint state if there are any issues
oldTaints := []interface{}{}
oldNodeConfigSchemaContainer := v.([]interface{})
if len(oldNodeConfigSchemaContainer) != 0 {
oldNodeConfigSchema := oldNodeConfigSchemaContainer[0].(map[string]interface{})
if vt, ok := oldNodeConfigSchema["taint"]; ok && len(vt.([]interface{})) > 0 {
oldTaints = vt.([]interface{})
}
}

config = append(config, map[string]interface{}{
"machine_type": c.MachineType,
"disk_size_gb": c.DiskSizeGb,
Expand All @@ -1123,26 +1152,27 @@ func flattenNodeConfig(c *container.NodeConfig) []map[string]interface{} {
"metadata": c.Metadata,
"image_type": c.ImageType,
"labels": c.Labels,
"resource_labels": c.ResourceLabels,
"resource_labels": c.ResourceLabels,
"tags": c.Tags,
"preemptible": c.Preemptible,
"spot": c.Spot,
"min_cpu_platform": c.MinCpuPlatform,
"shielded_instance_config": flattenShieldedInstanceConfig(c.ShieldedInstanceConfig),
"taint": flattenTaints(c.Taints),
"taint": flattenTaints(c.Taints, oldTaints),
"effective_taints": flattenEffectiveTaints(c.Taints),
"workload_metadata_config": flattenWorkloadMetadataConfig(c.WorkloadMetadataConfig),
<% unless version == 'ga' -%>
"sandbox_config": flattenSandboxConfig(c.SandboxConfig),
"sandbox_config": flattenSandboxConfig(c.SandboxConfig),
"host_maintenance_policy": flattenHostMaintenancePolicy(c.HostMaintenancePolicy),
<% end -%>
"confidential_nodes": flattenConfidentialNodes(c.ConfidentialNodes),
"boot_disk_kms_key": c.BootDiskKmsKey,
"kubelet_config": flattenKubeletConfig(c.KubeletConfig),
"linux_node_config": flattenLinuxNodeConfig(c.LinuxNodeConfig),
"node_group": c.NodeGroup,
"boot_disk_kms_key": c.BootDiskKmsKey,
"kubelet_config": flattenKubeletConfig(c.KubeletConfig),
"linux_node_config": flattenLinuxNodeConfig(c.LinuxNodeConfig),
"node_group": c.NodeGroup,
"advanced_machine_features": flattenAdvancedMachineFeaturesConfig(c.AdvancedMachineFeatures),
"sole_tenant_config": flattenSoleTenantConfig(c.SoleTenantConfig),
"fast_socket": flattenFastSocket(c.FastSocket),
"sole_tenant_config": flattenSoleTenantConfig(c.SoleTenantConfig),
"fast_socket": flattenFastSocket(c.FastSocket),
})

if len(c.OauthScopes) > 0 {
Expand Down Expand Up @@ -1273,7 +1303,31 @@ func flattenGKEReservationAffinity(c *container.ReservationAffinity) []map[strin
return result
}

func flattenTaints(c []*container.NodeTaint) []map[string]interface{} {
// flattenTaints records the set of taints already present in state.
func flattenTaints(c []*container.NodeTaint, oldTaints []interface{}) []map[string]interface{} {
taintKeys := map[string]struct{}{}
for _, raw := range oldTaints {
data := raw.(map[string]interface{})
taintKey := data["key"].(string)
taintKeys[taintKey] = struct{}{}
}

result := []map[string]interface{}{}
for _, taint := range c {
if _, ok := taintKeys[taint.Key]; ok {
result = append(result, map[string]interface{}{
"key": taint.Key,
"value": taint.Value,
"effect": taint.Effect,
})
}
}

return result
}

// flattenEffectiveTaints records the complete set of taints returned from GKE.
func flattenEffectiveTaints(c []*container.NodeTaint) []map[string]interface{} {
result := []map[string]interface{}{}
for _, taint := range c {
result = append(result, map[string]interface{}{
Expand All @@ -1282,10 +1336,10 @@ func flattenTaints(c []*container.NodeTaint) []map[string]interface{} {
"effect": taint.Effect,
})
}

return result
}


func flattenWorkloadMetadataConfig(c *container.WorkloadMetadataConfig) []map[string]interface{} {
result := []map[string]interface{}{}
if c != nil {
Expand Down Expand Up @@ -1352,74 +1406,6 @@ func containerNodePoolLabelsSuppress(k, old, new string, d *schema.ResourceData)

return true
}

func containerNodePoolTaintSuppress(k, old, new string, d *schema.ResourceData) bool {
// Node configs are embedded into multiple resources (container cluster and
// container node pool) so we determine the node config key dynamically.
idx := strings.Index(k, ".taint.")
if idx < 0 {
return false
}

root := k[:idx]

// Right now, GKE only applies its own out-of-band labels when you enable
// Sandbox. We only need to perform diff suppression in this case;
// otherwise, the default Terraform behavior is fine.
o, n := d.GetChange(root + ".sandbox_config.0.sandbox_type")
if o == nil || n == nil {
return false
}

// Pull the entire changeset as a list rather than trying to deal with each
// element individually.
o, n = d.GetChange(root + ".taint")
if o == nil || n == nil {
return false
}

type taintType struct {
Key, Value, Effect string
}

taintSet := make(map[taintType]struct{})

// Add all new taints to set.
for _, raw := range n.([]interface{}) {
data := raw.(map[string]interface{})
taint := taintType{
Key: data["key"].(string),
Value: data["value"].(string),
Effect: data["effect"].(string),
}
taintSet[taint] = struct{}{}
}

// Remove all current taints, skipping GKE-managed keys if not present in
// the new configuration.
for _, raw := range o.([]interface{}) {
data := raw.(map[string]interface{})
taint := taintType{
Key: data["key"].(string),
Value: data["value"].(string),
Effect: data["effect"].(string),
}
if _, ok := taintSet[taint]; ok {
delete(taintSet, taint)
} else if !strings.HasPrefix(taint.Key, "sandbox.gke.io/") && taint.Key != "kubernetes.io/arch" {
// User-provided taint removed in new configuration.
return false
}
}

// If, at this point, the set still has elements, the new configuration
// added an additional taint.
if len(taintSet) > 0 {
return false
}

return true
}
<% end -%>

func flattenKubeletConfig(c *container.NodeKubeletConfig) []map[string]interface{} {
Expand Down Expand Up @@ -1494,4 +1480,4 @@ func flattenHostMaintenancePolicy(c *container.HostMaintenancePolicy) []map[stri

return result
}
<% end -%>
<% end -%>
2 changes: 1 addition & 1 deletion mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2682,7 +2682,7 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro
return fmt.Errorf("Error setting default_max_pods_per_node: %s", err)
}
}
if err := d.Set("node_config", flattenNodeConfig(cluster.NodeConfig)); err != nil {
if err := d.Set("node_config", flattenNodeConfig(cluster.NodeConfig, d.Get("node_config"))); err != nil {
return err
}
if err := d.Set("project", project); err != nil {
Expand Down
16 changes: 9 additions & 7 deletions mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1235,17 +1235,19 @@ func TestAccContainerCluster_withNodeConfig(t *testing.T) {
Config: testAccContainerCluster_withNodeConfig(clusterName),
},
{
ResourceName: "google_container_cluster.with_node_config",
ImportState: true,
ImportStateVerify: true,
ResourceName: "google_container_cluster.with_node_config",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"node_config.0.taint"},
},
{
Config: testAccContainerCluster_withNodeConfigUpdate(clusterName),
},
{
ResourceName: "google_container_cluster.with_node_config",
ImportState: true,
ImportStateVerify: true,
ResourceName: "google_container_cluster.with_node_config",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"node_config.0.taint"},
},
},
})
Expand Down Expand Up @@ -1541,7 +1543,7 @@ func TestAccContainerCluster_withSandboxConfig(t *testing.T) {
ResourceName: "google_container_cluster.with_sandbox_config",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"min_master_version"},
ImportStateVerifyIgnore: []string{"min_master_version", "node_config.0.taint"},
},
{
// GKE sets automatic labels and taints on nodes. This makes
Expand Down
2 changes: 1 addition & 1 deletion mmv1/third_party/terraform/services/container/resource_container_node_pool.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1112,7 +1112,7 @@ func flattenNodePool(d *schema.ResourceData, config *transport_tpg.Config, np *c
"initial_node_count": np.InitialNodeCount,
"node_locations": schema.NewSet(schema.HashString, tpgresource.ConvertStringArrToInterface(np.Locations)),
"node_count": nodeCount,
"node_config": flattenNodeConfig(np.Config),
"node_config": flattenNodeConfig(np.Config, d.Get(prefix + "node_config")),
"instance_group_urls": igmUrls,
"managed_instance_group_urls": managedIgmUrls,
"version": np.Version,
Expand Down
Loading

0 comments on commit c896b46

Please sign in to comment.