From d5f3b05dbb047bf649fc31faae121da6e90f0c14 Mon Sep 17 00:00:00 2001 From: Anjan Nath Date: Fri, 27 Sep 2024 13:59:31 +0530 Subject: [PATCH 1/4] remove unused UbuntuRequest struct this got mistakenly added back during rebase of PR #284 in 42dcadfb8d8eef798bcc4519be816ba72f9bad2d --- pkg/provider/azure/action/linux/linux.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pkg/provider/azure/action/linux/linux.go b/pkg/provider/azure/action/linux/linux.go index 2a8d29c3..87ae4650 100644 --- a/pkg/provider/azure/action/linux/linux.go +++ b/pkg/provider/azure/action/linux/linux.go @@ -46,16 +46,6 @@ type LinuxRequest struct { SpotTolerance spotAzure.EvictionRate } -type UbuntuRequest struct { - Prefix string - Location string - VMSize string - Version string - Username string - Spot bool - SpotTolerance spotAzure.EvictionRate -} - func Create(r *LinuxRequest) (err error) { if len(r.VMSizes) == 0 { vmSizes, err := r.InstanceRequest.GetMachineTypes() From ccb36f2708b8f5dae93d15adc566dc87e9a8b5af Mon Sep 17 00:00:00 2001 From: Anjan Nath Date: Fri, 27 Sep 2024 14:01:53 +0530 Subject: [PATCH 2/4] add azure fedora support this adds new command 'mapt azure fedora' that can create fedora instances on azure, supports arm64, x86_64 arches and versions 40, Rawhide and 41-Prerelease --- cmd/mapt/cmd/azure/azure.go | 1 + cmd/mapt/cmd/azure/hosts/linux.go | 11 ++++++++-- pkg/provider/azure/action/linux/imageref.go | 16 ++++++++++++++ pkg/provider/azure/action/linux/linux.go | 4 +++- .../module/virtual-machine/virtual-machine.go | 21 +++++++++++++------ 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/cmd/mapt/cmd/azure/azure.go b/cmd/mapt/cmd/azure/azure.go index 060ade07..fe964a14 100644 --- a/cmd/mapt/cmd/azure/azure.go +++ b/cmd/mapt/cmd/azure/azure.go @@ -28,6 +28,7 @@ func GetCmd() *cobra.Command { hosts.GetWindowsDesktopCmd(), hosts.GetUbuntuCmd(), hosts.GetRHELCmd(), + hosts.GetFedoraCmd(), services.GetAKSCmd()) return c } diff --git a/cmd/mapt/cmd/azure/hosts/linux.go b/cmd/mapt/cmd/azure/hosts/linux.go index 39430842..21f13c8e 100644 --- a/cmd/mapt/cmd/azure/hosts/linux.go +++ b/cmd/mapt/cmd/azure/hosts/linux.go @@ -21,13 +21,16 @@ const ( cmdUbuntu = "ubuntu" cmdUbuntuDesc = "ubuntu operations" cmdRHEL = "rhel" - cmdRHELDesc = "ubuntu operations" + cmdRHELDesc = "RHEL operations" + cmdFedora = "fedora" + cmdFedoraDesc = "fedora operations" defaultVMSize = "Standard_D8as_v5" paramLinuxVersion = "version" - paramLinuxVersionDesc = "linux version. Version should be formmated as X.Y (Major.minor)" + paramLinuxVersionDesc = "linux version. Version should be formated as X.Y (Major.minor)" defaultUbuntuVersion = "24.04" defaultRHELVersion = "9.4" + defaultFedoraVersion = "40.0" ) func GetUbuntuCmd() *cobra.Command { @@ -38,6 +41,10 @@ func GetRHELCmd() *cobra.Command { return getLinuxCmd(cmdRHEL, cmdRHELDesc, azureLinux.RHEL, defaultRHELVersion) } +func GetFedoraCmd() *cobra.Command { + return getLinuxCmd(cmdFedora, cmdFedoraDesc, azureLinux.Fedora, defaultFedoraVersion) +} + func getLinuxCmd(cmd, cmdDesc string, ostype azureLinux.OSType, defaultOSVersion string) *cobra.Command { c := &cobra.Command{ Use: cmd, diff --git a/pkg/provider/azure/action/linux/imageref.go b/pkg/provider/azure/action/linux/imageref.go index 1aca7355..ee7bd116 100644 --- a/pkg/provider/azure/action/linux/imageref.go +++ b/pkg/provider/azure/action/linux/imageref.go @@ -13,10 +13,14 @@ const ( Fedora ) +const fedoraImageGalleryBase = "/CommunityGalleries/Fedora-5e266ba4-2250-406d-adad-5d73860d958f/Images/" + type imageReference struct { publisher string offer string sku string + // community gallery image ID + id string } var ( @@ -40,6 +44,14 @@ var ( sku: "server", }, }, + Fedora: { + "x86_64": { + id: fedoraImageGalleryBase + "Fedora-Cloud-%s-x64/Versions/latest", + }, + "arm64": { + id: fedoraImageGalleryBase + "Fedora-Cloud-%s-Arm64/Versions/latest", + }, + }, } ) @@ -60,6 +72,10 @@ func getImageRef(osTarget OSType, arch string, version string) (*imageReference, offer: ir.offer, sku: fmt.Sprintf(ir.sku, versions[0], versions[1]), }, nil + case Fedora: + return &imageReference{ + id: fmt.Sprintf(ir.id, versions[0]), + }, nil } return nil, fmt.Errorf("os type not supported") diff --git a/pkg/provider/azure/action/linux/linux.go b/pkg/provider/azure/action/linux/linux.go index 87ae4650..ce428ebc 100644 --- a/pkg/provider/azure/action/linux/linux.go +++ b/pkg/provider/azure/action/linux/linux.go @@ -16,8 +16,8 @@ import ( "github.com/redhat-developer/mapt/pkg/provider/util/command" "github.com/redhat-developer/mapt/pkg/provider/util/instancetypes" "github.com/redhat-developer/mapt/pkg/provider/util/output" - "github.com/redhat-developer/mapt/pkg/util" spotAzure "github.com/redhat-developer/mapt/pkg/spot/azure" + "github.com/redhat-developer/mapt/pkg/util" "github.com/redhat-developer/mapt/pkg/util/logging" resourcesUtil "github.com/redhat-developer/mapt/pkg/util/resources" ) @@ -120,6 +120,7 @@ func (r *LinuxRequest) deployer(ctx *pulumi.Context) error { if err != nil { return err } + vmr := virtualmachine.VirtualMachineRequest{ Prefix: r.Prefix, ComponentID: azureLinuxID, @@ -129,6 +130,7 @@ func (r *LinuxRequest) deployer(ctx *pulumi.Context) error { Publisher: ir.publisher, Offer: ir.offer, Sku: ir.sku, + ImageID: ir.id, AdminUsername: r.Username, PrivateKey: privateKey, SpotPrice: spotPrice, diff --git a/pkg/provider/azure/module/virtual-machine/virtual-machine.go b/pkg/provider/azure/module/virtual-machine/virtual-machine.go index bd3c71d8..325d3986 100644 --- a/pkg/provider/azure/module/virtual-machine/virtual-machine.go +++ b/pkg/provider/azure/module/virtual-machine/virtual-machine.go @@ -28,6 +28,8 @@ type VirtualMachineRequest struct { Offer string Sku string SpotPrice *float64 + // community galary image ID + ImageID string // Windows required AdminUsername string // Linux required @@ -38,6 +40,18 @@ type VirtualMachineRequest struct { // Create virtual machine based on request + export to context // adminusername and adminuserpassword func (r *VirtualMachineRequest) Create(ctx *pulumi.Context) (*compute.VirtualMachine, error) { + var imageReferenceArgs compute.ImageReferenceArgs + if len(r.ImageID) > 0 { + imageReferenceArgs = compute.ImageReferenceArgs{ + CommunityGalleryImageId: pulumi.String(r.ImageID)} + } else { + imageReferenceArgs = compute.ImageReferenceArgs{ + Publisher: pulumi.String(r.Publisher), + Offer: pulumi.String(r.Offer), + Sku: pulumi.String(r.Sku), + Version: pulumi.String("latest"), + } + } vmArgs := &compute.VirtualMachineArgs{ VmName: pulumi.String(maptContext.RunID()), Location: r.ResourceGroup.Location, @@ -53,12 +67,7 @@ func (r *VirtualMachineRequest) Create(ctx *pulumi.Context) (*compute.VirtualMac VmSize: pulumi.String(r.VMSize), }, StorageProfile: compute.StorageProfileArgs{ - ImageReference: compute.ImageReferenceArgs{ - Publisher: pulumi.String(r.Publisher), - Offer: pulumi.String(r.Offer), - Sku: pulumi.String(r.Sku), - Version: pulumi.String("latest"), - }, + ImageReference: imageReferenceArgs, OsDisk: compute.OSDiskArgs{ Name: pulumi.String(maptContext.RunID()), DiskSizeGB: pulumi.Int(diskSize), From e50175ab3ab7ea60686f33dfed98fd2927ab09a1 Mon Sep 17 00:00:00 2001 From: Anjan Nath Date: Fri, 27 Sep 2024 14:10:03 +0530 Subject: [PATCH 3/4] docs: add doc for azure fedora operations --- docs/azure/fedora.md | 87 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 docs/azure/fedora.md diff --git a/docs/azure/fedora.md b/docs/azure/fedora.md new file mode 100644 index 00000000..16556d7b --- /dev/null +++ b/docs/azure/fedora.md @@ -0,0 +1,87 @@ +# Overview + +mapt offers several operations to manage environments within azure: + +## Fedora + +It creates / destroy a Fedora machine ready to be included within the CI/CD system. Features included within the offering: + +* Setup ssh for the user user with a self generated private key + +### Operations + +#### Create + +This will create a fedora instance accordig to params specificed: + +```bash +podman run -it --rm quay.io/redhat-developer/mapt:0.7.0-dev azure fedora create -h +create + +Usage: + mapt azure fedora create [flags] + +Flags: + --arch string architecture for the machine. Allowed x86_64 or arm64 (default "x86_64") + --conn-details-output string path to export host connection information (host, username and privateKey) + --cpus int32 Number of CPUs for the cloud instance (default 8) + -h, --help help for create + --location string If spot is passed location will be calculated based on spot results. Otherwise localtion will be used to create resources. (default "West US") + --memory int32 Amount of RAM for the cloud instance in GiB (default 64) + --nested-virt Use cloud instance that has nested virtualization support + --spot if spot is set the spot prices across all regions will be cheked and machine will be started on best spot option (price / eviction) + --spot-eviction-tolerance string if spot is enable we can define the minimum tolerance level of eviction. Allowed value are: lowest, low, medium, high or highest (default "lowest") + --tags stringToString tags to add on each resource (--tags name1=value1,name2=value2) (default []) + --username string username for general user. SSH accessible + rdp with generated password (default "rhqp") + --version string linux version. Version should be formated as X.Y (Major.minor) + --vmsize strings set specific size for the VM and ignore any CPUs, Memory and Arch parameters set. Type requires to allow nested virtualization + +Global Flags: + --backed-url string backed for stack state. Can be a local path with format file:///path/subpath or s3 s3://existing-bucket + --project-name string project name to identify the instance of the stack +``` +> *NOTE:* Fedora distro version needs to be in the form x.y, but official fedora versions contain just the major version. For fedora 40 use `--version 40.0` + +> *NOTE:* You can list all the available fedora images using `az`, please refer: https://fedoramagazine.org/launch-fedora-40-in-microsoft-azure for more information + +It will crete a fedora instance and will give as result several files located at path defined by `--conn-details-output`: + + +* username: file containing the username for worker user +* id_rsa: file containing the private key for worker user +* host: file containing the public ip for the instance + +Also it will create a state folder holding the state for the created resources at azure, the path for this folder is defined within `--backed-url`, the content from that folder it is required with the same project name (`--project-name`) in order to detroy the resources. + +When running the container image it is required to pass the authetication information as variables, following a sample snipped on how to create +a instance with default values: + +```bash +podman run -d --rm \ + -v ${PWD}:/workspace:z \ + -e ARM_TENANT_ID=${ati_value} \ + -e ARM_SUBSCRIPTION_ID=${asi_value} \ + -e ARM_CLIENT_ID=${aci_value} \ + -e ARM_CLIENT_SECRET=${acs_lue} \ + quay.io/redhat-developer/mapt:0.7.0-dev azure \ + fedora create \ + --project-name "fedora-40" \ + --backed-url "file:///workspace" \ + --conn-details-output "/workspace" \ + --spot +``` + +The following is a snippet on how to destroy the resources: + +```bash +podman run -d --rm \ + -v ${PWD}:/workspace:z \ + -e ARM_TENANT_ID=${ati_value} \ + -e ARM_SUBSCRIPTION_ID=${asi_value} \ + -e ARM_CLIENT_ID=${aci_value} \ + -e ARM_CLIENT_SECRET=${acs_lue} \ + quay.io/redhat-developer/mapt:0.7.0-dev azure \ + fedora destroy \ + --project-name "fedora-40" \ + --backed-url "file:///workspace" +``` From 6f161a86e68f6b296d11bd74d32cf9b36425f065 Mon Sep 17 00:00:00 2001 From: Anjan Nath Date: Mon, 30 Sep 2024 18:53:52 +0530 Subject: [PATCH 4/4] add tekton task for creation of fedora instance on azure --- README.md | 1 + tkn/infra-azure-fedora.yaml | 168 ++++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 tkn/infra-azure-fedora.yaml diff --git a/README.md b/README.md index 988f43a8..52e5e93a 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Also mapt offers a set of features wich are transversal to each type of target m | RHEL | x86, arm64 | AWS | Customizable | [info](docs/aws/rhel.md) | [task](tkn/infra-aws-rhel.yaml) | a,s | | RHEL | x86, arm64 | Azure | Virtualized | [info](docs/azure/rhel.md) | [task](tkn/infra-azure-rhel.yaml) | s | | Fedora | x86, arm64 | AWS | Customizable | [info](docs/aws/fedora.md) | [task](tkn/infra-aws-fedora.yaml) | a,s | +| Fedora | x86, arm64 | Azure | Customizable | [info](docs/azure/fedora.md) | [task](tkn/infra-azure-fedora.yaml) | a,s | | Ubuntu | x86 | Azure | Virtualized | [info](docs/azure/ubuntu.md) | - | s | Features: diff --git a/tkn/infra-azure-fedora.yaml b/tkn/infra-azure-fedora.yaml new file mode 100644 index 00000000..e8988ec0 --- /dev/null +++ b/tkn/infra-azure-fedora.yaml @@ -0,0 +1,168 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: infra-azure-rhel + labels: + app.kubernetes.io/version: "0.7.0-dev" + annotations: + tekton.dev/pipelines.minVersion: "0.44.x" + tekton.dev/categories: infrastructure + tekton.dev/tags: infrastructure, azure + tekton.dev/displayName: "azure manager" + tekton.dev/platforms: "linux/amd64" +spec: + description: | + This task will provision / decomission fedora on azure + + The output will give required information to connect within the remote provisioned host + + params: + - name: project-name + description: identifier for project. + - name: backed-url + description: | + If we want to backed resources externally we can use az storage setting this param(i.e azblob://existing-storage). + + If default will be store on storage workspace at path set by param ws-output-path. + default: "''" + - name: operation + description: operation to execute within the infrastructure. Current values (create, destroy) + default: create + - name: arch + description: architecture for the target machine. Allowed x86_64 or arm64 (default "x86_64") + default: 'x86_64' + - name: version + description: this task will spin a RHEL image. This param 'ill set the version for it. Default 9.4. + default: '40.0' + - name: vmsize + description: size for the machine + default: "''" + - name: cpus + description: number of cpus for the machine + default: '8' + - name: memory + description: amount of ram in GB for the machine + default: '64' + - name: nested-virt + description: nested virtualization support on the machine + default: 'true' + - name: spot + description: in case spot is set to true it 'ill check for best spot price and create the VM on the target region + default: 'true' + - name: spot-eviction-tolerance + description: 'if spot is enable we can define the minimum tolerance level of eviction. Allowed value are: lowest, low, medium, high or highest' + default: 'lowest' + - name: workspace-resources-path + description: path on workspace where to store ephemeral assets related with the provisioning + - name: remove-lock + description: in case a previous run fails the stack can be locked. This value allows to control if remove lock + default: 'true' + - name: tags + description: tags for the resources created on the providers + default: "''" + + results: + - name: host + description: ip to connect to the provisioned machine + - name: username + description: username to connect to the provisioned machine + - name: key + description: filename for the private key. The key is located at workspace-resources-path + + steps: + - name: provisioner + image: quay.io/redhat-developer/mapt:v0.7.0-dev + imagePullPolicy: Always + script: | + #!/bin/sh + + # Added verbosity + set -xuo + + # Credentials + export ARM_TENANT_ID=$(cat /opt/credentials/tenant_id) + export ARM_SUBSCRIPTION_ID=$(cat /opt/credentials/subscription_id) + export ARM_CLIENT_ID=$(cat /opt/credentials/client_id) + export ARM_CLIENT_SECRET=$(cat /opt/credentials/client_secret) + if ! [ -f /opt/credentials/storage_account ]; then + export AZURE_STORAGE_ACCOUNT=$(cat /opt/credentials/storage_account) + fi + if ! [ -f /opt/credentials/storage_key ]; then + export AZURE_STORAGE_KEY=$(cat /opt/credentials/storage_key) + fi + + # Output folder + workspace_path=$(workspaces.pipelines-data.path)/$(params.workspace-resources-path) + mkdir -p ${workspace_path} + + # Remove lock + if [[ $(params.remove-lock) == "true" ]]; then + rm -rf ${workspace_path}/.pulumi/locks/* + fi + + # Run mapt + cmd="mapt azure fedora $(params.operation) " + cmd="$cmd --project-name $(params.project-name) " + if [[ $(params.backed-url) != "" ]]; then + cmd="$cmd --backed-url $(params.backed-url) " + else + cmd="$cmd --backed-url file://${workspace_path} " + fi + if [[ $(params.operation) == "create" ]]; then + cmd="$cmd --conn-details-output ${workspace_path} " + cmd="$cmd --arch $(params.arch) " + cmd="$cmd --cpus $(params.cpus) " + cmd="$cmd --memory $(params.memory) " + if [[ $(params.nested-virt) == "true" ]]; then + cmd="$cmd --nested-virt " + fi + cmd="$cmd --version $(params.version) " + if [[ $(params.vmsize) != "" ]]; then + cmd="$cmd --vmsize $(params.vmsize) " + fi + if [[ $(params.spot) == "true" ]]; then + cmd="$cmd --spot " + cmd="$cmd --spot-eviction-tolerance $(params.spot-eviction-tolerance) " + fi + if [[ $(params.tags) != "" ]]; then + cmd="$cmd --tags $(params.tags) " + fi + fi + eval "${cmd}" + + create_exit_code=$? + + # set task results + cat "${workspace_path}/host" | tee $(results.host.path) + cat "${workspace_path}/username" | tee $(results.username.path) + echo -n "id_rsa" | tee $(results.key.path) + + if [[ ${create_exit_code} -ne 0 ]]; then + exit 1 + fi + + resources: + requests: + memory: "200Mi" + cpu: "100m" + limits: + memory: "600Mi" + cpu: "300m" + + workspaces: + - name: pipelines-data + description: workspace to store outputs to connect within the target machine + state file for the infrastructure + - name: az-credentials + description: | + ocp secret holding the azure credentials. Secret should be accessible to this task. + + To be a valid secret it should contains the following fields: + * tenant_id + * subscription_id + * client_id + * client_secret + * storage_account (optional if we use remote az storage) + * storage_key (optional if we use remote az storage) + mountPath: /opt/credentials +