From 03d5997be9810a6c8503159b09a64153b77010bc Mon Sep 17 00:00:00 2001 From: Alex Tymchuk Date: Thu, 29 Jun 2023 18:40:59 +0300 Subject: [PATCH] PMM-12266 fix parsing yum date format el9 (2.38.0) (#2327) * PMM-12266 fix parsing yum date format for EL9 * PMM-12266 format code * PMM-12266 fix a redundant fn variable --- update/docker-compose.yml | 3 +- update/pkg/yum/info.go | 7 +- update/pkg/yum/info_test.go | 345 +++++++++++++++++++++++++++++++++++- update/pkg/yum/yum.go | 2 +- 4 files changed, 347 insertions(+), 10 deletions(-) diff --git a/update/docker-compose.yml b/update/docker-compose.yml index 30e80f28b8..5be6dd1cbf 100644 --- a/update/docker-compose.yml +++ b/update/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: '3' services: pmm-update-server: image: ${PMM_SERVER_IMAGE:-percona/pmm-server:2} @@ -7,7 +6,7 @@ services: environment: # for tests - PMM_SERVER_IMAGE=${PMM_SERVER_IMAGE:-percona/pmm-server:2} - - GO_VERSION=${GO_VERSION:-1.17.x} + - GO_VERSION=${GO_VERSION:-1.20.x} - PATH=/root/go/bin:$PATH - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN} volumes: diff --git a/update/pkg/yum/info.go b/update/pkg/yum/info.go index cfe2977478..f705d2922e 100644 --- a/update/pkg/yum/info.go +++ b/update/pkg/yum/info.go @@ -68,7 +68,12 @@ func parseInfo(lines []string, firstKey string) (map[string]string, error) { } func parseInfoTime(s string) (time.Time, error) { - return time.Parse("Mon Jan 2 15:04:05 2006", s) + layout := "Mon 2 Jan 2006 15:04:05 PM UTC" // layout for EL9, default + v, err := getRHELVersion() + if err == nil && v == "7" { + layout = "Mon Jan 2 15:04:05 2006" // change the layout for EL7 + } + return time.Parse(layout, s) } // fullVersion returns full (ugly) package version. diff --git a/update/pkg/yum/info_test.go b/update/pkg/yum/info_test.go index a402d15692..d8cc827c80 100644 --- a/update/pkg/yum/info_test.go +++ b/update/pkg/yum/info_test.go @@ -24,7 +24,11 @@ import ( "github.com/stretchr/testify/require" ) -func TestParseInfo(t *testing.T) { +func TestParseInfoEL7(t *testing.T) { + v, _ := getRHELVersion() + if v == "9" { + t.Skip("Skip running EL7 tests on EL9") + } t.Run("Installed", func(t *testing.T) { stdout := strings.Split(` Loading "fastestmirror" plugin @@ -147,7 +151,7 @@ func TestParseInfo(t *testing.T) { }) t.Run("Available", func(t *testing.T) { - // yum --verbose --showduplicates info available pmm-update, abbrivated + // yum --verbose --showduplicates info available pmm-update, abbreviated stdout := strings.Split(` Loading "fastestmirror" plugin Loading "ovl" plugin @@ -219,7 +223,7 @@ func TestParseInfo(t *testing.T) { }) t.Run("AvailableGA", func(t *testing.T) { - // yum --verbose --showduplicates info available pmm-update, abbrivated + // yum --verbose --showduplicates info available pmm-update, abbreviated stdout := strings.Split(` Available Packages Name : pmm-update @@ -338,12 +342,341 @@ func TestParseInfo(t *testing.T) { }) } +func TestParseInfoEL9(t *testing.T) { + v, _ := getRHELVersion() + if v == "7" { + t.Skip("Skip running EL9 tests on EL7") + } + t.Run("Installed EL9", func(t *testing.T) { + stdout := strings.Split(` + Starting "yum --verbose info installed pmm-managed" ... + Loaded plugins: builddep, changelog, config-manager, copr, debug, debuginfo-install, download, generate_completion_cache, groups-manager, needs-restarting, playground, repoclosure, repodiff, repograph, repomanage, reposync, system-upgrade + YUM version: 4.14.0 + cachedir: /var/cache/dnf + Unknown configuration option: async = 1 in /etc/yum.repos.d/clickhouse.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/local.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/nginx.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/percona-ppg-11.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/percona-ppg-14.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/pmm2-server.repo + User-Agent: constructed: 'libdnf (Oracle Linux Server 9.2; server; Linux.x86_64)' + Installed Packages + Name : pmm-managed + Version : 2.39.0 + Release : 20.2306271313.b6d58b6.el9 + Architecture : x86_64 + Size : 125 M + Source : pmm-managed-2.39.0-20.2306271313.b6d58b6.el9.src.rpm + Repository : @System + From repo : local + Packager : None + Buildtime : Tue 27 Jun 2023 01:13:03 PM UTC + Install time : Tue 27 Jun 2023 01:31:05 PM UTC + Installed by : System + Summary : Percona Monitoring and Management management daemon + URL : https://github.com/percona/pmm + License : AGPLv3 + Description : pmm-managed manages configuration of PMM server components (VictoriaMetrics, + : Grafana, etc.) and exposes API for that. Those APIs are used by pmm-admin tool. + : See PMM docs for more information. + `, "\n") + expected := map[string]string{ + "Name": "pmm-managed", + "Version": "2.39.0", + "Release": "20.2306271313.b6d58b6.el9", + "Architecture": "x86_64", + "Size": "125 M", + "Source": "pmm-managed-2.39.0-20.2306271313.b6d58b6.el9.src.rpm", + "Repository": "@System", + "From repo": "local", + "Packager": "None", + "Buildtime": "Tue 27 Jun 2023 01:13:03 PM UTC", + "Install time": "Tue 27 Jun 2023 01:31:05 PM UTC", + "Installed by": "System ", + "Summary": "Percona Monitoring and Management management daemon", + "URL": "https://github.com/percona/pmm", + "License": "AGPLv3", + "Description": "pmm-managed manages configuration of PMM server components (VictoriaMetrics, " + + "Grafana, etc.) and exposes API for that. Those APIs are used by pmm-admin tool. " + + "See PMM docs for more information.", + } + actual, err := parseInfo(stdout, "Name") + require.NoError(t, err) + assert.Equal(t, expected, actual) + buildtime, err := parseInfoTime(actual["Buildtime"]) + require.NoError(t, err) + assert.Equal(t, time.Date(2023, 6, 27, 13, 13, 3, 0, time.UTC), buildtime) + assert.Equal(t, "2.39.0-20.2306271313.b6d58b6.el9", fullVersion(actual)) + assert.Equal(t, "2.39.0", niceVersion(actual)) + }) + + t.Run("Updates EL9", func(t *testing.T) { + // yum --verbose info updates pmm-update + stdout := strings.Split(` + Loaded plugins: builddep, changelog, config-manager, copr, debug, debuginfo-install, download, generate_completion_cache, groups-manager, needs-restarting, playground, repoclosure, repodiff, repograph, repomanage, reposync, system-upgrade + YUM version: 4.14.0 + cachedir: /var/cache/dnf + Unknown configuration option: async = 1 in /etc/yum.repos.d/clickhouse.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/local.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/nginx.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/percona-ppg-11.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/percona-ppg-14.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/pmm2-server.repo + User-Agent: constructed: 'libdnf (Oracle Linux Server 9.2; server; Linux.x86_64)' + repo: using cache for: ol9_developer_EPEL + ol9_developer_EPEL: using metadata from Wed 28 Jun 2023 02:28:50 PM UTC. + repo: using cache for: ol9_baseos_latest + ol9_baseos_latest: using metadata from Fri 23 Jun 2023 04:59:24 AM UTC. + repo: using cache for: ol9_appstream + ol9_appstream: using metadata from Fri 23 Jun 2023 05:03:17 AM UTC. + repo: using cache for: percona-release-x86_64 + percona-release-x86_64: using metadata from Mon 26 Jun 2023 01:02:27 PM UTC. + repo: using cache for: percona-release-noarch + percona-release-noarch: using metadata from Wed 06 Jul 2022 08:25:44 PM UTC. + repo: using cache for: percona-testing-x86_64 + percona-testing-x86_64: using metadata from Wed 28 Jun 2023 05:27:06 PM UTC. + repo: using cache for: percona-testing-noarch + percona-testing-noarch: using metadata from Wed 06 Jul 2022 08:20:55 PM UTC. + repo: using cache for: percona-ppg-11 + percona-ppg-11: using metadata from Mon 22 May 2023 08:40:15 AM UTC. + repo: using cache for: percona-ppg-14 + percona-ppg-14: using metadata from Wed 28 Jun 2023 02:57:51 PM UTC. + repo: using cache for: prel-release-noarch + prel-release-noarch: using metadata from Thu 16 Sep 2021 06:35:55 AM UTC. + repo: using cache for: pmm2-server + pmm2-server: using metadata from Wed 28 Jun 2023 02:46:09 PM UTC. + Last metadata expiration check: 1:42:51 ago on Wed 28 Jun 2023 07:06:43 PM UTC. + Available Upgrades + Name : pmm-update + Version : 2.39.0 + Release : 67.2306281336.d0d7fcb.el9 + Architecture : noarch + Size : 886 k + Source : pmm-update-2.39.0-67.2306281336.d0d7fcb.el9.src.rpm + Repository : pmm2-server + Packager : None + Buildtime : Wed 28 Jun 2023 01:36:03 PM UTC + Summary : Tool for updating packages and OS configuration for PMM Server + URL : https://github.com/percona/pmm + License : AGPLv3 + Description : Tool for updating packages and OS configuration for PMM Server + `, "\n") + expected := map[string]string{ + "Name": "pmm-update", + "Architecture": "noarch", + "Version": "2.39.0", + "Release": "67.2306281336.d0d7fcb.el9", + "Size": "886 k", + "Source": "pmm-update-2.39.0-67.2306281336.d0d7fcb.el9.src.rpm", + "Repository": "pmm2-server", + "Packager": "None", + "Buildtime": "Wed 28 Jun 2023 01:36:03 PM UTC", + "Summary": "Tool for updating packages and OS configuration for PMM Server", + "URL": "https://github.com/percona/pmm", + "License": "AGPLv3", + "Description": "Tool for updating packages and OS configuration for PMM Server", + } + actual, err := parseInfo(stdout, "Name") + require.NoError(t, err) + assert.Equal(t, expected, actual) + buildtime, err := parseInfoTime(actual["Buildtime"]) + require.NoError(t, err) + assert.Equal(t, time.Date(2023, 6, 28, 13, 36, 3, 0, time.UTC), buildtime) + assert.Equal(t, "2.39.0-67.2306281336.d0d7fcb.el9", fullVersion(actual)) + assert.Equal(t, "2.39.0", niceVersion(actual)) + }) + + t.Run("AvailableGA EL9", func(t *testing.T) { + // yum --verbose --showduplicates info available pmm-update (just two versions) + stdout := strings.Split(` + Loaded plugins: builddep, changelog, config-manager, copr, debug, debuginfo-install, download, generate_completion_cache, groups-manager, needs-restarting, playground, repoclosure, repodiff, repograph, repomanage, reposync, system-upgrade + YUM version: 4.14.0 + cachedir: /var/cache/dnf + Unknown configuration option: async = 1 in /etc/yum.repos.d/clickhouse.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/local.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/nginx.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/percona-ppg-11.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/percona-ppg-14.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/pmm2-server.repo + User-Agent: constructed: 'libdnf (Oracle Linux Server 9.2; server; Linux.x86_64)' + repo: using cache for: ol9_developer_EPEL + ol9_developer_EPEL: using metadata from Wed 28 Jun 2023 02:28:50 PM UTC. + repo: using cache for: ol9_baseos_latest + ol9_baseos_latest: using metadata from Fri 23 Jun 2023 04:59:24 AM UTC. + repo: using cache for: ol9_appstream + ol9_appstream: using metadata from Fri 23 Jun 2023 05:03:17 AM UTC. + repo: using cache for: percona-release-x86_64 + percona-release-x86_64: using metadata from Mon 26 Jun 2023 01:02:27 PM UTC. + repo: using cache for: percona-release-noarch + percona-release-noarch: using metadata from Wed 06 Jul 2022 08:25:44 PM UTC. + repo: using cache for: percona-testing-x86_64 + percona-testing-x86_64: using metadata from Wed 28 Jun 2023 05:27:06 PM UTC. + repo: using cache for: percona-testing-noarch + percona-testing-noarch: using metadata from Wed 06 Jul 2022 08:20:55 PM UTC. + repo: using cache for: percona-ppg-11 + percona-ppg-11: using metadata from Mon 22 May 2023 08:40:15 AM UTC. + repo: using cache for: percona-ppg-14 + percona-ppg-14: using metadata from Wed 28 Jun 2023 02:57:51 PM UTC. + repo: using cache for: prel-release-noarch + prel-release-noarch: using metadata from Thu 16 Sep 2021 06:35:55 AM UTC. + repo: using cache for: pmm2-server + pmm2-server: using metadata from Wed 28 Jun 2023 02:46:09 PM UTC. + Last metadata expiration check: 1:18:00 ago on Wed 28 Jun 2023 07:06:43 PM UTC. + Available Packages + Name : pmm-update + Version : 2.39.0 + Release : 67.2306280932.70f3748.el9 + Architecture : noarch + Size : 887 k + Source : pmm-update-2.39.0-67.2306280932.70f3748.el9.src.rpm + Repository : pmm2-server + Packager : None + Buildtime : Wed 28 Jun 2023 09:32:21 AM UTC + Summary : Tool for updating packages and OS configuration for PMM Server + URL : https://github.com/percona/pmm + License : AGPLv3 + Description : Tool for updating packages and OS configuration for PMM Server + + Name : pmm-update + Version : 2.39.0 + Release : 67.2306281012.fe8e947.el9 + Architecture : noarch + Size : 887 k + Source : pmm-update-2.39.0-67.2306281012.fe8e947.el9.src.rpm + Repository : pmm2-server + Packager : None + Buildtime : Wed 28 Jun 2023 10:12:02 AM UTC + Summary : Tool for updating packages and OS configuration for PMM Server + URL : https://github.com/percona/pmm + License : AGPLv3 + Description : Tool for updating packages and OS configuration for PMM Server + `, "\n") + expected := map[string]string{ + "Name": "pmm-update", + "Architecture": "noarch", + "Version": "2.39.0", + "Release": "67.2306280932.70f3748.el9", + "Size": "887 k", + "Source": "pmm-update-2.39.0-67.2306280932.70f3748.el9.src.rpm", + "Repository": "pmm2-server", + "Packager": "None", + "Buildtime": "Wed 28 Jun 2023 09:32:21 AM UTC", + "Summary": "Tool for updating packages and OS configuration for PMM Server", + "URL": "https://github.com/percona/pmm", + "License": "AGPLv3", + "Description": "Tool for updating packages and OS configuration for PMM Server", + } + actual, err := parseInfo(stdout, "Name") + assert.EqualError(t, err, "second `Name` encountered") + assert.Equal(t, expected, actual) + buildtime, err := parseInfoTime(actual["Buildtime"]) + require.NoError(t, err) + assert.Equal(t, time.Date(2023, 6, 28, 9, 32, 21, 0, time.UTC), buildtime) + assert.Equal(t, "2.39.0-67.2306280932.70f3748.el9", fullVersion(actual)) + assert.Equal(t, "2.39.0", niceVersion(actual)) + }) + + t.Run("Empty EL9", func(t *testing.T) { + // yum --verbose info updates pmm-managed + // "Error: No matching Packages to list" goes to stderr. + // The output below is generated when there are no updates available. + stdout := strings.Split(` + Loaded plugins: builddep, changelog, config-manager, copr, debug, debuginfo-install, download, generate_completion_cache, groups-manager, needs-restarting, playground, repoclosure, repodiff, repograph, repomanage, reposync, system-upgrade + YUM version: 4.14.0 + cachedir: /var/cache/dnf + Unknown configuration option: async = 1 in /etc/yum.repos.d/clickhouse.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/local.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/nginx.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/percona-ppg-11.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/percona-ppg-14.repo + Unknown configuration option: async = 1 in /etc/yum.repos.d/pmm2-server.repo + User-Agent: constructed: 'libdnf (Oracle Linux Server 9.2; server; Linux.x86_64)' + repo: using cache for: ol9_developer_EPEL + ol9_developer_EPEL: using metadata from Wed 28 Jun 2023 02:28:50 PM UTC. + repo: using cache for: ol9_baseos_latest + ol9_baseos_latest: using metadata from Fri 23 Jun 2023 04:59:24 AM UTC. + repo: using cache for: ol9_appstream + ol9_appstream: using metadata from Fri 23 Jun 2023 05:03:17 AM UTC. + repo: using cache for: percona-release-x86_64 + percona-release-x86_64: using metadata from Mon 26 Jun 2023 01:02:27 PM UTC. + repo: using cache for: percona-release-noarch + percona-release-noarch: using metadata from Wed 06 Jul 2022 08:25:44 PM UTC. + repo: using cache for: percona-testing-x86_64 + percona-testing-x86_64: using metadata from Wed 28 Jun 2023 05:27:06 PM UTC. + repo: using cache for: percona-testing-noarch + percona-testing-noarch: using metadata from Wed 06 Jul 2022 08:20:55 PM UTC. + repo: using cache for: percona-ppg-11 + percona-ppg-11: using metadata from Mon 22 May 2023 08:40:15 AM UTC. + repo: using cache for: percona-ppg-14 + percona-ppg-14: using metadata from Wed 28 Jun 2023 02:57:51 PM UTC. + repo: using cache for: prel-release-noarch + prel-release-noarch: using metadata from Thu 16 Sep 2021 06:35:55 AM UTC. + repo: using cache for: pmm2-server + pmm2-server: using metadata from Wed 28 Jun 2023 02:46:09 PM UTC. + Last metadata expiration check: 0:59:54 ago on Wed 28 Jun 2023 07:06:43 PM UTC. + `, "\n") + actual, err := parseInfo(stdout, "Name") + require.NoError(t, err) + assert.Empty(t, actual) + }) + + t.Run("RepoInfo EL9", func(t *testing.T) { + // yum repoinfo pmm2-server. + stdout := strings.Split(` + Last metadata expiration check: 9:26:06 ago on Wed 28 Jun 2023 09:26:18 AM UTC. + Repo-id : pmm2-server + Repo-name : PMM Server YUM repository - x86_64 + Repo-status : enabled + Repo-revision : 1687873070 + Repo-updated : Tue 27 Jun 2023 01:25:23 PM UTC + Repo-pkgs : 478 + Repo-available-pkgs: 478 + Repo-size : 3.7 G + Repo-baseurl : https://repo.percona.com/pmm2-components/yum/experimental/9/RPMS/x86_64/ + Repo-expire : 172,800 second(s) (last: Wed 28 Jun 2023 09:26:18 AM UTC) + Repo-filename : /etc/yum.repos.d/pmm2-server.repo + Total packages: 478 + `, "\n") + expected := map[string]string{ + "Repo-id": "pmm2-server", + "Repo-name": "PMM Server YUM repository - x86_64", + "Repo-status": "enabled", + "Repo-revision": "1687873070", + "Repo-updated": "Tue 27 Jun 2023 01:25:23 PM UTC", + "Repo-pkgs": "478", + "Repo-available-pkgs": "478", + "Repo-size": "3.7 G", + "Repo-baseurl": "https://repo.percona.com/pmm2-components/yum/experimental/9/RPMS/x86_64/", + "Repo-expire": "172,800 second(s) (last: Wed 28 Jun 2023 09:26:18 AM UTC)", + "Repo-filename": "/etc/yum.repos.d/pmm2-server.repo", + "Total packages": "478", + } + actual, err := parseInfo(stdout, "Repo-id") + require.NoError(t, err) + assert.Equal(t, expected, actual) + releasetime, err := parseInfoTime(actual["Repo-updated"]) + require.NoError(t, err) + assert.Equal(t, time.Date(2023, time.June, 27, 13, 25, 23, 0, time.UTC), releasetime) + }) +} + func TestGetRHELVersion(t *testing.T) { - t.Run("getRHELVersion", func(t *testing.T) { - // TODO enable once we pass to RHEL 9 for good - t.Skip("This test will fail on RHEL 7 and non-RHEL systems") + t.Run("getRHELVersion EL9", func(t *testing.T) { + actual, err := getRHELVersion() + if actual == "7" { + t.Skip("Skip running EL9 tests on EL7") + } expected := "9" + require.NoError(t, err) + assert.Equal(t, expected, actual) + }) + + t.Run("getRHELVersion EL7", func(t *testing.T) { actual, err := getRHELVersion() + if actual == "9" { + t.Skip("Skip running EL7 test on EL9") + } + expected := "7" require.NoError(t, err) assert.Equal(t, expected, actual) }) diff --git a/update/pkg/yum/yum.go b/update/pkg/yum/yum.go index 100a287aa6..f9b906f86d 100644 --- a/update/pkg/yum/yum.go +++ b/update/pkg/yum/yum.go @@ -113,7 +113,7 @@ func Check(ctx context.Context, name string) (*version.UpdateCheckResult, error) v, err := getRHELVersion() if err == nil && v == "7" { - // change the prop name for RHEL7, otherwise leave as is + // change the prop name for EL7 repoPropName = "Repo" } repo, ok := info[repoPropName]