From dbb2d87e05ca3729ca647791025e4d745b02c0e7 Mon Sep 17 00:00:00 2001 From: Andy Baugh Date: Mon, 25 Nov 2024 08:58:26 -0700 Subject: [PATCH] Remove blocker for JetBackup on Ubuntu Fix t/usage.t when noc_reccomendations file exists as well Fix bizzare errors in t/ test for ElevateScript that failed due to local configuration when running the testsuite during development. I did think about maybe moving the logic for enabling the repo for the jetbackup tier, but at the same time I didn't feel I had the time before Thanksgiving break to genericize that properly, as the manual page for sources.list tells me I'd need a more complex regexp, etc. for detecting what a disabled repo even should look like in the "generic" case. Since I wasn't too hot on running readdir and then figuring out what "looks like it isn't a list/sources file matching my criterion" I punted on that. --- elevate-cpanel | 67 ++++++++++++++++++++++------- lib/Elevate/Components/JetBackup.pm | 36 ++++++++-------- lib/Elevate/OS/Ubuntu20.pm | 3 ++ lib/Elevate/PkgMgr/APT.pm | 65 +++++++++++++++++++++++++++- t/components-ElevateScript.t | 34 +++++++++------ t/components-JetBackup.t | 13 +----- t/usage.t | 9 ++-- 7 files changed, 164 insertions(+), 63 deletions(-) diff --git a/elevate-cpanel b/elevate-cpanel index b0d8d90b..a8490c97 100755 --- a/elevate-cpanel +++ b/elevate-cpanel @@ -3074,9 +3074,22 @@ EOS : $repos->{'jetapps-beta'} ? 'jetapps-beta' : 'jetapps-stable'; # Just give up and choose stable if you can't guess. INFO("Jetbackup tier '$jetbackup_tier' detected. Not removing jetbackup. Will re-install it after elevate."); + if ( ref Elevate::PkgMgr::instance() eq 'Elevate::PkgMgr::APT' ) { + + if ( -f "/etc/apt/sources.list.d/$jetbackup_tier.list.disabled" ) { + rename( "/etc/apt/sources.list.d/$jetbackup_tier.list.disabled", "/etc/apt/sources.list.d/$jetbackup_tier.list" ) || WARN("Couldn't enable repository for $jetbackup_tier: $!"); + } + } my @reinstall = Elevate::PkgMgr::get_installed_pkgs_in_repo(qw/jetapps jetapps-stable jetapps-beta jetapps-edge/); + + push @reinstall, 'jetbackup5-cpanel' if !grep { $_ eq 'jetbackup5-cpanel' } @reinstall; unshift @reinstall, $jetbackup_tier; + if ( Elevate::OS::needs_leapp() && Cpanel::Pkgr::is_installed('jetphp81-zip') ) { + Elevate::PkgMgr::remove_no_dependencies('jetphp81-zip'); + push @reinstall, 'jetphp81-zip' if !grep { $_ eq 'jetphp81-zip' } @reinstall; + } + my $data = { tier => $jetbackup_tier, packages => \@reinstall, @@ -3084,8 +3097,6 @@ EOS Elevate::StageFile::update_stage_file( { 'reinstall' => { 'jetbackup' => $data } } ); - Elevate::PkgMgr::remove_no_dependencies('jetphp81-zip'); - return; } @@ -3101,7 +3112,6 @@ EOS my $pkgmgr_options = [ '--enablerepo=jetapps', "--enablerepo=$tier" ]; - Elevate::PkgMgr::install_with_options( $pkgmgr_options, ['jetphp81-zip'] ); Elevate::PkgMgr::update_with_options( $pkgmgr_options, \@packages ); return; @@ -3109,22 +3119,12 @@ EOS sub check ($self) { - $self->_blocker_jetbackup_is_supported(); $self->_blocker_old_jetbackup; return; } - sub _blocker_jetbackup_is_supported ($self) { - return unless Cpanel::Pkgr::is_installed('jetbackup'); - return if Elevate::OS::supports_jetbackup(); - - my $name = Elevate::OS::default_upgrade_to(); - return $self->has_blocker( <<~"END" ); - ELevate does not currently support JetBackup for upgrades of $name. - Support for JetBackup on $name will be added in a future version of ELevate. - END - } + sub _blocker_jetbackup_is_supported { return undef } sub _blocker_old_jetbackup ($self) { @@ -7220,6 +7220,8 @@ deb-src https://repo.mysql.com/apt/ubuntu/ jammy mysql-8.0}, deb https://wp-toolkit.plesk.com/cPanel/Ubuntu-22.04-x86_64/latest/wp-toolkit/ ./ deb https://wp-toolkit.plesk.com/cPanel/Ubuntu-22.04-x86_64/latest/thirdparty/ ./}, + + map { my $thing = $_; "jetapps-$_.list" => "deb [arch=amd64] https://repo.jetlicense.com/ubuntu jammy/$_ main" } qw{base plugins alpha beta edge rc release stable}, }; use constant supported_cpanel_mysql_versions => qw{ @@ -7400,10 +7402,13 @@ deb https://wp-toolkit.plesk.com/cPanel/Ubuntu-22.04-x86_64/latest/thirdparty/ . use cPstrict; - use File::Copy (); + use File::Copy (); + use File::Slurper (); use Elevate::OS (); + use Cpanel::Pkgr (); + # use Log::Log4perl qw(:easy); INIT { Log::Log4perl->import(qw{:easy}); } @@ -7576,6 +7581,38 @@ deb https://wp-toolkit.plesk.com/cPanel/Ubuntu-22.04-x86_64/latest/thirdparty/ . return; } + my %repo2filepart = ( + map { my $dollar_underscore = $_; "jetapps-$dollar_underscore" => "repo.jetlicense.com_ubuntu_dists_jammy_$dollar_underscore" } qw{base plugins alpha beta edge rc release stable}, + ); + + sub get_installed_pkgs_in_repo ( $self, @repos ) { + return unless scalar @repos; + + my @apt_args = (APT_NON_INTERACTIVE_ARGS); + $self->ssystem_and_die( $Elevate::PkgMgr::APT::apt_get, @apt_args, 'update' ); + + my $dir2read = '/var/lib/apt/lists'; + opendir( my $dh, $dir2read ) or die "Can't open $dir2read: $!"; + my @pkglists = grep { m/_Packages$/ } readdir($dh); + closedir($dh); + $repos[0] = 'jetapps-base' if $repos[0] eq 'jetapps'; + my @installed; + foreach my $repo (@repos) { + my $filepart = $repo2filepart{$repo}; + foreach my $pkglist (@pkglists) { + if ( $pkglist =~ qr/^$filepart/ ) { + my @lines = File::Slurper::read_lines("$dir2read/$pkglist"); + + my %pkgs = map { substr( $_, 9 ) => 1 } grep { m/^Package: / } @lines; + push @installed, sort grep { Cpanel::Pkgr::is_installed($_) } keys %pkgs; + last; + } + } + } + + return @installed; + } + 1; } # --- END lib/Elevate/PkgMgr/APT.pm diff --git a/lib/Elevate/Components/JetBackup.pm b/lib/Elevate/Components/JetBackup.pm index d05b32da..a3da646b 100644 --- a/lib/Elevate/Components/JetBackup.pm +++ b/lib/Elevate/Components/JetBackup.pm @@ -48,9 +48,25 @@ sub pre_distro_upgrade ($self) { : $repos->{'jetapps-beta'} ? 'jetapps-beta' : 'jetapps-stable'; # Just give up and choose stable if you can't guess. INFO("Jetbackup tier '$jetbackup_tier' detected. Not removing jetbackup. Will re-install it after elevate."); + if ( ref Elevate::PkgMgr::instance() eq 'Elevate::PkgMgr::APT' ) { + + # We need to enable that repo if possible and run apt update so that we can fetch the pkglist for it + if ( -f "/etc/apt/sources.list.d/$jetbackup_tier.list.disabled" ) { + rename( "/etc/apt/sources.list.d/$jetbackup_tier.list.disabled", "/etc/apt/sources.list.d/$jetbackup_tier.list" ) || WARN("Couldn't enable repository for $jetbackup_tier: $!"); + } + } my @reinstall = Elevate::PkgMgr::get_installed_pkgs_in_repo(qw/jetapps jetapps-stable jetapps-beta jetapps-edge/); + + # Force certain things onto the list no matter what + push @reinstall, 'jetbackup5-cpanel' if !grep { $_ eq 'jetbackup5-cpanel' } @reinstall; unshift @reinstall, $jetbackup_tier; + # Remove this package because leapp will remove it as it depends on libzip.so.2 which isn't available in 8. + if ( Elevate::OS::needs_leapp() && Cpanel::Pkgr::is_installed('jetphp81-zip') ) { + Elevate::PkgMgr::remove_no_dependencies('jetphp81-zip'); + push @reinstall, 'jetphp81-zip' if !grep { $_ eq 'jetphp81-zip' } @reinstall; + } + my $data = { tier => $jetbackup_tier, packages => \@reinstall, @@ -58,9 +74,6 @@ sub pre_distro_upgrade ($self) { Elevate::StageFile::update_stage_file( { 'reinstall' => { 'jetbackup' => $data } } ); - # Remove this package because leapp will remove it as it depends on libzip.so.2 which isn't available in 8. - Elevate::PkgMgr::remove_no_dependencies('jetphp81-zip'); - return; } @@ -76,7 +89,8 @@ sub post_distro_upgrade ($self) { my $pkgmgr_options = [ '--enablerepo=jetapps', "--enablerepo=$tier" ]; - Elevate::PkgMgr::install_with_options( $pkgmgr_options, ['jetphp81-zip'] ); + # Technically, this isn't possible on ubuntu, but we need it for cent. + # It falls through to just update thankfully. Elevate::PkgMgr::update_with_options( $pkgmgr_options, \@packages ); return; @@ -84,24 +98,12 @@ sub post_distro_upgrade ($self) { sub check ($self) { - $self->_blocker_jetbackup_is_supported(); $self->_blocker_old_jetbackup; return; } -# Support for JetBackup on Ubuntu is scheduled via RE-668 -# For now, we block to reduce scope for the initial release -sub _blocker_jetbackup_is_supported ($self) { - return unless Cpanel::Pkgr::is_installed('jetbackup'); - return if Elevate::OS::supports_jetbackup(); - - my $name = Elevate::OS::default_upgrade_to(); - return $self->has_blocker( <<~"END" ); - ELevate does not currently support JetBackup for upgrades of $name. - Support for JetBackup on $name will be added in a future version of ELevate. - END -} +sub _blocker_jetbackup_is_supported { return undef } sub _blocker_old_jetbackup ($self) { diff --git a/lib/Elevate/OS/Ubuntu20.pm b/lib/Elevate/OS/Ubuntu20.pm index 57c8af56..73bbee2b 100644 --- a/lib/Elevate/OS/Ubuntu20.pm +++ b/lib/Elevate/OS/Ubuntu20.pm @@ -46,6 +46,9 @@ deb https://wp-toolkit.plesk.com/cPanel/Ubuntu-22.04-x86_64/latest/wp-toolkit/ . # WP Toolkit Thirdparties deb https://wp-toolkit.plesk.com/cPanel/Ubuntu-22.04-x86_64/latest/thirdparty/ ./}, + + # If you don't assign $_ it is a syntax error in this context. Can't cheat with postfix for either. + map { my $thing = $_; "jetapps-$_.list" => "deb [arch=amd64] https://repo.jetlicense.com/ubuntu jammy/$_ main" } qw{base plugins alpha beta edge rc release stable}, }; use constant supported_cpanel_mysql_versions => qw{ diff --git a/lib/Elevate/PkgMgr/APT.pm b/lib/Elevate/PkgMgr/APT.pm index ad5f6b28..67085764 100644 --- a/lib/Elevate/PkgMgr/APT.pm +++ b/lib/Elevate/PkgMgr/APT.pm @@ -12,10 +12,13 @@ Logic wrapping the DEBIAN based package managers use cPstrict; -use File::Copy (); +use File::Copy (); +use File::Slurper (); use Elevate::OS (); +use Cpanel::Pkgr (); + use Log::Log4perl qw(:easy); use parent 'Elevate::PkgMgr::Base'; @@ -256,4 +259,64 @@ sub remove_pkgs_from_repos ( $self, @pkg_list ) { return; } +=head1 get_installed_pkgs_in_repo + +The "repo" in this case looks different from how it'd look in yum (name wise) +as we're ultimately just reading from /var/lib/apt/lists/ for the cached +list of packages from whatever URL. + +As such i've made a small translation hash here so that callers which so far +still need this don't have to get updated. + +Yeah it's a pain in the rear but I'd rather not add more abstraction around +this right now (I'd rather fix Cpanel::RepoQuery). + +=cut + +# So, I *could* use Cpanel::RepoQuery here, but it isn't quite suited to what +# we are doing here. First off we know more about the repos we care about, +# second of all there's a bug in RepoQuery: +# perl -MCpanel::RepoQuery -MData::Dumper -e 'print Data::Dumper::Dumper(Cpanel::RepoQuery::get_all_packages_from_repo("deb [arch=amd64] https://repo.jetlicense.com/ubuntu jammy/base main"));' +# Error open (<:unix) on '/var/lib/apt/lists/repo.jetlicense.com_ubuntu jammy_base main-mirrorlist_._Packages': No such file or directory at /usr/local/cpanel/Cpanel/RepoQuery/Apt.pm line 143. +# +# This works fine where we use it (cpanel-plugins) but isn't yet updated to be +# more generic. +# +# Also, excuse the tedious var naming. It's a syntax error to just use $_ here. +my %repo2filepart = ( + map { my $dollar_underscore = $_; "jetapps-$dollar_underscore" => "repo.jetlicense.com_ubuntu_dists_jammy_$dollar_underscore" } qw{base plugins alpha beta edge rc release stable}, +); + +sub get_installed_pkgs_in_repo ( $self, @repos ) { + return unless scalar @repos; + + # Forcibly update (not upgrade) apt so that you have the list on disk + # XXX This could mean a lot of apt update invocations. May want an extra + # caching layer here at some point if it becomes a problem. + my @apt_args = (APT_NON_INTERACTIVE_ARGS); + $self->ssystem_and_die( $Elevate::PkgMgr::APT::apt_get, @apt_args, 'update' ); + + my $dir2read = '/var/lib/apt/lists'; + opendir( my $dh, $dir2read ) or die "Can't open $dir2read: $!"; + my @pkglists = grep { m/_Packages$/ } readdir($dh); + closedir($dh); + $repos[0] = 'jetapps-base' if $repos[0] eq 'jetapps'; + my @installed; + foreach my $repo (@repos) { + my $filepart = $repo2filepart{$repo}; + foreach my $pkglist (@pkglists) { + if ( $pkglist =~ qr/^$filepart/ ) { + my @lines = File::Slurper::read_lines("$dir2read/$pkglist"); + + # Use hash to dedupe + my %pkgs = map { substr( $_, 9 ) => 1 } grep { m/^Package: / } @lines; + push @installed, sort grep { Cpanel::Pkgr::is_installed($_) } keys %pkgs; + last; + } + } + } + + return @installed; +} + 1; diff --git a/t/components-ElevateScript.t b/t/components-ElevateScript.t index 5d31a100..3492af64 100644 --- a/t/components-ElevateScript.t +++ b/t/components-ElevateScript.t @@ -14,7 +14,7 @@ use Test2::Tools::Explain; use Test2::Plugin::NoWarnings; use Test2::Tools::Exception; -use Test::MockFile 0.032; +use Test::MockFile 0.032 qw{nostrict}; use Test::MockModule qw/strict/; use lib $FindBin::Bin . "/lib"; @@ -29,7 +29,7 @@ my $cpev = cpev->new; my $script = $cpev->get_blocker('ElevateScript'); { - $0 = '/root/elevate-cpanel'; + local $0 = '/root/elevate-cpanel'; is( $script->_blocker_wrong_location(), { @@ -39,7 +39,7 @@ my $script = $cpev->get_blocker('ElevateScript'); q{We need elevate-cpanel to live in /scripts/} ); - $0 = ''; + local $0 = ''; is( $script->_blocker_wrong_location(), { @@ -49,9 +49,13 @@ my $script = $cpev->get_blocker('ElevateScript'); q{Handle if \$0 is broken.} ); - $0 = '/scripts/elevate-cpanel'; - is( $script->_blocker_wrong_location(), 0, "\$0 can be /scripts/" ); - $0 = '/usr/local/cpanel/scripts/elevate-cpanel'; + # Cwd::abs_path will return undef if the file doesn't exist + SKIP: { + skip "/scripts does not exist on this host, assertion will fail due to Cwd::abs_path returning undef in that case", 1 if ( !-d '/scripts' ); + local $0 = '/scripts/elevate-cpanel'; + is( $script->_blocker_wrong_location(), 0, "\$0 can be /scripts/" ) || diag explain $script->_blocker_wrong_location(); + } + local $0 = '/usr/local/cpanel/scripts/elevate-cpanel'; is( $script->_blocker_wrong_location(), 0, "\$0 can be /usr/local/cpanel/scripts/" ); } @@ -68,8 +72,8 @@ my $script = $cpev->get_blocker('ElevateScript'); is( $components->blockers, - [ - { + bag { + item { id => q[Elevate::Components::ElevateScript::_is_up_to_date], msg => <<~'EOS', The script could not fetch information about the latest version. @@ -77,8 +81,9 @@ my $script = $cpev->get_blocker('ElevateScript'); Pass the --skip-elevate-version-check flag to skip this check. EOS - } - ], + }; + etc; + }, "blocks when info about latest version can't be fetched" ); @@ -87,8 +92,8 @@ my $script = $cpev->get_blocker('ElevateScript'); is( $components->blockers, - [ - { + bag { + item { id => q[Elevate::Components::ElevateScript::_is_up_to_date], msg => <<~'EOS', The script could not fetch information about the latest version. @@ -96,8 +101,9 @@ my $script = $cpev->get_blocker('ElevateScript'); Pass the --skip-elevate-version-check flag to skip this check. EOS - } - ], + }; + etc; + }, "blocks when the installed script isn't the latest release" ); diff --git a/t/components-JetBackup.t b/t/components-JetBackup.t index 3350a759..35058dbd 100644 --- a/t/components-JetBackup.t +++ b/t/components-JetBackup.t @@ -84,23 +84,12 @@ my $jb = $cpev->get_blocker('JetBackup'); my $jb_mock = Test::MockModule->new('Elevate::Components::JetBackup'); - for my $os ( 'cent', 'cloud' ) { + for my $os ( 'cent', 'cloud', 'ubuntu' ) { set_os_to($os); my $expected_target_os = Elevate::OS::upgrade_to_pretty_name(); is( $jb->_blocker_jetbackup_is_supported(), undef, "JetBackup is supported for upgrades to $expected_target_os" ); } - - set_os_to('ubuntu'); - like( - $jb->_blocker_jetbackup_is_supported(), - { - id => q[Elevate::Components::JetBackup::_blocker_jetbackup_is_supported], - msg => qr/ELevate does not currently support JetBackup for upgrades of/, - }, - 'Jetbackup is not yet supported for Ubuntu upgrades', - ); - } done_testing(); diff --git a/t/usage.t b/t/usage.t index c7b74259..2bab7523 100644 --- a/t/usage.t +++ b/t/usage.t @@ -24,7 +24,8 @@ use Elevate::Usage; require $FindBin::Bin . '/../elevate-cpanel'; -my $mock_usage = Test::MockModule->new('Elevate::Usage'); +my $mock_no_noc = Test::MockFile->file('/var/cpanel/elevate-noc-recommendations'); +my $mock_usage = Test::MockModule->new('Elevate::Usage'); my $parse_results = {}; @@ -282,6 +283,7 @@ foreach my $test_hr (@TEST_DATA) { my $user_has_been_prompted = 0; +require IO::Prompt; my $mock_io_prompt = Test::MockModule->new('IO::Prompt'); $mock_io_prompt->redefine( prompt => sub { @@ -289,15 +291,14 @@ $mock_io_prompt->redefine( return 1; } ); - my $cpev = cpev->new->_init('--start'); $cpev->give_last_chance(); -is $user_has_been_prompted, 1, 'IP::Prompt invoked without the non-interactive option'; +is $user_has_been_prompted, 1, 'IO::Prompt invoked without the non-interactive option'; $user_has_been_prompted = 0; $cpev = cpev->new->_init( '--start', '--non-interactive' ); $cpev->give_last_chance(); -is $user_has_been_prompted, 0, 'IP::Prompt not invoked with the non-interactive option'; +is $user_has_been_prompted, 0, 'IO::Prompt not invoked with the non-interactive option'; done_testing(); exit;