From fa61fac0c5229f469155a3441897c57e2671562e Mon Sep 17 00:00:00 2001 From: "d.savuljesku" Date: Mon, 21 Oct 2024 10:20:20 +0200 Subject: [PATCH] Add restore support for farm --- README.md | 31 +++++- composer.json | 4 +- src/Commands/WikiBackup.php | 67 ++++++++---- src/Commands/WikiRestore.php | 67 +++++++++++- src/DatabaseImporter.php | 5 +- src/FarmInstanceSettingsManager.php | 161 ++++++++++++++++++++++++++++ src/FarmInstanceSettingsReader.php | 69 ------------ src/NullRestoreProfile.php | 2 - 8 files changed, 304 insertions(+), 102 deletions(-) create mode 100644 src/FarmInstanceSettingsManager.php delete mode 100644 src/FarmInstanceSettingsReader.php diff --git a/README.md b/README.md index e110f96..2b1e82f 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Also, you may want to exclude some data tables. It can be done that way: ``` -### Backing up/restoring farming instance +### Backing up farming instance Important: Database connection params, whether read from setting file or specified in `profile`, must refer to the main (`w`) wiki database, as that DB holds information on all instances. @@ -136,4 +136,31 @@ with `profile.json`: } ``` -will export all active instance of the farm, and then export `w` itself \ No newline at end of file +will export all active instance of the farm, and then export `w` itself + +### Restore wiki farm instance + +**Only instances that were backed up using this tool can be restored safely!** + +Since instance settings are stored in DB, when backing up, extra file `filesystem/settings.json` will be generated, +which is then used on restore. + +When restoring a farm instance, profile file __must__ be used, with `db-options.connection` set +to the main wiki database. Optionally, set `farm-options.instances-dir` to the root directory that holds instances. + +```json +{ + "db-options": { + "connection": { + "dbserver": "127.0.0.1", + "dbuser": "root", + "dbpassword": "...", + "dbname": "w" + } + }, + "farm-options": { + "instances-dir": "/path/to/instances" + } +} +``` + diff --git a/composer.json b/composer.json index b00733f..5ad8b99 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,9 @@ "symfony/console": "v3.4.9", "symfony/filesystem": "~5", "ifsnop/mysqldump-php": "v2.4", - "ext-pdo": "*" + "ext-pdo": "*", + "ext-json": "*", + "ext-zip": "*" }, "require-dev": { "phpunit/phpunit": "^8.5" diff --git a/src/Commands/WikiBackup.php b/src/Commands/WikiBackup.php index 55e0c40..53ab06a 100644 --- a/src/Commands/WikiBackup.php +++ b/src/Commands/WikiBackup.php @@ -4,7 +4,7 @@ use DateTime; use Exception; -use MWStake\MediaWiki\CliAdm\FarmInstanceSettingsReader; +use MWStake\MediaWiki\CliAdm\FarmInstanceSettingsManager; use MWStake\MediaWiki\CliAdm\IBackupProfile; use MWStake\MediaWiki\CliAdm\JSONBackupProfile; use MWStake\MediaWiki\CliAdm\DefaultBackupProfile; @@ -82,6 +82,9 @@ class WikiBackup extends Command { /** @var array */ private $skipDbPrefixes = []; + /** @var FarmInstanceSettingsManager|null */ + private $farmSettingsReader = null; + /** * * @var ZipArchive @@ -229,18 +232,18 @@ private function setupFarmEnvironment( array $options ) { throw new Exception( "Could not connect to management database: " . $ex->getMessage() ); } - $settingsReader = new FarmInstanceSettingsReader( $mainPdo, $settingsTable ); + $this->farmSettingsReader = new FarmInstanceSettingsManager( $mainPdo, $settingsTable ); if ( $this->instanceName === '*' ) { $this->output->writeln( "Backing up all instances ..." ); // Backup all instances $mainDbName = $this->dbname; $mainDbPrefix = $this->dbprefix; $originalMWRoot = $this->mediawikiRoot; - $activeInstances = $settingsReader->getAllActiveInstances(); + $activeInstances = $this->farmSettingsReader->getAllActiveInstances(); foreach ( $activeInstances as $instanceName ) { $this->instanceName = $instanceName; $this->mediawikiRoot = "$instancesDir/$instanceName"; - if ( $this->setupSingleFarmInstance( $settingsReader, $instanceName ) ) { + if ( $this->setupSingleFarmInstance( $this->farmSettingsReader, $instanceName ) ) { try { $this->doBackup(); } catch ( Exception $ex ) { @@ -257,22 +260,21 @@ private function setupFarmEnvironment( array $options ) { $this->dbprefix = $mainDbPrefix; $this->mediawikiRoot = $originalMWRoot; $this->isFarmContext = false; - $this->skipDbPrefixes = $settingsReader->getAllInstancePrefixes( $this->dbname ); + $this->skipDbPrefixes = $this->farmSettingsReader->getAllInstancePrefixes( $this->dbname ); } else { $this->mediawikiRoot = "$instancesDir/$this->instanceName"; - if ( !$this->setupSingleFarmInstance( $settingsReader, $this->instanceName ) ) { - throw new Exception( "Could not read settings for instance '{$this->instanceName}'" ); + if ( !$this->setupSingleFarmInstance( $this->instanceName ) ) { + throw new Exception( "Could not read settings for instance '$this->instanceName'" ); } } } /** - * @param FarmInstanceSettingsReader $settingsReader * @param string $instanceName * @return bool */ - private function setupSingleFarmInstance( FarmInstanceSettingsReader $settingsReader, string $instanceName ) { - $settings = $settingsReader->getSettings( $instanceName ); + private function setupSingleFarmInstance( string $instanceName ) { + $settings = $this->farmSettingsReader->getSettings( $instanceName ); if ( !$settings ) { return false; } @@ -378,10 +380,40 @@ protected function addImagesFolder() { } private function addCustomFilesAndFolders() { + $toBackup = $this->getCustomFilesToBackup(); + $backupCount = count( $toBackup ); + if ( $this->isFarmContext ) { + $backupCount++; + $settings = $this->farmSettingsReader->getFullSettings( $this->instanceName ); + if ( $settings === null ) { + throw new Exception( "Could not read settings for instance '$this->instanceName'" ); + } + $res = file_put_contents( + $this->mediawikiRoot . '/settings.json', + json_encode( $settings, JSON_PRETTY_PRINT ) + ); + if ( !$res ) { + throw new Exception( "Could not write instance settings file" ); + } + $toBackup[] = $this->mediawikiRoot . '/settings.json'; + } + + $progressBar = new ProgressBar( $this->output, $backupCount ); + $this->output->writeln( "Adding 'custom-paths' ..." ); + foreach( $toBackup as $customFile ) { + $localPath = preg_replace( '#^' . preg_quote( $this->mediawikiRoot ) . '#', '', $customFile ); + $this->zip->addFile( $customFile, "filesystem/$localPath" ); + $progressBar->advance(); + } + $progressBar->finish(); + $this->output->write( "\n" ); + } + + private function getCustomFilesToBackup(): array { $filesystemOptions = $this->profile->getFSBackupOptions(); $customPaths = $filesystemOptions['include-custom-paths'] ?? []; if ( empty( $customPaths ) ) { - return; + return []; } $customFilesToBackup = []; foreach( $customPaths as $customPath ) { @@ -404,18 +436,7 @@ private function addCustomFilesAndFolders() { $customFilesToBackup[] = $fileInfo->getPathname(); } } - $progressBar = new ProgressBar( - $this->output, - count( $customFilesToBackup ) - ); - $this->output->writeln( "Adding 'custom-paths' ..." ); - foreach( $customFilesToBackup as $customFile ) { - $localPath = preg_replace( '#^' . preg_quote( $this->mediawikiRoot ) . '#', '', $customFile ); - $this->zip->addFile( $customFile, "filesystem/$localPath" ); - $progressBar->advance(); - } - $progressBar->finish(); - $this->output->write( "\n" ); + return $customFilesToBackup; } protected $tmpDumpFilepath = ''; diff --git a/src/Commands/WikiRestore.php b/src/Commands/WikiRestore.php index 1d8663d..fa52d87 100644 --- a/src/Commands/WikiRestore.php +++ b/src/Commands/WikiRestore.php @@ -3,6 +3,7 @@ namespace MWStake\MediaWiki\CliAdm\Commands; use DateTime; +use MWStake\MediaWiki\CliAdm\FarmInstanceSettingsManager; use RecursiveIteratorIterator; use RecursiveDirectoryIterator; use Symfony\Component\Console\Command\Command; @@ -80,6 +81,23 @@ class WikiRestore extends Command { */ protected $filesystem = null; + /** + * @var bool + */ + private $isFarmContext = false; + + /** + * + * @var FarmInstanceSettingsManager + */ + private $farmSettingsManager = null; + + /** + * + * @var string + */ + private $instanceName = ''; + /** * * @var array @@ -139,6 +157,9 @@ protected function execute( Input\InputInterface $input, OutputInterface $output $this->readWikiConfig(); $this->importFilesystem(); $this->importDatabase(); + if ( $this->isFarmContext ) { + $this->farmSettingsManager->setInstanceSetting( $this->instanceName, 'sfi_status', 'ready' ); + } $this->removeTempWorkDir(); $this->outputEndInfo( $output ); } @@ -256,6 +277,49 @@ private function readWikiConfig() { $this->settings = $settingsReader->getSettingsFromDirectory( "{$this->tmpWorkingDir}/filesystem" ); + $this->setupFarmEnvironment(); + } + + /** + * @return void + * @throws Exception + */ + private function setupFarmEnvironment() { + $instanceSettingsFile = "$this->tmpWorkingDir/filesystem/settings.json"; + if ( !file_exists( $instanceSettingsFile ) ) { + return; + } + $mainDbConnectionOptions = $this->profile->getDBImportOptions()['connection'] ?? null; + if ( !$mainDbConnectionOptions ) { + throw new Exception( + "No main database connection options found in profile, but it's required when in farm context" + ); + } + $host = $mainDbConnectionOptions['dbserver'] ?? 'localhost'; + $mainPdo = new PDO( + "mysql:host=$host;dbname={$mainDbConnectionOptions['dbname']}", + $mainDbConnectionOptions['dbuser'], + $mainDbConnectionOptions['dbpassword'] + ); + $settingsTable = ( $mainDbConnectionOptions['dbprefix' ] ?? '' ) . 'simple_farmer_instances'; + $this->farmSettingsManager = new FarmInstanceSettingsManager( $mainPdo, $settingsTable ); + $settings = $this->farmSettingsManager->getSettingsFromFile( $instanceSettingsFile ); + if ( !$settings ) { + throw new Exception( "Failed to read settings.json for farm instance" ); + } + $farmOptions = $this->profile->getOptions()['farm-options'] ?? null; + $instancesDir = $farmOptions['instances-dir'] ?? $this->mediawikiRoot . '/_sf_instances'; + $this->instanceName = $settings['path']; + $this->settings['dbname'] = $settings['dbname']; + $this->settings['dbserver'] = $mainDbConnectionOptions['dbserver']; + $this->settings['dbuser'] = $mainDbConnectionOptions['dbuser']; + $this->settings['dbpassword'] = $mainDbConnectionOptions['dbpassword']; + $this->settings['dbprefix'] = $settings['dbprefix']; + $this->settings['wikiName'] = $settings['displayName']; + $this->mediawikiRoot = $instancesDir . '/' . $settings['path']; + if ( !$this->farmSettingsManager->assertInstanceEntryInitializedFromFile( $this->instanceName, $instanceSettingsFile ) ) { + throw new Exception( "Failed to init farm entry" ); + } } /** @@ -270,9 +334,6 @@ private function makePDO() { 'dbpassword' => $this->settings['dbpassword'] ]; - $profileDBOptions = $this->profile->getDBImportOptions(); - $connection = $profileDBOptions['connection'] + $connection; - $dsn = "mysql:host={$connection['dbserver']}"; $pdo = new PDO( $dsn, $connection['dbuser'], $connection['dbpassword'] ); $pdo->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); diff --git a/src/DatabaseImporter.php b/src/DatabaseImporter.php index 9497229..9f7df72 100644 --- a/src/DatabaseImporter.php +++ b/src/DatabaseImporter.php @@ -96,7 +96,9 @@ private function doImport( $pathname ) { try { $this->pdo->beginTransaction(); $this->pdo->exec( $tmpLine ); - $this->pdo->commit(); + if ($this->pdo->inTransaction() ) { + $this->pdo->commit(); + } } catch (PDOException $e) { $this->output->writeln( "Error performing Query: " . $tmpLine . ": " @@ -114,7 +116,6 @@ private function doImport( $pathname ) { if ($errorDetect) { return false; } - } private function isCommentLine( $line ) { diff --git a/src/FarmInstanceSettingsManager.php b/src/FarmInstanceSettingsManager.php new file mode 100644 index 0000000..a004a24 --- /dev/null +++ b/src/FarmInstanceSettingsManager.php @@ -0,0 +1,161 @@ +mainPdo = $mainPdo; + $this->settingsTable = $settingsTable; + } + + /** + * @param string $instanceName + * @return array|null + */ + public function getFullSettings( string $instanceName ): ?array { + // Retrieve all items where conditions $this->settingsTable.sfi_path=$this->instanceName + $query = 'SELECT * FROM ' . $this->settingsTable . ' WHERE sfi_path = \'' . $instanceName . '\' LIMIT 1'; + $res = $this->mainPdo->query( $query ); + $res->setFetchMode( PDO::FETCH_ASSOC ); + return $res->fetch() ?? null; + } + + public function getSettings( string $instanceName ): ?array { + return $this->rowToSettings( $this->getFullSettings( $instanceName ) ); + } + + /** + * @return array + */ + public function getAllActiveInstances(): array { + $query = 'SELECT sfi_path FROM ' . $this->settingsTable . ' WHERE sfi_status=\'ready\''; + $res = $this->mainPdo->query( $query ); + $res->setFetchMode( PDO::FETCH_ASSOC ); + $instances = []; + foreach ( $res as $row ) { + $instances[] = $row['sfi_path']; + } + return $instances; + } + + /** + * @param string $dbName + * @return array + */ + public function getAllInstancePrefixes( string $dbName ): array { + $query = 'SELECT sfi_db_prefix FROM ' . $this->settingsTable . ' WHERE sfi_database=\'' . $dbName . '\''; + $res = $this->mainPdo->query( $query ); + $res->setFetchMode( PDO::FETCH_ASSOC ); + $prefixes = []; + foreach ( $res as $row ) { + $prefixes[] = $row['sfi_db_prefix']; + } + return $prefixes; + } + + /** + * @param string $file + * @return array|null + */ + public function getSettingsFromFile( string $file ): ?array { + return $this->rowToSettings( $this->readInstanceSettingsFile( $file ) ); + } + + /** + * @param array|null $row + * @return array|null + */ + private function rowToSettings( ?array $row ) { + if ( $row ) { + return [ + 'path' => $row['sfi_path'], + 'displayName' => $row['sfi_display_name'], + 'dbname' => $row['sfi_database'], + 'dbprefix' => $row['sfi_db_prefix'], + ]; + } + return null; + } + + /** + * @param string $file + * @return array|null + */ + private function readInstanceSettingsFile( string $file ): ?array { + $fopen = fopen( $file, 'r' ); + if ( !$fopen ) { + return null; + } + $settings = json_decode( fread( $fopen, filesize( $file ) ), true ); + fclose( $fopen ); + if ( !$settings ) { + return null; + } + return $settings; + } + + /** + * @param string $instanceName + * @param string $file + * @return bool + */ + public function assertInstanceEntryInitializedFromFile( string $instanceName, string $file ): bool { + $checkQuery = 'SELECT COUNT(*) as cnt FROM ' . $this->settingsTable . ' WHERE sfi_path = \'' . $instanceName . '\' LIMIT 1'; + + $res = $this->mainPdo->query( $checkQuery ); + $res->setFetchMode( PDO::FETCH_ASSOC ); + $row = $res->fetch(); + if ( $row['cnt'] > 0 ) { + return true; + } + return $this->initInstanceEntryFromFile( $file ); + } + + /** + * @param string $file + * @return bool + */ + private function initInstanceEntryFromFile( string $file ): bool { + $raw = $this->readInstanceSettingsFile( $file ); + if ( !$raw ) { + return false; + } + $raw['sfi_status'] = 'initializing'; + $fields = array_keys( $raw ); + $query = 'INSERT INTO ' . $this->settingsTable . ' (' . implode( ',', $fields ) . ') VALUES (:' . implode( ',:', $fields ) . ')'; + $stmt = $this->mainPdo->prepare( $query ); + if ( !$stmt ) { + return false; + } + foreach ( $raw as $field => $value ) { + $stmt->bindValue( ':' . $field, $value ); + } + return $stmt->execute( $raw ); + } + + /** + * @param string $instanceName + * @param string $field + * @param string $value + * @return void + */ + public function setInstanceSetting( string $instanceName, string $field, string $value ) { + $query = 'UPDATE ' . $this->settingsTable . ' SET ' . $field . ' = \'' . $value . '\' WHERE sfi_path = \'' . $instanceName . '\''; + $this->mainPdo->query( $query ); + } +} \ No newline at end of file diff --git a/src/FarmInstanceSettingsReader.php b/src/FarmInstanceSettingsReader.php deleted file mode 100644 index 92d500b..0000000 --- a/src/FarmInstanceSettingsReader.php +++ /dev/null @@ -1,69 +0,0 @@ -mainPdo = $mainPdo; - $this->settingsTable = $settingsTable; - } - - public function getSettings( string $instanceName ): ?array { - // Retrieve all items where conditions $this->settingsTable.sfi_path=$this->instanceName - $query = 'SELECT * FROM ' . $this->settingsTable . ' WHERE sfi_path = \'' . $instanceName . '\' LIMIT 1'; - $res = $this->mainPdo->query( $query ); - $res->setFetchMode( \PDO::FETCH_ASSOC ); - $row = $res->fetch(); - if ( $row ) { - return [ - 'displayName' => $row['sfi_display_name'], - 'dbname' => $row['sfi_database'], - 'dbprefix' => $row['sfi_db_prefix'], - ]; - } - return null; - } - - /** - * @return array - */ - public function getAllActiveInstances(): array { - $query = 'SELECT sfi_path FROM ' . $this->settingsTable . ' WHERE sfi_status=\'ready\''; - $res = $this->mainPdo->query( $query ); - $res->setFetchMode( \PDO::FETCH_ASSOC ); - $instances = []; - foreach ( $res as $row ) { - $instances[] = $row['sfi_path']; - } - return $instances; - } - - /** - * @param string $dbName - * @return array - */ - public function getAllInstancePrefixes( string $dbName ): array { - $query = 'SELECT sfi_db_prefix FROM ' . $this->settingsTable . ' WHERE sfi_database=\'' . $dbName . '\''; - $res = $this->mainPdo->query( $query ); - $res->setFetchMode( \PDO::FETCH_ASSOC ); - $prefixes = []; - foreach ( $res as $row ) { - $prefixes[] = $row['sfi_db_prefix']; - } - return $prefixes; - } -} \ No newline at end of file diff --git a/src/NullRestoreProfile.php b/src/NullRestoreProfile.php index 1d46fb7..83dc2d0 100644 --- a/src/NullRestoreProfile.php +++ b/src/NullRestoreProfile.php @@ -2,8 +2,6 @@ namespace MWStake\MediaWiki\CliAdm; -use MWStake\MediaWiki\CliAdm\IRestoreProfile; - class NullRestoreProfile implements IRestoreProfile { /**