From 2b10ed9e3a7b30bc085ba3a307f8b297cf79af35 Mon Sep 17 00:00:00 2001 From: mreiden Date: Tue, 3 Aug 2021 23:45:25 -0500 Subject: [PATCH 1/5] Require splitbrain/php-archive --- composer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/composer.json b/composer.json index a14ee9e..d65a330 100644 --- a/composer.json +++ b/composer.json @@ -19,6 +19,7 @@ "ext-zlib": "*", "php": ">=7.1.0", "geoip2/geoip2": "~2.0", + "splitbrain/php-archive": "~1.2", "symfony/http-kernel": "~2.8|~3.0|~4.0|~5.0", "symfony/dependency-injection": "~2.8|~3.0|~4.0|~5.0", "symfony/expression-language": "~2.8|~3.0|~4.0|~5.0", From 0922b4a43c99f9c6a5fb2380796de9bee4cc170a Mon Sep 17 00:00:00 2001 From: mreiden Date: Wed, 4 Aug 2021 00:43:33 -0500 Subject: [PATCH 2/5] Reduce memory usage of geoip2:update command Use splitbrain/php-archive to extract the database file in the geoip2:update command. This reduces memory usage and prevents an out of memory exception when run with Fedora's default 128MB memory_limit. --- src/Downloader/MaxMindDownloader.php | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/Downloader/MaxMindDownloader.php b/src/Downloader/MaxMindDownloader.php index 3bc80de..2148cad 100644 --- a/src/Downloader/MaxMindDownloader.php +++ b/src/Downloader/MaxMindDownloader.php @@ -12,6 +12,7 @@ namespace GpsLab\Bundle\GeoIP2Bundle\Downloader; use Psr\Log\LoggerInterface; +use splitbrain\PHPArchive\Tar; use Symfony\Component\Filesystem\Filesystem; /** @@ -56,31 +57,26 @@ public function download(string $url, string $target): void { $id = uniqid('', true); $tmp_zip = sprintf('%s/%s_GeoLite2.tar.gz', sys_get_temp_dir(), $id); - $tmp_unzip = sprintf('%s/%s_GeoLite2.tar', sys_get_temp_dir(), $id); $tmp_untar = sprintf('%s/%s_GeoLite2', sys_get_temp_dir(), $id); // remove old files and folders for correct overwrite it - $this->fs->remove([$tmp_zip, $tmp_unzip, $tmp_untar]); + $this->fs->remove([$tmp_zip, $tmp_untar]); $this->logger->debug(sprintf('Beginning download of file %s', $url)); $this->fs->copy($url, $tmp_zip, true); $this->logger->debug(sprintf('Download complete to %s', $tmp_zip)); - $this->logger->debug(sprintf('De-compressing file to %s', $tmp_unzip)); + $this->logger->debug(sprintf('Extracting archive file to %s', $tmp_zip)); $this->fs->mkdir(dirname($target), 0755); - // decompress gz file - $zip = new \PharData($tmp_zip); - $tar = $zip->decompress(); - - $this->logger->debug('Decompression complete'); - $this->logger->debug(sprintf('Extract tar file to %s', $tmp_untar)); - - // extract tar archive - $tar->extractTo($tmp_untar); - unset($zip, $tar); + // extract tar.gz archive + $tar = new Tar(); + $tar->open($tmp_zip); + $tar->extract($tmp_untar); + $tar->close(); + unset($tar); $this->logger->debug('Tar archive extracted'); @@ -104,12 +100,12 @@ public function download(string $url, string $target): void } if (!$database) { - throw new \RuntimeException('Not found GeoLite2 database in archive.'); + throw new \RuntimeException('GeoLite2 database was not found in archive.'); } $this->fs->copy($database, $target, true); $this->fs->chmod($target, 0755); - $this->fs->remove([$tmp_zip, $tmp_unzip, $tmp_untar]); + $this->fs->remove([$tmp_zip, $tmp_untar]); $this->logger->debug(sprintf('Database moved to %s', $target)); } From f1b0d589e08905651025392a0c2aaeb7c908b8f1 Mon Sep 17 00:00:00 2001 From: mreiden Date: Wed, 4 Aug 2021 17:35:12 -0500 Subject: [PATCH 3/5] MaxMindDownloader: - Fix log message - Break when file found --- src/Downloader/MaxMindDownloader.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Downloader/MaxMindDownloader.php b/src/Downloader/MaxMindDownloader.php index 2148cad..d068124 100644 --- a/src/Downloader/MaxMindDownloader.php +++ b/src/Downloader/MaxMindDownloader.php @@ -67,7 +67,7 @@ public function download(string $url, string $target): void $this->fs->copy($url, $tmp_zip, true); $this->logger->debug(sprintf('Download complete to %s', $tmp_zip)); - $this->logger->debug(sprintf('Extracting archive file to %s', $tmp_zip)); + $this->logger->debug(sprintf('Extracting archive file to %s', $tmp_untar)); $this->fs->mkdir(dirname($target), 0755); @@ -84,7 +84,10 @@ public function download(string $url, string $target): void $database = ''; $files = glob(sprintf('%s/**/*.mmdb', $tmp_untar)) ?: []; foreach ($files as $file) { - // expected something like that "GeoLite2-City_20200114" + // Use any .mmdb file in the archive, but stop searching if one matches the format expected below + $database = $file; + + // expected filename like "GeoLite2-City_YYYYMMDD" if (preg_match('/(?[^\/]+)_(?\d{4})(?\d{2})(?\d{2})/', $file, $match)) { $this->logger->debug(sprintf( 'Found %s database updated at %s-%s-%s in %s', @@ -94,9 +97,8 @@ public function download(string $url, string $target): void $match['day'], $file )); + break; } - - $database = $file; } if (!$database) { From 1514d47bf0c2315d4a6612169c9a27299e505c6d Mon Sep 17 00:00:00 2001 From: Mark Reidenbach Date: Wed, 18 Aug 2021 10:16:05 -0500 Subject: [PATCH 4/5] Use optional splitbrain extractor if installed --- README.md | 7 +++++ composer.json | 3 ++ src/Downloader/MaxMindDownloader.php | 45 ++++++++++++++++++---------- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index e4c57c8..8620cb9 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,13 @@ You can update several databases: php bin/console geoip2:update city country ``` +Optionally installing splitbrain/php-archive uses significantly less memory when updating a database +and can avoid out of memory errors: + +``` +composer req splitbrain/php-archive +``` + ### Download GeoIP database You can download custom database with console command: diff --git a/composer.json b/composer.json index d65a330..ddefb94 100644 --- a/composer.json +++ b/composer.json @@ -31,5 +31,8 @@ "phpunit/phpunit": "~7.0|~8.0|~9.0", "phpstan/phpstan": "^0.12", "phpstan/phpstan-phpunit": "^0.12" + }, + "suggest": { + "splitbrain/php-archive": "Greatly reduces memory usage for the geoip2:update command" } } diff --git a/src/Downloader/MaxMindDownloader.php b/src/Downloader/MaxMindDownloader.php index d068124..18adfa3 100644 --- a/src/Downloader/MaxMindDownloader.php +++ b/src/Downloader/MaxMindDownloader.php @@ -57,26 +57,43 @@ public function download(string $url, string $target): void { $id = uniqid('', true); $tmp_zip = sprintf('%s/%s_GeoLite2.tar.gz', sys_get_temp_dir(), $id); + $tmp_unzip = sprintf('%s/%s_GeoLite2.tar', sys_get_temp_dir(), $id); $tmp_untar = sprintf('%s/%s_GeoLite2', sys_get_temp_dir(), $id); // remove old files and folders for correct overwrite it - $this->fs->remove([$tmp_zip, $tmp_untar]); + $this->fs->remove([$tmp_zip, $tmp_unzip, $tmp_untar]); $this->logger->debug(sprintf('Beginning download of file %s', $url)); $this->fs->copy($url, $tmp_zip, true); $this->logger->debug(sprintf('Download complete to %s', $tmp_zip)); - $this->logger->debug(sprintf('Extracting archive file to %s', $tmp_untar)); $this->fs->mkdir(dirname($target), 0755); - // extract tar.gz archive - $tar = new Tar(); - $tar->open($tmp_zip); - $tar->extract($tmp_untar); - $tar->close(); - unset($tar); + if (class_exists(Tar::class)) { + $this->logger->debug(sprintf('Extracting archive file to %s', $tmp_untar)); + + // extract tar.gz archive + $tar = new Tar(); + $tar->open($tmp_zip); + $tar->extract($tmp_untar); + $tar->close(); + unset($tar); + } else { + $this->logger->debug(sprintf('De-compressing file to %s', $tmp_unzip)); + + // decompress gz file + $zip = new \PharData($tmp_zip); + $tar = $zip->decompress(); + + $this->logger->debug('Decompression complete'); + $this->logger->debug(sprintf('Extract tar file to %s', $tmp_untar)); + + // extract tar archive + $tar->extractTo($tmp_untar); + unset($zip, $tar); + } $this->logger->debug('Tar archive extracted'); @@ -84,10 +101,7 @@ public function download(string $url, string $target): void $database = ''; $files = glob(sprintf('%s/**/*.mmdb', $tmp_untar)) ?: []; foreach ($files as $file) { - // Use any .mmdb file in the archive, but stop searching if one matches the format expected below - $database = $file; - - // expected filename like "GeoLite2-City_YYYYMMDD" + // expected something like that "GeoLite2-City_20200114" if (preg_match('/(?[^\/]+)_(?\d{4})(?\d{2})(?\d{2})/', $file, $match)) { $this->logger->debug(sprintf( 'Found %s database updated at %s-%s-%s in %s', @@ -97,17 +111,18 @@ public function download(string $url, string $target): void $match['day'], $file )); - break; } + + $database = $file; } if (!$database) { - throw new \RuntimeException('GeoLite2 database was not found in archive.'); + throw new \RuntimeException('Not found GeoLite2 database in archive.'); } $this->fs->copy($database, $target, true); $this->fs->chmod($target, 0755); - $this->fs->remove([$tmp_zip, $tmp_untar]); + $this->fs->remove([$tmp_zip, $tmp_unzip, $tmp_untar]); $this->logger->debug(sprintf('Database moved to %s', $target)); } From 7bcae4f5ff99c03760cdb2e377f786ab942f0b42 Mon Sep 17 00:00:00 2001 From: Mark Reidenbach Date: Thu, 19 Aug 2021 08:09:03 -0500 Subject: [PATCH 5/5] Remove splitbrain/php-archive dependency --- composer.json | 1 - 1 file changed, 1 deletion(-) diff --git a/composer.json b/composer.json index ddefb94..54ed0fb 100644 --- a/composer.json +++ b/composer.json @@ -19,7 +19,6 @@ "ext-zlib": "*", "php": ">=7.1.0", "geoip2/geoip2": "~2.0", - "splitbrain/php-archive": "~1.2", "symfony/http-kernel": "~2.8|~3.0|~4.0|~5.0", "symfony/dependency-injection": "~2.8|~3.0|~4.0|~5.0", "symfony/expression-language": "~2.8|~3.0|~4.0|~5.0",