Use single pool

vimeo · Aug 31, 2023 · 2e5d936 · 2e5d936
1 parent b88c848
commit 2e5d936
Show file tree

Hide file tree

Showing 13 changed files with 134 additions and 86 deletions.
diff --git a/src/Psalm/Codebase.php b/src/Psalm/Codebase.php
@@ -2,6 +2,7 @@
 
 namespace Psalm;
 
+use Amp\Parallel\Worker\WorkerPool;
 use Exception;
 use InvalidArgumentException;
 use LanguageServerProtocol\Command;
@@ -37,6 +38,7 @@
 use Psalm\Internal\Codebase\TaintFlowGraph;
 use Psalm\Internal\DataFlow\TaintSink;
 use Psalm\Internal\DataFlow\TaintSource;
+use Psalm\Internal\Fork\Pool;
 use Psalm\Internal\LanguageServer\PHPMarkdownContent;
 use Psalm\Internal\LanguageServer\Reference;
 use Psalm\Internal\MethodIdentifier;
@@ -500,9 +502,9 @@ public function addFilesToAnalyze(array $files_to_analyze): void
     /**
      * Scans all files their related files
      */
-    public function scanFiles(int $threads = 1): void
+    public function scanFiles(): void
     {
-        $has_changes = $this->scanner->scanFiles($this->classlikes, $threads);
+        $has_changes = $this->scanner->scanFiles($this->classlikes);
 
         if ($has_changes) {
             $this->populator->populateCodebase();

diff --git a/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php b/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php
@@ -13,6 +13,7 @@
 use Psalm\FileManipulation;
 use Psalm\Internal\Codebase\TaintFlowGraph;
 use Psalm\Internal\FileManipulation\FileManipulationBuffer;
+use Psalm\Internal\Fork\Pool;
 use Psalm\Internal\LanguageServer\LanguageServer;
 use Psalm\Internal\MethodIdentifier;
 use Psalm\Internal\Provider\ClassLikeStorageProvider;
@@ -76,6 +77,7 @@
 use function mkdir;
 use function number_format;
 use function preg_match;
+use function Psl\Type\array_key;
 use function rename;
 use function sprintf;
 use function strlen;
@@ -123,8 +125,6 @@ class ProjectAnalyzer
 
     public bool $show_issues = true;
 
-    public int $threads;
-
     /**
      * @var array<string, bool>
      */
@@ -167,6 +167,11 @@ class ProjectAnalyzer
      */
     public array $generated_report_options;
 
+    /** @internal */
+    public Pool $pool;
+
+    /** @internal */
+    public int $threads;
     /**
      * @var array<int, class-string<CodeIssue>>
      */
@@ -271,6 +276,10 @@ public function __construct(
         }
 
         self::$instance = $this;
+
+        if ($threads > 1) {
+            $this->pool = new Pool($threads);
+        }
     }
 
     private function clearCacheDirectoryIfConfigOrComposerLockfileChanged(): void
@@ -392,6 +401,7 @@ public function serverMode(LanguageServer $server): void
 
         if ($usable_cpus > 1) {
             $this->threads = $usable_cpus;
+            $this->pool = new Pool($usable_cpus);
         }
 
         $server->logInfo("Initializing: Initialize Plugins...");
@@ -414,6 +424,12 @@ public function canReportIssues(string $file_path): bool
     {
         return isset($this->project_files[$file_path]);
     }
+    public function __sleep(): array
+    {
+        $vars = get_object_vars($this);
+        unset($vars['pool']);
+        return array_keys($vars);
+    }
 
     private function generatePHPVersionMessage(): string
     {
@@ -505,7 +521,7 @@ public function check(string $base_dir, bool $is_diff = false): void
 
             $this->config->initializePlugins($this);
 
-            $this->codebase->scanFiles($this->threads);
+            $this->codebase->scanFiles();
 
             $this->codebase->infer_types_from_usage = true;
         } else {
@@ -528,7 +544,7 @@ public function check(string $base_dir, bool $is_diff = false): void
 
                     $this->config->initializePlugins($this);
 
-                    $this->codebase->scanFiles($this->threads);
+                    $this->codebase->scanFiles();
                 } else {
                     $diff_no_files = true;
                 }
@@ -549,7 +565,6 @@ public function check(string $base_dir, bool $is_diff = false): void
 
         $this->codebase->analyzer->analyzeFiles(
             $this,
-            $this->threads,
             $this->codebase->alter_code,
             true,
         );
@@ -905,15 +920,14 @@ public function checkDir(string $dir_name): void
 
         $this->config->initializePlugins($this);
 
-        $this->codebase->scanFiles($this->threads);
+        $this->codebase->scanFiles();
 
         $this->config->visitStubFiles($this->codebase, $this->progress);
 
         $this->progress->startAnalyzingFiles();
 
         $this->codebase->analyzer->analyzeFiles(
             $this,
-            $this->threads,
             $this->codebase->alter_code,
             $this->codebase->find_unused_code === 'always',
         );
@@ -1015,15 +1029,14 @@ public function checkFile(string $file_path): void
 
         $this->config->initializePlugins($this);
 
-        $this->codebase->scanFiles($this->threads);
+        $this->codebase->scanFiles();
 
         $this->config->visitStubFiles($this->codebase, $this->progress);
 
         $this->progress->startAnalyzingFiles();
 
         $this->codebase->analyzer->analyzeFiles(
             $this,
-            $this->threads,
             $this->codebase->alter_code,
             $this->codebase->find_unused_code === 'always',
         );
@@ -1059,15 +1072,14 @@ public function checkPaths(array $paths_to_check): void
         $this->config->initializePlugins($this);
 
 
-        $this->codebase->scanFiles($this->threads);
+        $this->codebase->scanFiles();
 
         $this->config->visitStubFiles($this->codebase, $this->progress);
 
         $this->progress->startAnalyzingFiles();
 
         $this->codebase->analyzer->analyzeFiles(
             $this,
-            $this->threads,
             $this->codebase->alter_code,
             $this->codebase->find_unused_code === 'always',
         );

diff --git a/src/Psalm/Internal/Cli/Psalm.php b/src/Psalm/Internal/Cli/Psalm.php
@@ -13,7 +13,6 @@
 use Psalm\Internal\Codebase\ReferenceMapGenerator;
 use Psalm\Internal\Composer;
 use Psalm\Internal\ErrorHandler;
-use Psalm\Internal\Fork\Pool;
 use Psalm\Internal\Fork\PsalmRestarter;
 use Psalm\Internal\IncludeCollector;
 use Psalm\Internal\Provider\ClassLikeStorageCacheProvider;
@@ -72,15 +71,12 @@
 use function strlen;
 use function strpos;
 use function substr;
-use function version_compare;
 
 use const DIRECTORY_SEPARATOR;
 use const JSON_THROW_ON_ERROR;
 use const LC_CTYPE;
 use const PHP_EOL;
-use const PHP_OS;
 use const PHP_URL_SCHEME;
-use const PHP_VERSION;
 use const STDERR;
 
 // phpcs:disable PSR1.Files.SideEffects

diff --git a/src/Psalm/Internal/Codebase/Analyzer.php b/src/Psalm/Internal/Codebase/Analyzer.php
@@ -231,7 +231,6 @@ public function canReportIssues(string $file_path): bool
 
     public function analyzeFiles(
         ProjectAnalyzer $project_analyzer,
-        int $pool_size,
         bool $alter_code,
         bool $consolidate_analyzed_data = false
     ): void {
@@ -248,7 +247,7 @@ public function analyzeFiles(
             [$this->file_provider, 'fileExists'],
         );
 
-        $this->doAnalysis($project_analyzer, $pool_size);
+        $this->doAnalysis($project_analyzer);
 
         $scanned_files = $codebase->scanner->getScannedFiles();
 
@@ -300,16 +299,16 @@ public function analyzeFiles(
         }
     }
 
-    private function doAnalysis(ProjectAnalyzer $project_analyzer, int $pool_size): void
+    private function doAnalysis(ProjectAnalyzer $project_analyzer): void
     {
         $this->progress->start(count($this->files_to_analyze));
 
         ksort($this->files_to_analyze);
 
         $codebase = $project_analyzer->getCodebase();
 
-        if ($pool_size > 1 && count($this->files_to_analyze) > $pool_size) {
-            $shuffle_count = $pool_size + 1;
+        if ($project_analyzer->threads > 1 && count($this->files_to_analyze) > $project_analyzer->threads) {
+            $shuffle_count = $project_analyzer->threads + 1;
 
             $file_paths = array_values($this->files_to_analyze);
 
@@ -338,8 +337,7 @@ private function doAnalysis(ProjectAnalyzer $project_analyzer, int $pool_size):
 
             $this->progress->debug('Forking analysis' . "\n");
 
-            $forked_pool_data = Pool::run(
-                $pool_size,
+            $forked_pool_data = $project_analyzer->pool->run(
                 $new_file_paths,
                 new InitAnalyzerTask(),
                 AnalyzerTask::class,

diff --git a/src/Psalm/Internal/Codebase/Scanner.php b/src/Psalm/Internal/Codebase/Scanner.php
@@ -5,6 +5,7 @@
 use Psalm\Codebase;
 use Psalm\Config;
 use Psalm\Internal\Analyzer\IssueData;
+use Psalm\Internal\Analyzer\ProjectAnalyzer;
 use Psalm\Internal\ErrorHandler;
 use Psalm\Internal\Fork\InitScannerTask;
 use Psalm\Internal\Fork\Pool;
@@ -272,12 +273,12 @@ public function queueClassLikeForScanning(
         }
     }
 
-    public function scanFiles(ClassLikes $classlikes, int $pool_size = 1): bool
+    public function scanFiles(ClassLikes $classlikes): bool
     {
         $has_changes = false;
         while ($this->files_to_scan || $this->classes_to_scan) {
             if ($this->files_to_scan) {
-                if ($this->scanFilePaths($pool_size)) {
+                if ($this->scanFilePaths()) {
                     $has_changes = true;
                 }
             } else {
@@ -296,7 +297,7 @@ private function shouldScan(string $file_path): bool
                 || (isset($this->files_to_deep_scan[$file_path]) && !$this->scanned_files[$file_path]));
     }
 
-    private function scanFilePaths(int $pool_size): bool
+    private function scanFilePaths(): bool
     {
         $files_to_scan = array_filter(
             $this->files_to_scan,
@@ -309,19 +310,16 @@ private function scanFilePaths(int $pool_size): bool
             return false;
         }
 
-        if (!$this->is_forked && $pool_size > 1 && count($files_to_scan) > 512) {
-            $pool_size = ceil(min($pool_size, count($files_to_scan) / 256));
-        } else {
-            $pool_size = 1;
-        }
+        $project_analyzer = ProjectAnalyzer::getInstance();
+        $pool_size = $this->is_forked ? 1 : $project_analyzer->threads;
 
         if ($pool_size > 1) {
             $this->progress->debug('Forking process for scanning' . PHP_EOL);
 
+
             // Run scanning one file at a time, splitting the set of
             // files up among a given number of child processes.
-            $forked_pool_data = Pool::run(
-                $pool_size,
+            $forked_pool_data = ProjectAnalyzer::getInstance()->pool->run(
                 $files_to_scan,
                 new InitScannerTask(),
                 ScannerTask::class,

diff --git a/src/Psalm/Internal/Fork/InitAnalyzerTask.php b/src/Psalm/Internal/Fork/InitAnalyzerTask.php
@@ -8,9 +8,9 @@
 use Psalm\Internal\Analyzer\ProjectAnalyzer;
 use Psalm\Internal\Codebase\TaintFlowGraph;
 
-final class InitAnalyzerTask extends InitTask
+final class InitAnalyzerTask implements Task
 {
-    public function init(): void
+    public function run(Channel $channel, Cancellation $cancellation): mixed
     {
         $project_analyzer = ProjectAnalyzer::getInstance();
         $codebase = $project_analyzer->getCodebase();
@@ -30,5 +30,7 @@ public function init(): void
         $file_reference_provider->setFileReferencesToMissingClassMembers([]);
         $file_reference_provider->setReferencesToMixedMemberNames([]);
         $file_reference_provider->setMethodParamUses([]);
+
+        return null;
     }
 }
diff --git a/src/Psalm/Internal/Fork/InitScannerTask.php b/src/Psalm/Internal/Fork/InitScannerTask.php
@@ -5,18 +5,29 @@
 use Amp\Cancellation;
 use Amp\Parallel\Worker\Task;
 use Amp\Sync\Channel;
-use Assert\Assertion;
+use Psalm\Config;
 use Psalm\Internal\Analyzer\ProjectAnalyzer;
 use Psalm\Internal\CliUtils;
 use Psalm\Internal\ErrorHandler;
 use Psalm\Internal\Provider\ClassLikeStorageProvider;
 use Psalm\Internal\Provider\FileStorageProvider;
+use Psalm\Internal\VersionUtils;
+use Psalm\IssueBuffer;
+
+use function cli_get_process_title;
+use function cli_set_process_title;
+use function define;
+use function function_exists;
+use function gc_collect_cycles;
+use function gc_disable;
+use function ini_get;
+use function ini_set;
 
 use const PHP_EOL;
 
-final class InitScannerTask extends InitTask
+final class InitScannerTask implements Task
 {
-    public function init(): void
+    final public function run(Channel $channel, Cancellation $cancellation): mixed
     {
         $analyzer = ProjectAnalyzer::getInstance();
         $analyzer->progress->debug('Initialising forked process for scanning' . PHP_EOL);
@@ -31,5 +42,7 @@ public function init(): void
         $statements_provider->resetDiffs();
 
         $analyzer->progress->debug('Have initialised forked process for scanning' . PHP_EOL);
+
+        return null;
     }
 }