diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7438b7f2..36b46190e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,13 +26,16 @@ jobs: ruby: ruby-3.1 - os: macos-latest ruby: ruby-3.0 + execution: + - bundle exec rspec spec/unit + - bundle exec mutant test run spec/unit steps: - uses: actions/checkout@v3 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} bundler-cache: true - - run: bundle exec rspec spec/unit + - run: ${{ matrix.execution }} ruby-mutant: name: Mutation coverage runs-on: ${{ matrix.os }} diff --git a/Gemfile.lock b/Gemfile.lock index 64fac9fe8..9e2497647 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -17,11 +17,11 @@ GEM remote: https://rubygems.org/ specs: ast (2.4.2) - diff-lcs (1.5.0) + diff-lcs (1.5.1) json (2.7.1) language_server-protocol (3.17.0.3) parallel (1.24.0) - parser (3.3.0.2) + parser (3.3.0.5) ast (~> 2.4.1) racc racc (1.7.3) @@ -44,11 +44,11 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) rspec-support (3.12.1) - rubocop (1.59.0) + rubocop (1.60.2) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) - parser (>= 3.2.2.4) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) @@ -58,11 +58,11 @@ GEM rubocop-ast (1.30.0) parser (>= 3.2.1.0) ruby-progressbar (1.13.0) - sorbet-runtime (0.5.11180) + sorbet-runtime (0.5.11222) unicode-display_width (2.5.0) - unparser (0.6.12) + unparser (0.6.13) diff-lcs (~> 1.3) - parser (>= 3.2.2.4) + parser (>= 3.3.0) PLATFORMS ruby diff --git a/docs/configuration.md b/docs/configuration.md index ed7b5b4bd..7292c821e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -89,6 +89,7 @@ integration: # Below shows an example configuring rspec to use a static seed from the config file. arguments: - --fail-fast # rspec integration default, keep this when specifying manual options! + - --force-color # rspec integration default, keep this if colored output is not an issue for you - --seed # option - '0' # option value, needs to be a string. - spec # rspec integration default, tell rspec integration where to find specs diff --git a/lib/mutant.rb b/lib/mutant.rb index 138252509..883215efc 100644 --- a/lib/mutant.rb +++ b/lib/mutant.rb @@ -204,6 +204,8 @@ module Mutant require 'mutant/expression/namespace' require 'mutant/expression/parser' require 'mutant/test' + require 'mutant/test/runner' + require 'mutant/test/runner/sink' require 'mutant/timer' require 'mutant/integration' require 'mutant/integration/null' @@ -243,6 +245,7 @@ module Mutant require 'mutant/reporter/cli/printer/mutation_result' require 'mutant/reporter/cli/printer/status_progressive' require 'mutant/reporter/cli/printer/subject_result' + require 'mutant/reporter/cli/printer/test' require 'mutant/reporter/cli/format' require 'mutant/repository' require 'mutant/repository/diff' diff --git a/lib/mutant/bootstrap.rb b/lib/mutant/bootstrap.rb index 7b6f3bc77..e0f0b6a30 100644 --- a/lib/mutant/bootstrap.rb +++ b/lib/mutant/bootstrap.rb @@ -47,6 +47,32 @@ def self.call(env) selected_subjects.flat_map(&:mutations) end + setup_integration( + env: env, + mutations: mutations, + selected_subjects: selected_subjects + ) + end + end + # rubocop:enable Metrics/MethodLength + + # Run test only bootstrap + # + # @param [Env] env + # + # @return [Either] + def self.call_test(env) + env.record(:bootstrap) do + setup_integration( + env: load_hooks(env), + mutations: [], + selected_subjects: [] + ) + end + end + + def self.setup_integration(env:, mutations:, selected_subjects:) + env.record(__method__) do Integration.setup(env).fmap do |integration| env.with( integration: integration, @@ -57,7 +83,6 @@ def self.call(env) end end end - # rubocop:enable Metrics/MethodLength def self.load_hooks(env) env.record(__method__) do diff --git a/lib/mutant/cli/command/environment/test.rb b/lib/mutant/cli/command/environment/test.rb index ffdb8a8b2..c1ba3ae60 100644 --- a/lib/mutant/cli/command/environment/test.rb +++ b/lib/mutant/cli/command/environment/test.rb @@ -8,6 +8,16 @@ class Test < self NAME = 'test' SHORT_DESCRIPTION = 'test subcommands' + private + + def bootstrap + env = Env.empty(world, @config) + + env + .record(:config) { Config.load(cli_config: @config, world: world) } + .bind { |config| Bootstrap.call_test(env.with(config: config)) } + end + class List < self NAME = 'list' SHORT_DESCRIPTION = 'List tests detected in the environment' @@ -28,7 +38,24 @@ def list_tests(env) end end - SUBCOMMANDS = [List].freeze + class Run < self + NAME = 'run' + SHORT_DESCRIPTION = 'Run tests' + SUBCOMMANDS = EMPTY_ARRAY + + private + + def parse_remaining_arguments(arguments) + arguments.each(&method(:add_integration_argument)) + Either::Right.new(self) + end + + def action + bootstrap.bind(&Mutant::Test::Runner.method(:call)) + end + end + + SUBCOMMANDS = [List, Run].freeze end # Test end # Environment end # Command diff --git a/lib/mutant/integration.rb b/lib/mutant/integration.rb index bf2a6eb68..28905b375 100644 --- a/lib/mutant/integration.rb +++ b/lib/mutant/integration.rb @@ -4,7 +4,11 @@ module Mutant # Abstract base class mutant test framework integrations class Integration - include AbstractType, Adamantium, Anima.new(:arguments, :expression_parser, :world) + include AbstractType, Adamantium, Anima.new( + :arguments, + :expression_parser, + :world + ) LOAD_MESSAGE = <<~'MESSAGE' Unable to load integration mutant-%s: diff --git a/lib/mutant/integration/rspec.rb b/lib/mutant/integration/rspec.rb index 3dfdb8180..ee9c23a68 100644 --- a/lib/mutant/integration/rspec.rb +++ b/lib/mutant/integration/rspec.rb @@ -24,7 +24,7 @@ class Rspec < self ALL_EXPRESSION = Expression::Namespace::Recursive.new(scope_name: nil) EXPRESSION_CANDIDATE = /\A([^ ]+)(?: )?/ EXIT_SUCCESS = 0 - DEFAULT_CLI_OPTIONS = %w[--fail-fast spec].freeze + DEFAULT_CLI_OPTIONS = %w[--fail-fast --force-color spec].freeze TEST_ID_FORMAT = 'rspec:%d:%s/%s' private_constant(*constants(false)) @@ -42,7 +42,7 @@ def initialize(*) # # @return [self] def setup - @runner.setup($stderr, $stdout) + @runner.setup(world.stderr, world.stdout) example_group_map reset_examples self @@ -55,6 +55,7 @@ def setup # # @return [Result::Test] def call(tests) + reset_examples setup_examples(tests.map(&all_tests_index)) start = timer.now passed = @runner.run_specs(@rspec_world.ordered_example_groups).equal?(EXIT_SUCCESS) diff --git a/lib/mutant/reporter/cli.rb b/lib/mutant/reporter/cli.rb index d8e52bc7e..8a8281003 100644 --- a/lib/mutant/reporter/cli.rb +++ b/lib/mutant/reporter/cli.rb @@ -28,6 +28,16 @@ def start(env) self end + # Report test start + # + # @param [Env] env + # + # @return [self] + def test_start(env) + write(format.test_start(env)) + self + end + # Report progress object # # @param [Parallel::Status] status @@ -38,6 +48,14 @@ def progress(status) self end + # Report progress object + # + # @return [self] + def test_progress(status) + write(format.test_progress(status)) + self + end + # Report delay in seconds # # @return [Float] @@ -65,6 +83,16 @@ def report(env) self end + # Report env + # + # @param [Result::Env] env + # + # @return [self] + def test_report(env) + Printer::Test::EnvResult.call(output: output, object: env) + self + end + private def write(frame) diff --git a/lib/mutant/reporter/cli/format.rb b/lib/mutant/reporter/cli/format.rb index b92e7c4d5..ecdfd8344 100644 --- a/lib/mutant/reporter/cli/format.rb +++ b/lib/mutant/reporter/cli/format.rb @@ -18,11 +18,14 @@ class Format # Progress representation # - # @param [Runner::Status] status - # # @return [String] abstract_method :progress + # Progress representation + # + # @return [String] + abstract_method :test_progress + # Report delay in seconds # # @return [Float] @@ -69,6 +72,13 @@ def start(env) format(Printer::Env, env) end + # Test start representation + # + # @return [String] + def test_start(env) + format(Printer::Test::Env, env) + end + # Progress representation # # @return [String] @@ -76,6 +86,13 @@ def progress(status) format(Printer::StatusProgressive, status) end + # Progress representation + # + # @return [String] + def test_progress(status) + format(Printer::Test::StatusProgressive, status) + end + private def new_buffer diff --git a/lib/mutant/reporter/cli/printer/test.rb b/lib/mutant/reporter/cli/printer/test.rb new file mode 100644 index 000000000..f455fbb0a --- /dev/null +++ b/lib/mutant/reporter/cli/printer/test.rb @@ -0,0 +1,155 @@ +# frozen_string_literal: true + +module Mutant + class Reporter + class CLI + class Printer + class Test < self + # Printer for test config + class Config < self + + # Report configuration + # + # @param [Mutant::Config] config + # + # @return [undefined] + # + def run + info('Fail-Fast: %s', object.fail_fast) + info('Integration: %s', object.integration.name || 'null') + info('Jobs: %s', object.jobs || 'auto') + end + end # Config + + # Env printer + class Env < self + delegate( + :amount_available_tests, + :amount_mutations, + :amount_selected_tests, + :amount_subjects, + :amount_all_tests, + :config, + :test_subject_ratio + ) + + FORMATS = [ + [:info, 'Tests: %s', :amount_all_tests] + ].each(&:freeze) + + # Run printer + # + # @return [undefined] + def run + info('Test environment:') + visit(Config, config) + FORMATS.each do |report, format, value| + __send__(report, format, __send__(value)) + end + end + end # Env + + class EnvProgress < self + delegate( + :amount_test_results, + :amount_tests_failed, + :amount_tests_success, + :env, + :runtime, + :testtime + ) + + FORMATS = [ + [:info, 'Test-Results: %0d', :amount_test_results ], + [:info, 'Test-Failed: %0d', :amount_tests_failed ], + [:info, 'Test-Success: %0d', :amount_tests_success ], + [:info, 'Runtime: %0.2fs', :runtime ], + [:info, 'Testtime: %0.2fs', :testtime ], + [:info, 'Efficiency: %0.2f%%', :efficiency_percent ] + ].each(&:freeze) + + # Run printer + # + # @return [undefined] + def run + visit(Env, object.env) + FORMATS.each do |report, format, value| + __send__(report, format, __send__(value)) + end + end + + private + + def efficiency_percent + (testtime / runtime) * 100 + end + end # EnvProgress + + # Full env result reporter + class EnvResult < self + + # Run printer + # + # @return [undefined] + def run + visit_collection(Result, object.failed_test_results) + visit(EnvProgress, object) + end + + end # EnvResult + + # Reporter for mutation results + class Result < self + + # Run report printer + # + # @return [undefined] + def run + puts(object.output) + end + + end # Result + + # Reporter for progressive output format on scheduler Status objects + class StatusProgressive < self + FORMAT = 'progress: %02d/%02d failed: %d runtime: %0.02fs testtime: %0.02fs tests/s: %0.02f' + + delegate( + :amount_test_results, + :amount_tests, + :amount_tests_failed, + :testtime, + :overhead, + :runtime + ) + + # Run printer + # + # @return [undefined] + def run + status( + FORMAT, + amount_test_results, + amount_tests, + amount_tests_failed, + runtime, + testtime, + tests_per_second + ) + end + + private + + def object + super().payload + end + + def tests_per_second + amount_test_results / runtime + end + end # StatusProgressive + end # Test + end # Printer + end # CLI + end # Reporter +end # Mutant diff --git a/lib/mutant/result.rb b/lib/mutant/result.rb index b6dd11dd9..0103f8882 100644 --- a/lib/mutant/result.rb +++ b/lib/mutant/result.rb @@ -91,6 +91,10 @@ def failed_subject_results sum :amount_timeouts, :subject_results sum :killtime, :subject_results + def overhead + runtime - killtime + end + # Amount of mutations # # @return [Integer] @@ -105,13 +109,71 @@ def amount_mutations def stop? env.config.fail_fast && !subject_results.all?(&:success?) end - end # Env + # TestEnv result object + class TestEnv + include Result, Anima.new( + :env, + :runtime, + :test_results + ) + + def overhead + runtime - testtime + end + + def inspect + '' + end + + # Test if run is successful + # + # @return [Boolean] + def success? + amount_tests_failed.equal?(0) + end + memoize :success? + + # Failed subject results + # + # @return [Array] + def failed_test_results + test_results.reject(&:success?) + end + memoize :failed_test_results + + def stop? + env.config.fail_fast && !test_results.all?(&:success?) + end + + def testtime + test_results.map(&:runtime).reduce(0, :+) + end + + def amount_tests + env.integration.all_tests.length + end + + def amount_test_results + test_results.length + end + + def amount_tests_failed + failed_test_results.length + end + + def amount_tests_success + test_results.count(&:passed) + end + end # TestEnv + # Test result class Test include Anima.new(:passed, :runtime) + alias_method :success?, :passed + class VoidValue < self include Singleton @@ -127,6 +189,13 @@ def initialize end # VoidValue end # Test + # Test result + class Test2 + include Anima.new(:passed, :runtime, :output) + + alias_method :success?, :passed + end # Test2 + # Subject result class Subject include CoverageMetric, Result, Anima.new( diff --git a/lib/mutant/test/runner.rb b/lib/mutant/test/runner.rb new file mode 100644 index 000000000..5894c5573 --- /dev/null +++ b/lib/mutant/test/runner.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Mutant + class Test + module Runner + # Run against env + # + # @return [Either] + def self.call(env) + reporter(env).test_start(env) + + Either::Right.new(run_tests(env)) + end + + def self.run_tests(env) + reporter = reporter(env) + + env + .record(:tests) { run_driver(reporter, async_driver(env)) } + .tap { |result| env.record(:report) { reporter.test_report(result) } } + end + private_class_method :run_tests + + def self.async_driver(env) + Parallel.async(env.world, test_config(env)) + end + private_class_method :async_driver + + def self.run_driver(reporter, driver) + Signal.trap('INT') do + driver.stop + end + + loop do + status = driver.wait_timeout(reporter.delay) + break status.payload if status.done? + reporter.test_progress(status) + end + end + private_class_method :run_driver + + def self.test_config(env) + Parallel::Config.new( + block: ->(index) { run_test_index(env, index) }, + jobs: env.config.jobs, + on_process_start: ->(index) {}, + process_name: 'mutant-test-runner-process', + sink: Sink.new(env), + source: Parallel::Source::Array.new(jobs: env.integration.all_tests.each_index.to_a.reverse), + thread_name: 'mutant-test-runner-thread' + ) + end + private_class_method :test_config + + def self.run_test_index(env, test_index) + test_result = nil + + output = with_capture(env.world) do + test_result = env.integration.call([env.integration.all_tests.fetch(test_index)]) + end + + Result::Test2.new( + passed: test_result.passed, + runtime: test_result.runtime, + output: output + ) + end + private_class_method :run_test_index + + # rubocop:disable Metrics/MethodLength + def self.with_capture(world) + stderr, stdout = world.stderr, world.stdout + original_stdout = stderr.dup + original_stderr = stderr.dup + Tempfile.create('capture') do |io| + stdout.reopen(io) + stdout.sync = true + stderr.reopen(stdout) + yield + io.rewind + io.read + end + ensure + stdout.reopen(original_stdout) + stderr.reopen(original_stderr) + end + # rubocop:enable Metrics/MethodLength + + def self.reporter(env) + env.config.reporter + end + private_class_method :reporter + end # Runner + end # Test +end # Mutant diff --git a/lib/mutant/test/runner/sink.rb b/lib/mutant/test/runner/sink.rb new file mode 100644 index 000000000..dd37d9cbb --- /dev/null +++ b/lib/mutant/test/runner/sink.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Mutant + class Test + module Runner + class Sink + include Concord.new(:env) + + # Initialize object + # + # @return [undefined] + def initialize(*) + super + @start = env.world.timer.now + @test_results = [] + end + + # Runner status + # + # @return [Result::Env] + def status + Result::TestEnv.new( + env: env, + runtime: env.world.timer.now - @start, + test_results: @test_results + ) + end + + # Test if scheduling stopped + # + # @return [Boolean] + def stop? + status.stop? + end + + # Handle mutation finish + # + # @return [self] + def result(result) + @test_results << result + self + end + end # Sink + end # Runner + end # Test +end # Mutant diff --git a/scripts/devloop.sh b/scripts/devloop.sh index e17dfda4c..d9aa2ca6a 100755 --- a/scripts/devloop.sh +++ b/scripts/devloop.sh @@ -1,5 +1,5 @@ -while inotifywait lib/**/*.rb meta/**/*.rb spec/**/*.rb Gemfile Gemfile.shared mutant.gemspec; do - bundle exec rspec spec/unit -fd --fail-fast --order default \ - && bundle exec mutant run --since main --fail-fast --zombie -- 'Mutant*' \ - && bundle exec rubocop +while inotifywait {lib,spec}/**/*.rb Gemfile Gemfile.shared mutant.gemspec; do + bundle exec mutant environment test run --fail-fast spec/unit # \ +## && bundle exec mutant run --since main --fail-fast --zombie -- 'Mutant*' +## && bundle exec rubocop done diff --git a/spec/support/corpus.rb b/spec/support/corpus.rb index 6c88dd3b9..8743574ce 100644 --- a/spec/support/corpus.rb +++ b/spec/support/corpus.rb @@ -210,7 +210,7 @@ def effective_ruby_paths # # @return [Pathname] def repo_path - TMP.join(name) + TMP.join("#{name}-#{Digest::SHA256.hexdigest(inspect)}") end # Test if installation should be skipped diff --git a/spec/unit/mutant/bootstrap_spec.rb b/spec/unit/mutant/bootstrap_spec.rb index 729c0b960..0d930e21c 100644 --- a/spec/unit/mutant/bootstrap_spec.rb +++ b/spec/unit/mutant/bootstrap_spec.rb @@ -179,6 +179,12 @@ def require(_); end arguments: [:mutation_generate], reaction: { yields: [] } }, + { + receiver: world, + selector: :record, + arguments: [:setup_integration], + reaction: { yields: [] } + }, { receiver: Mutant::Integration, selector: :setup, diff --git a/spec/unit/mutant/cli_spec.rb b/spec/unit/mutant/cli_spec.rb index 88b11c673..9064c6c0b 100644 --- a/spec/unit/mutant/cli_spec.rb +++ b/spec/unit/mutant/cli_spec.rb @@ -730,6 +730,11 @@ def self.main_body bootstrap_result end + allow(Mutant::Bootstrap).to receive(:call_test) do |env| + events << [:bootstrap, env.inspect] + bootstrap_result + end + allow(Mutant::Mutation::Runner).to receive(:call) do |env| events << [:runner, env.inspect] runner_result diff --git a/spec/unit/mutant/integration/rspec_spec.rb b/spec/unit/mutant/integration/rspec_spec.rb index 444c9d8c1..ed73c681e 100644 --- a/spec/unit/mutant/integration/rspec_spec.rb +++ b/spec/unit/mutant/integration/rspec_spec.rb @@ -11,7 +11,7 @@ ) end - let(:expected_rspec_cli) { %w[--fail-fast spec] } + let(:expected_rspec_cli) { %w[--fail-fast --force-color spec] } let(:integration_arguments) { [] } let(:rspec_options) { instance_double(RSpec::Core::ConfigurationOptions) } let(:rspec_runner) { instance_double(RSpec::Core::Runner) } @@ -191,8 +191,8 @@ before do expect(rspec_runner).to receive(:setup) do |error, stdout| - expect(error).to be($stderr) - expect(stdout).to be($stdout) + expect(error).to be(world.stderr) + expect(stdout).to be(world.stdout) end end @@ -206,8 +206,8 @@ before do expect(rspec_runner).to receive(:setup) do |error, stdout| - expect(error).to be($stderr) - expect(stdout).to be($stdout) + expect(error).to be(world.stderr) + expect(stdout).to be(world.stdout) end allow(rspec_runner).to receive_messages(run_specs: exit_status) end