From 53645fe64fe739d7b273c88282e38e23b8750abd Mon Sep 17 00:00:00 2001 From: Markus Schirp Date: Sun, 23 Apr 2023 23:19:01 +0000 Subject: [PATCH] Add test only runner --- .github/workflows/ci.yml | 5 +- docs/configuration.md | 2 +- lib/mutant.rb | 5 + lib/mutant/bootstrap.rb | 28 +- lib/mutant/cli/command/environment/test.rb | 39 +- lib/mutant/env.rb | 9 +- lib/mutant/hooks.rb | 1 + lib/mutant/integration.rb | 6 +- lib/mutant/integration/minitest.rb | 1 + lib/mutant/integration/null.rb | 1 + lib/mutant/integration/rspec.rb | 29 +- lib/mutant/reporter.rb | 23 +- lib/mutant/reporter/cli.rb | 28 ++ lib/mutant/reporter/cli/format.rb | 21 +- lib/mutant/reporter/cli/printer/test.rb | 138 +++++++ lib/mutant/result.rb | 55 ++- lib/mutant/test/runner.rb | 62 ++++ lib/mutant/test/runner/sink.rb | 51 +++ lib/mutant/timer.rb | 9 + lib/mutant/world.rb | 2 + scripts/devloop.sh | 4 +- .../integration/mutant/isolation/fork_spec.rb | 4 +- spec/spec_helper.rb | 8 +- spec/support/corpus.rb | 2 +- spec/support/shared_context.rb | 2 + spec/unit/mutant/bootstrap_spec.rb | 348 ++++++++++-------- spec/unit/mutant/cli_spec.rb | 152 +++++--- spec/unit/mutant/env_spec.rb | 51 ++- spec/unit/mutant/integration/rspec_spec.rb | 60 ++- spec/unit/mutant/integration_spec.rb | 1 + .../reporter/cli/printer/test/config_spec.rb | 53 +++ .../cli/printer/test/env_result_spec.rb | 46 +++ .../reporter/cli/printer/test/result_spec.rb | 17 + .../printer/test/status_progressive_spec.rb | 48 +++ spec/unit/mutant/reporter/cli_spec.rb | 66 ++++ spec/unit/mutant/result/test_env_spec.rb | 159 ++++++++ spec/unit/mutant/result/test_spec.rb | 1 + spec/unit/mutant/test/runner/sink_spec.rb | 251 +++++++++++++ spec/unit/mutant/test/runner_spec.rb | 238 ++++++++++++ spec/unit/mutant/timer_spec.rb | 22 +- 40 files changed, 1831 insertions(+), 217 deletions(-) create mode 100644 lib/mutant/reporter/cli/printer/test.rb create mode 100644 lib/mutant/test/runner.rb create mode 100644 lib/mutant/test/runner/sink.rb create mode 100644 spec/unit/mutant/reporter/cli/printer/test/config_spec.rb create mode 100644 spec/unit/mutant/reporter/cli/printer/test/env_result_spec.rb create mode 100644 spec/unit/mutant/reporter/cli/printer/test/result_spec.rb create mode 100644 spec/unit/mutant/reporter/cli/printer/test/status_progressive_spec.rb create mode 100644 spec/unit/mutant/result/test_env_spec.rb create mode 100644 spec/unit/mutant/test/runner/sink_spec.rb create mode 100644 spec/unit/mutant/test/runner_spec.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43da9bb05..d5c02416c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,13 +26,16 @@ jobs: ruby: ruby-3.1 - os: macos-latest ruby: ruby-3.0 + execution: + - bundle exec rspec spec/unit + - bundle exec mutant environment test run spec/unit steps: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} bundler-cache: true - - run: bundle exec rspec spec/unit + - run: ${{ matrix.execution }} ruby-mutant: name: Mutation coverage runs-on: ${{ matrix.os }} diff --git a/docs/configuration.md b/docs/configuration.md index ed7b5b4bd..0a44f82f7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -182,7 +182,7 @@ mutation: ignore_patterns: - send{selector=log} # Select full mutation operators by default mutant only applies the light set - # Only difference between full and light right now is that light does not apply + # Only difference between full and light right now is that light does not apply # `#== -> #eql?` mutation # At this moment there is no CLI equivalent for this setting. operators: full # or `light` diff --git a/lib/mutant.rb b/lib/mutant.rb index 4c6523f33..fa91f9ab6 100644 --- a/lib/mutant.rb +++ b/lib/mutant.rb @@ -205,6 +205,8 @@ module Mutant require 'mutant/expression/namespace' require 'mutant/expression/parser' require 'mutant/test' + require 'mutant/test/runner' + require 'mutant/test/runner/sink' require 'mutant/timer' require 'mutant/integration' require 'mutant/integration/null' @@ -244,6 +246,7 @@ module Mutant require 'mutant/reporter/cli/printer/mutation_result' require 'mutant/reporter/cli/printer/status_progressive' require 'mutant/reporter/cli/printer/subject_result' + require 'mutant/reporter/cli/printer/test' require 'mutant/reporter/cli/format' require 'mutant/repository' require 'mutant/repository/diff' @@ -328,7 +331,9 @@ module Mutant recorder: recorder, stderr: $stderr, stdout: $stdout, + tempfile: Tempfile, thread: Thread, + time: Time, timer: timer ) diff --git a/lib/mutant/bootstrap.rb b/lib/mutant/bootstrap.rb index 7b6f3bc77..555173e59 100644 --- a/lib/mutant/bootstrap.rb +++ b/lib/mutant/bootstrap.rb @@ -47,6 +47,32 @@ def self.call(env) selected_subjects.flat_map(&:mutations) end + setup_integration( + env: env, + mutations: mutations, + selected_subjects: selected_subjects + ) + end + end + # rubocop:enable Metrics/MethodLength + + # Run test only bootstrap + # + # @param [Env] env + # + # @return [Either] + def self.call_test(env) + env.record(:bootstrap) do + setup_integration( + env: load_hooks(env), + mutations: [], + selected_subjects: [] + ) + end + end + + def self.setup_integration(env:, mutations:, selected_subjects:) + env.record(__method__) do Integration.setup(env).fmap do |integration| env.with( integration: integration, @@ -57,7 +83,7 @@ def self.call(env) end end end - # rubocop:enable Metrics/MethodLength + private_class_method :setup_integration def self.load_hooks(env) env.record(__method__) do diff --git a/lib/mutant/cli/command/environment/test.rb b/lib/mutant/cli/command/environment/test.rb index ffdb8a8b2..5e3758cd4 100644 --- a/lib/mutant/cli/command/environment/test.rb +++ b/lib/mutant/cli/command/environment/test.rb @@ -8,6 +8,21 @@ class Test < self NAME = 'test' SHORT_DESCRIPTION = 'test subcommands' + private + + def parse_remaining_arguments(arguments) + arguments.each(&method(:add_integration_argument)) + Either::Right.new(self) + end + + def bootstrap + env = Env.empty(world, @config) + + env + .record(:config) { Config.load(cli_config: @config, world: world) } + .bind { |config| Bootstrap.call_test(env.with(config: config)) } + end + class List < self NAME = 'list' SHORT_DESCRIPTION = 'List tests detected in the environment' @@ -28,7 +43,29 @@ def list_tests(env) end end - SUBCOMMANDS = [List].freeze + class Run < self + NAME = 'run' + SHORT_DESCRIPTION = 'Run tests' + SUBCOMMANDS = EMPTY_ARRAY + + private + + def action + bootstrap + .bind(&Mutant::Test::Runner.public_method(:call)) + .bind(&method(:from_result)) + end + + def from_result(result) + if result.success? + Either::Right.new(nil) + else + Either::Left.new('Test failures, exiting nonzero!') + end + end + end + + SUBCOMMANDS = [List, Run].freeze end # Test end # Environment end # Command diff --git a/lib/mutant/env.rb b/lib/mutant/env.rb index c5a7c4694..add38d847 100644 --- a/lib/mutant/env.rb +++ b/lib/mutant/env.rb @@ -66,10 +66,18 @@ def cover_index(mutation_index) ) end + def run_test_index(test_index) + integration.call([integration.all_tests.fetch(test_index)]) + end + def emit_mutation_worker_process_start(index:) hooks.run(:mutation_worker_process_start, index: index) end + def emit_test_worker_process_start(index:) + hooks.run(:test_worker_process_start, index: index) + end + # The test selections # # @return Hash{Mutation => Enumerable} @@ -175,7 +183,6 @@ def run_mutation_tests(mutation, tests) def timer world.timer end - end # Env # rubocop:enable Metrics/ClassLength end # Mutant diff --git a/lib/mutant/hooks.rb b/lib/mutant/hooks.rb index e9b1f5879..136a9db6f 100644 --- a/lib/mutant/hooks.rb +++ b/lib/mutant/hooks.rb @@ -10,6 +10,7 @@ class Hooks mutation_insert_post mutation_insert_pre mutation_worker_process_start + test_worker_process_start ].product([EMPTY_ARRAY]).to_h.transform_values(&:freeze).freeze MESSAGE = 'Unknown hook %s' diff --git a/lib/mutant/integration.rb b/lib/mutant/integration.rb index bf2a6eb68..28905b375 100644 --- a/lib/mutant/integration.rb +++ b/lib/mutant/integration.rb @@ -4,7 +4,11 @@ module Mutant # Abstract base class mutant test framework integrations class Integration - include AbstractType, Adamantium, Anima.new(:arguments, :expression_parser, :world) + include AbstractType, Adamantium, Anima.new( + :arguments, + :expression_parser, + :world + ) LOAD_MESSAGE = <<~'MESSAGE' Unable to load integration mutant-%s: diff --git a/lib/mutant/integration/minitest.rb b/lib/mutant/integration/minitest.rb index 735ffc091..7475160ab 100644 --- a/lib/mutant/integration/minitest.rb +++ b/lib/mutant/integration/minitest.rb @@ -109,6 +109,7 @@ def call(tests) reporter.report Result::Test.new( + output: '', passed: reporter.passed?, runtime: timer.now - start ) diff --git a/lib/mutant/integration/null.rb b/lib/mutant/integration/null.rb index 8e5ebb52c..0f3b5d26a 100644 --- a/lib/mutant/integration/null.rb +++ b/lib/mutant/integration/null.rb @@ -18,6 +18,7 @@ def all_tests # @return [Result::Test] def call(_tests) Result::Test.new( + output: '', passed: true, runtime: 0.0 ) diff --git a/lib/mutant/integration/rspec.rb b/lib/mutant/integration/rspec.rb index 3dfdb8180..a0ebac61d 100644 --- a/lib/mutant/integration/rspec.rb +++ b/lib/mutant/integration/rspec.rb @@ -19,8 +19,9 @@ class Integration # * location # Is NOT enough. It would not be unique. So we add an "example index" # for unique reference. + # + # rubocop:disable Metrics/ClassLength class Rspec < self - ALL_EXPRESSION = Expression::Namespace::Recursive.new(scope_name: nil) EXPRESSION_CANDIDATE = /\A([^ ]+)(?: )?/ EXIT_SUCCESS = 0 @@ -29,6 +30,11 @@ class Rspec < self private_constant(*constants(false)) + def freeze + super() if @setup_elapsed + self + end + # Initialize rspec integration # # @return [undefined] @@ -42,10 +48,15 @@ def initialize(*) # # @return [self] def setup - @runner.setup($stderr, $stdout) - example_group_map + @setup_elapsed = timer.elapsed do + @runner.setup(world.stderr, world.stdout) + fail 'Rspec setup failure' if RSpec.world.respond_to?(:rspec_is_quitting) && RSpec.world.rspec_is_quitting + example_group_map + end + @runner.configuration.force(color_mode: :on) + @runner.configuration.reporter reset_examples - self + freeze end memoize :setup @@ -54,15 +65,24 @@ def setup # @param [Enumerable] tests # # @return [Result::Test] + # + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/MethodLength def call(tests) + reset_examples setup_examples(tests.map(&all_tests_index)) + @runner.configuration.start_time = world.time.now - @setup_elapsed start = timer.now passed = @runner.run_specs(@rspec_world.ordered_example_groups).equal?(EXIT_SUCCESS) + @runner.configuration.reset_reporter Result::Test.new( + output: '', passed: passed, runtime: timer.now - start ) end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/MethodLength # All tests # @@ -149,5 +169,6 @@ def all_examples @rspec_world.example_groups.flat_map(&:descendants).flat_map(&:examples) end end # Rspec + # rubocop:enable Metrics/ClassLength end # Integration end # Mutant diff --git a/lib/mutant/reporter.rb b/lib/mutant/reporter.rb index 0641200c6..a50560f9d 100644 --- a/lib/mutant/reporter.rb +++ b/lib/mutant/reporter.rb @@ -19,13 +19,27 @@ class Reporter # @return [self] abstract_method :start - # Report collector state + # Report test start + # + # @param [Env] env + # + # @return [self] + abstract_method :test_start + + # Report final state # # @param [Runner::Collector] collector # # @return [self] abstract_method :report + # Report final test state + # + # @param [Runner::Collector] collector + # + # @return [self] + abstract_method :test_report + # Report progress on object # # @param [Object] object @@ -33,6 +47,13 @@ class Reporter # @return [self] abstract_method :progress + # Report progress on object + # + # @param [Object] object + # + # @return [self] + abstract_method :test_progress + # The reporter delay # # @return [Float] diff --git a/lib/mutant/reporter/cli.rb b/lib/mutant/reporter/cli.rb index 066079cc4..aaac84225 100644 --- a/lib/mutant/reporter/cli.rb +++ b/lib/mutant/reporter/cli.rb @@ -29,6 +29,16 @@ def start(env) self end + # Report test start + # + # @param [Env] env + # + # @return [self] + def test_start(env) + write(format.test_start(env)) + self + end + # Report progress object # # @param [Parallel::Status] status @@ -39,6 +49,14 @@ def progress(status) self end + # Report progress object + # + # @return [self] + def test_progress(status) + write(format.test_progress(status)) + self + end + # Report delay in seconds # # @return [Float] @@ -66,6 +84,16 @@ def report(env) self end + # Report env + # + # @param [Result::Env] env + # + # @return [self] + def test_report(env) + Printer::Test::EnvResult.call(output: output, object: env) + self + end + private def write(frame) diff --git a/lib/mutant/reporter/cli/format.rb b/lib/mutant/reporter/cli/format.rb index b92e7c4d5..ecdfd8344 100644 --- a/lib/mutant/reporter/cli/format.rb +++ b/lib/mutant/reporter/cli/format.rb @@ -18,11 +18,14 @@ class Format # Progress representation # - # @param [Runner::Status] status - # # @return [String] abstract_method :progress + # Progress representation + # + # @return [String] + abstract_method :test_progress + # Report delay in seconds # # @return [Float] @@ -69,6 +72,13 @@ def start(env) format(Printer::Env, env) end + # Test start representation + # + # @return [String] + def test_start(env) + format(Printer::Test::Env, env) + end + # Progress representation # # @return [String] @@ -76,6 +86,13 @@ def progress(status) format(Printer::StatusProgressive, status) end + # Progress representation + # + # @return [String] + def test_progress(status) + format(Printer::Test::StatusProgressive, status) + end + private def new_buffer diff --git a/lib/mutant/reporter/cli/printer/test.rb b/lib/mutant/reporter/cli/printer/test.rb new file mode 100644 index 000000000..321a6ab8a --- /dev/null +++ b/lib/mutant/reporter/cli/printer/test.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +module Mutant + class Reporter + class CLI + class Printer + class Test < self + # Printer for test config + class Config < self + + # Report configuration + # + # @param [Mutant::Config] config + # + # @return [undefined] + # + def run + info('Fail-Fast: %s', object.fail_fast) + info('Integration: %s', object.integration.name || 'null') + info('Jobs: %s', object.jobs || 'auto') + end + end # Config + + # Env printer + class Env < self + delegate( + :amount_available_tests, + :amount_selected_tests, + :amount_all_tests, + :config + ) + + FORMATS = [ + ].each(&:freeze) + + # Run printer + # + # @return [undefined] + def run + info('Test environment:') + visit(Config, config) + info('Tests: %s', amount_all_tests) + end + end # Env + + # Full env result reporter + class EnvResult < self + delegate( + :amount_test_results, + :amount_tests_failed, + :amount_tests_success, + :runtime, + :testtime + ) + + FORMATS = [ + [:info, 'Test-Results: %0d', :amount_test_results ], + [:info, 'Test-Failed: %0d', :amount_tests_failed ], + [:info, 'Test-Success: %0d', :amount_tests_success ], + [:info, 'Runtime: %0.2fs', :runtime ], + [:info, 'Testtime: %0.2fs', :testtime ], + [:info, 'Efficiency: %0.2f%%', :efficiency_percent ] + ].each(&:freeze) + + private_constant(*constants(false)) + + # Run printer + # + # @return [undefined] + def run + visit_collection(Result, object.failed_test_results) + visit(Env, object.env) + FORMATS.each do |report, format, value| + __send__(report, format, __send__(value)) + end + end + + private + + def efficiency_percent + (testtime / runtime) * 100 + end + end # EnvResult + + # Reporter for test results + class Result < self + + # Run report printer + # + # @return [undefined] + def run + puts(object.output) + end + + end # Result + + # Reporter for progressive output format on scheduler Status objects + class StatusProgressive < self + FORMAT = 'progress: %02d/%02d failed: %d runtime: %0.02fs testtime: %0.02fs tests/s: %0.02f' + + delegate( + :amount_test_results, + :amount_tests, + :amount_tests_failed, + :testtime, + :runtime + ) + + # Run printer + # + # @return [undefined] + def run + status( + FORMAT, + amount_test_results, + amount_tests, + amount_tests_failed, + runtime, + testtime, + tests_per_second + ) + end + + private + + def object + super().payload + end + + def tests_per_second + amount_test_results / runtime + end + end # StatusProgressive + end # Test + end # Printer + end # CLI + end # Reporter +end # Mutant diff --git a/lib/mutant/result.rb b/lib/mutant/result.rb index b6dd11dd9..2965b1bbc 100644 --- a/lib/mutant/result.rb +++ b/lib/mutant/result.rb @@ -105,12 +105,62 @@ def amount_mutations def stop? env.config.fail_fast && !subject_results.all?(&:success?) end - end # Env + # TestEnv result object + class TestEnv + include Result, Anima.new( + :env, + :runtime, + :test_results + ) + + # Test if run is successful + # + # @return [Boolean] + def success? + amount_tests_failed.equal?(0) + end + memoize :success? + + # Failed subject results + # + # @return [Array] + def failed_test_results + test_results.reject(&:success?) + end + memoize :failed_test_results + + def stop? + env.config.fail_fast && !test_results.all?(&:success?) + end + + def testtime + test_results.map(&:runtime).sum(0.0) + end + + def amount_tests + env.integration.all_tests.length + end + + def amount_test_results + test_results.length + end + + def amount_tests_failed + failed_test_results.length + end + + def amount_tests_success + test_results.count(&:passed) + end + end # TestEnv + # Test result class Test - include Anima.new(:passed, :runtime) + include Anima.new(:passed, :runtime, :output) + + alias_method :success?, :passed class VoidValue < self include Singleton @@ -120,6 +170,7 @@ class VoidValue < self # @return [undefined] def initialize super( + output: '', passed: false, runtime: 0.0 ) diff --git a/lib/mutant/test/runner.rb b/lib/mutant/test/runner.rb new file mode 100644 index 000000000..e00093e5d --- /dev/null +++ b/lib/mutant/test/runner.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Mutant + class Test + module Runner + # Run against env + # + # @return [Either] + def self.call(env) + reporter(env).test_start(env) + + Either::Right.new(run_tests(env)) + end + + def self.run_tests(env) + reporter = reporter(env) + + env + .record(:tests) { run_driver(reporter, async_driver(env)) } + .tap { |result| env.record(:report) { reporter.test_report(result) } } + end + private_class_method :run_tests + + def self.async_driver(env) + Parallel.async(world: env.world, config: test_config(env)) + end + private_class_method :async_driver + + def self.run_driver(reporter, driver) + Signal.trap('INT') do + driver.stop + end + + loop do + status = driver.wait_timeout(reporter.delay) + break status.payload if status.done? + reporter.test_progress(status) + end + end + private_class_method :run_driver + + def self.test_config(env) + Parallel::Config.new( + block: env.method(:run_test_index), + jobs: env.config.jobs, + on_process_start: env.method(:emit_test_worker_process_start), + process_name: 'mutant-test-runner-process', + sink: Sink.new(env: env), + source: Parallel::Source::Array.new(jobs: env.integration.all_tests.each_index.to_a), + thread_name: 'mutant-test-runner-thread', + timeout: nil + ) + end + private_class_method :test_config + + def self.reporter(env) + env.config.reporter + end + private_class_method :reporter + end # Runner + end # Test +end # Mutant diff --git a/lib/mutant/test/runner/sink.rb b/lib/mutant/test/runner/sink.rb new file mode 100644 index 000000000..072afe945 --- /dev/null +++ b/lib/mutant/test/runner/sink.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Mutant + class Test + module Runner + class Sink + include Anima.new(:env) + + # Initialize object + # + # @return [undefined] + def initialize(*) + super + @start = env.world.timer.now + @test_results = [] + end + + # Runner status + # + # @return [Result::Env] + def status + Result::TestEnv.new( + env: env, + runtime: env.world.timer.now - @start, + test_results: @test_results + ) + end + + # Test if scheduling stopped + # + # @return [Boolean] + def stop? + status.stop? + end + + # Handle mutation finish + # + # @return [self] + def response(response) + if response.error + env.world.stderr.puts(response.log) + fail response.error + end + + @test_results << response.result.with(output: response.log) + self + end + end # Sink + end # Runner + end # Test +end # Mutant diff --git a/lib/mutant/timer.rb b/lib/mutant/timer.rb index b6e1f9e5e..dfdb99a82 100644 --- a/lib/mutant/timer.rb +++ b/lib/mutant/timer.rb @@ -4,6 +4,15 @@ module Mutant class Timer include Anima.new(:process) + # Monotonic elapsed time of block execution + # + # @return [Float] + def elapsed + start = now + yield + now - start + end + # The now monotonic time # # @return [Float] diff --git a/lib/mutant/world.rb b/lib/mutant/world.rb index 398b9623a..80c18eb0e 100644 --- a/lib/mutant/world.rb +++ b/lib/mutant/world.rb @@ -22,7 +22,9 @@ class World :recorder, :stderr, :stdout, + :tempfile, :thread, + :time, :timer ) diff --git a/scripts/devloop.sh b/scripts/devloop.sh index e4db067db..48c839274 100755 --- a/scripts/devloop.sh +++ b/scripts/devloop.sh @@ -1,5 +1,5 @@ while inotifywait lib/**/*.rb meta/**/*.rb spec/**/*.rb Gemfile Gemfile.shared mutant.gemspec; do - bundle exec rspec spec/unit -fd --fail-fast --order defined \ - && bundle exec mutant run --since main --fail-fast --zombie -- 'Mutant*' \ + bundle exec mutant environment test run --fail-fast spec/unit \ + && bundle exec mutant run --fail-fast --since main --zombie -- 'Mutant*' \ && bundle exec rubocop done diff --git a/spec/integration/mutant/isolation/fork_spec.rb b/spec/integration/mutant/isolation/fork_spec.rb index cd7422e32..20d996a8b 100644 --- a/spec/integration/mutant/isolation/fork_spec.rb +++ b/spec/integration/mutant/isolation/fork_spec.rb @@ -29,7 +29,7 @@ def apply(&block) end context 'with configured timeout' do - let(:timeout) { 0.1 } + let(:timeout) { 1.0 } context 'when block exits within timeout' do def apply @@ -55,7 +55,7 @@ def apply it 'returns successful result' do result = apply - expect(result.timeout).to be(0.1) + expect(result.timeout).to be(1.0) expect(result.value).to be(nil) end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 78d1a590c..1477d9a14 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -100,6 +100,7 @@ def undefined double('undefined') end + # rubocop:disable Metrics/AbcSize # rubocop:disable Metrics/MethodLength def fake_world Mutant::World.new( @@ -119,12 +120,15 @@ def fake_world process: class_double(Process), random: class_double(Random), recorder: instance_double(Mutant::Segment::Recorder), - stderr: instance_double(IO), - stdout: instance_double(IO), + stderr: instance_double(IO, :stderr), + stdout: instance_double(IO, :stdout), + tempfile: class_double(Tempfile), thread: class_double(Thread), + time: class_double(Time), timer: instance_double(Mutant::Timer) ) end + # rubocop:enable Metrics/AbcSize # rubocop:enable Metrics/MethodLength end # XSpecHelper diff --git a/spec/support/corpus.rb b/spec/support/corpus.rb index 6c88dd3b9..8743574ce 100644 --- a/spec/support/corpus.rb +++ b/spec/support/corpus.rb @@ -210,7 +210,7 @@ def effective_ruby_paths # # @return [Pathname] def repo_path - TMP.join(name) + TMP.join("#{name}-#{Digest::SHA256.hexdigest(inspect)}") end # Test if installation should be skipped diff --git a/spec/support/shared_context.rb b/spec/support/shared_context.rb index 08f4483e5..ce0803c3b 100644 --- a/spec/support/shared_context.rb +++ b/spec/support/shared_context.rb @@ -222,6 +222,7 @@ def setup_shared_context let(:mutation_a_test_result) do Mutant::Result::Test.new( + output: '', passed: false, runtime: 1.0 ) @@ -229,6 +230,7 @@ def setup_shared_context let(:mutation_b_test_result) do Mutant::Result::Test.new( + output: '', passed: false, runtime: 1.0 ) diff --git a/spec/unit/mutant/bootstrap_spec.rb b/spec/unit/mutant/bootstrap_spec.rb index 729c0b960..9a3435891 100644 --- a/spec/unit/mutant/bootstrap_spec.rb +++ b/spec/unit/mutant/bootstrap_spec.rb @@ -1,24 +1,16 @@ # frozen_string_literal: true RSpec.describe Mutant::Bootstrap do - let(:env_with_scopes) { env_initial } let(:hooks) { instance_double(Mutant::Hooks) } let(:integration) { instance_double(Mutant::Integration) } let(:integration_result) { Mutant::Either::Right.new(integration) } let(:kernel) { fake_kernel.new } let(:load_path) { instance_double(Array, :load_path) } - let(:match_warnings) { [] } let(:object_space) { class_double(ObjectSpace) } let(:start_expressions) { [] } let(:subject_expressions) { [] } let(:timer) { instance_double(Mutant::Timer) } - let(:fake_kernel) do - Class.new do - def require(_); end - end - end - let(:config) do Mutant::Config::DEFAULT.with( environment_variables: { 'foo' => 'bar' }, @@ -30,21 +22,20 @@ def require(_); end ) end - let(:matcher_config) do - Mutant::Matcher::Config::DEFAULT.with( - subjects: subject_expressions, - start_expressions: start_expressions - ) - end - let(:env_initial) do Mutant::Env.empty(world, config).with(hooks: hooks) end - let(:expected_env) do - env_with_scopes.with( - integration: integration, - selector: Mutant::Selector::Expression.new(integration: integration) + let(:fake_kernel) do + Class.new do + def require(_); end + end + end + + let(:matcher_config) do + Mutant::Matcher::Config::DEFAULT.with( + subjects: subject_expressions, + start_expressions: start_expressions ) end @@ -69,140 +60,155 @@ def require(_); end end end - let(:raw_expectations) do - [ - { - receiver: world, - selector: :record, - arguments: [:bootstrap], - reaction: { yields: [] } - }, - { - receiver: world, - selector: :record, - arguments: [:load_hooks], - reaction: { yields: [] } - }, - { - receiver: Mutant::Hooks, - selector: :load_config, - arguments: [config], - reaction: { return: hooks } - }, - { - receiver: world, - selector: :record, - arguments: [:infect], - reaction: { yields: [] } - }, - { - receiver: world, - selector: :record, - arguments: [:hooks_env_infection_pre], - reaction: { yields: [] } - }, - { - receiver: hooks, - selector: :run, - arguments: [:env_infection_pre, { env: env_initial }] - }, - { - receiver: world, - selector: :record, - arguments: [:require_target], - reaction: { yields: [] } - }, - { - receiver: world.environment_variables, - selector: :[]=, - arguments: %w[foo bar] - }, - { - receiver: load_path, - selector: :<<, - arguments: %w[include-a] - }, - { - receiver: load_path, - selector: :<<, - arguments: %w[include-b] - }, - { - receiver: kernel, - selector: :require, - arguments: %w[require-a] - }, - { - receiver: kernel, - selector: :require, - arguments: %w[require-b] - }, - { - receiver: world, - selector: :record, - arguments: [:hooks_env_infection_post], - reaction: { yields: [] } - }, - { - receiver: hooks, - selector: :run, - arguments: [:env_infection_post, { env: env_initial }] - }, - { - receiver: world, - selector: :record, - arguments: [:matchable_scopes], - reaction: { yields: [] } - }, - { - receiver: object_space, - selector: :each_object, - arguments: [Module], - reaction: { return: object_space_modules.each } - }, - *match_warnings, - { - receiver: world, - selector: :record, - arguments: [:subject_match], - reaction: { yields: [] } - }, - { - receiver: world, - selector: :record, - arguments: [:subject_select], - reaction: { yields: [] } - }, - { - receiver: world, - selector: :record, - arguments: [:mutation_generate], - reaction: { yields: [] } - }, - { - receiver: Mutant::Integration, - selector: :setup, - arguments: [env_with_scopes], - reaction: { return: integration_result } - - } - ] - end + describe '#call' do + let(:env_with_scopes) { env_initial } + let(:match_warnings) { [] } + + let(:expected_env) do + env_with_scopes.with( + integration: integration, + selector: Mutant::Selector::Expression.new(integration: integration) + ) + end - def self.expect_warnings - let(:match_warnings) do + let(:raw_expectations) do [ { - receiver: config.reporter, - selector: :warn, - arguments: [expected_warning], - reaction: { return: config.reporter } + receiver: world, + selector: :record, + arguments: [:bootstrap], + reaction: { yields: [] } + }, + { + receiver: world, + selector: :record, + arguments: [:load_hooks], + reaction: { yields: [] } + }, + { + receiver: Mutant::Hooks, + selector: :load_config, + arguments: [config], + reaction: { return: hooks } + }, + { + receiver: world, + selector: :record, + arguments: [:infect], + reaction: { yields: [] } + }, + { + receiver: world, + selector: :record, + arguments: [:hooks_env_infection_pre], + reaction: { yields: [] } + }, + { + receiver: hooks, + selector: :run, + arguments: [:env_infection_pre, { env: env_initial }] + }, + { + receiver: world, + selector: :record, + arguments: [:require_target], + reaction: { yields: [] } + }, + { + receiver: world.environment_variables, + selector: :[]=, + arguments: %w[foo bar] + }, + { + receiver: load_path, + selector: :<<, + arguments: %w[include-a] + }, + { + receiver: load_path, + selector: :<<, + arguments: %w[include-b] + }, + { + receiver: kernel, + selector: :require, + arguments: %w[require-a] + }, + { + receiver: kernel, + selector: :require, + arguments: %w[require-b] + }, + { + receiver: world, + selector: :record, + arguments: [:hooks_env_infection_post], + reaction: { yields: [] } + }, + { + receiver: hooks, + selector: :run, + arguments: [:env_infection_post, { env: env_initial }] + }, + { + receiver: world, + selector: :record, + arguments: [:matchable_scopes], + reaction: { yields: [] } + }, + { + receiver: object_space, + selector: :each_object, + arguments: [Module], + reaction: { return: object_space_modules.each } + }, + *match_warnings, + { + receiver: world, + selector: :record, + arguments: [:subject_match], + reaction: { yields: [] } + }, + { + receiver: world, + selector: :record, + arguments: [:subject_select], + reaction: { yields: [] } + }, + { + receiver: world, + selector: :record, + arguments: [:mutation_generate], + reaction: { yields: [] } + }, + { + receiver: world, + selector: :record, + arguments: [:setup_integration], + reaction: { yields: [] } + }, + { + receiver: Mutant::Integration, + selector: :setup, + arguments: [env_with_scopes], + reaction: { return: integration_result } } ] end - end - describe '.call' do + def self.expect_warnings + let(:match_warnings) do + [ + { + receiver: config.reporter, + selector: :warn, + arguments: [expected_warning], + reaction: { return: config.reporter } + } + ] + end + end + def apply described_class.call(Mutant::Env.empty(world, config)) end @@ -339,4 +345,54 @@ def object.name end end end + + describe '#call_test' do + def apply + described_class.call_test(env_initial) + end + + let(:expected_env) do + env_initial.with( + integration: integration, + selector: Mutant::Selector::Expression.new(integration: integration) + ) + end + + let(:raw_expectations) do + [ + { + receiver: world, + selector: :record, + arguments: [:bootstrap], + reaction: { yields: [] } + }, + { + receiver: world, + selector: :record, + arguments: [:load_hooks], + reaction: { yields: [] } + }, + { + receiver: Mutant::Hooks, + selector: :load_config, + arguments: [config], + reaction: { return: hooks } + }, + { + receiver: world, + selector: :record, + arguments: [:setup_integration], + reaction: { yields: [] } + }, + { + receiver: Mutant::Integration, + selector: :setup, + arguments: [env_initial], + reaction: { return: integration_result } + } + ] + end + + include_examples 'bootstrap call' + end end diff --git a/spec/unit/mutant/cli_spec.rb b/spec/unit/mutant/cli_spec.rb index 9801472f9..4c00ec97b 100644 --- a/spec/unit/mutant/cli_spec.rb +++ b/spec/unit/mutant/cli_spec.rb @@ -750,14 +750,6 @@ def self.main_body include_context 'license validation' - context 'with invalid expressions' do - let(:arguments) { super() + [''] } - - it 'returns expected message' do - expect(apply).to eql(left('Expression: "" is invalid')) - end - end - before do allow(Mutant::Config).to receive(:load) do |**attributes| events << [:load_config, attributes.inspect] @@ -774,11 +766,21 @@ def self.main_body bootstrap_result end + allow(Mutant::Bootstrap).to receive(:call_test) do |env| + events << [:test_bootstrap, env.inspect] + bootstrap_result + end + allow(Mutant::Mutation::Runner).to receive(:call) do |env| events << [:runner, env.inspect] runner_result end + allow(Mutant::Test::Runner).to receive(:call) do |env| + events << [:test_runner, env.inspect] + runner_result + end + allow(kernel).to receive(:sleep) do |time| events << [:sleep, time] time @@ -827,53 +829,121 @@ def self.main_body end end + shared_examples 'with additional test arguments' do + context 'with additioanl arguments' do + let(:arguments) { super() << 'spec/unit' } + + let(:expected_cli_config) do + config = super() + + config.with(integration: config.integration.with(arguments: %w[spec/unit])) + end + + include_examples 'CLI run' + end + end + context 'environment test list' do include_context 'environment' let(:arguments) { %w[environment test list] } + let(:expected_events) do + [ + %i[ + record + config + ], + [ + :load_config, + { cli_config: expected_cli_config, world: world }.inspect + ], + [ + :test_bootstrap, + Mutant::Env.empty(world, bootstrap_config).inspect + ], + [ + :stdout, + :puts, + 'All tests in environment: 3' + ], + [ + :stdout, + :puts, + 'test-a' + ], + [ + :stdout, + :puts, + 'test-b' + ], + [ + :stdout, + :puts, + 'test-c' + ] + ] + end + context 'without additional arguments' do let(:expected_exit) { true } - let(:expected_events) do + include_examples 'CLI run' + end + + include_examples 'with additional test arguments' + end + + context 'environment test run' do + include_context 'environment' + + let(:arguments) { %w[environment test run] } + + let(:expected_events) do + [ + %i[ + record + config + ], [ - %i[ - record - config - ], - [ - :load_config, - { cli_config: expected_cli_config, world: world }.inspect - ], - [ - :bootstrap, - Mutant::Env.empty(world, bootstrap_config).inspect - ], - [ - :stdout, - :puts, - 'All tests in environment: 3' - ], - [ - :stdout, - :puts, - 'test-a' - ], - [ - :stdout, - :puts, - 'test-b' - ], - [ - :stdout, + :load_config, + { cli_config: expected_cli_config, world: world }.inspect + ], + [ + :test_bootstrap, + Mutant::Env.empty(world, bootstrap_config).inspect + ], + [ + :test_runner, + env.inspect + ] + ] + end + + context 'without additional arguments' do + let(:expected_exit) { true } + + context 'when tests fail' do + let(:env_result) { instance_double(Mutant::Result::Env, success?: false) } + let(:expected_exit) { false } + + let(:expected_events) do + super() << [ + :stderr, :puts, - 'test-c' + 'Test failures, exiting nonzero!' ] - ] + end + + include_examples 'CLI run' end - include_examples 'CLI run' + context 'when tests succeed' do + include_examples 'CLI run' + end end + + include_examples 'with additional test arguments' end context 'environment subject list --print-warnings' do diff --git a/spec/unit/mutant/env_spec.rb b/spec/unit/mutant/env_spec.rb index 89f6d0910..271fe2a8b 100644 --- a/spec/unit/mutant/env_spec.rb +++ b/spec/unit/mutant/env_spec.rb @@ -77,6 +77,37 @@ def isolation_success(value) ) end + describe '#run_test_index' do + def apply + subject.run_test_index(test_index) + end + + let(:test_index) { 0 } + + let(:test_result) do + Mutant::Result::Test.new( + passed: true, + runtime: 0.1, + output: '' + ) + end + + let(:raw_expectations) do + [ + { + receiver: integration, + selector: :call, + arguments: [[test_a]], + reaction: { return: test_result } + } + ] + end + + it 'returns env result' do + verify_events { expect(apply).to eql(test_result) } + end + end + describe '#cover_index' do let(:mutation_index) { 0 } @@ -157,13 +188,31 @@ def apply allow(hooks).to receive_messages(run: undefined) end - it 'dispatcehs expected hook' do + it 'dispatches expected hook' do apply expect(hooks).to have_received(:run).with(:mutation_worker_process_start, index: index) end end + describe '#emit_test_worker_process_start' do + let(:index) { 0 } + + def apply + subject.emit_test_worker_process_start(index: index) + end + + before do + allow(hooks).to receive_messages(run: undefined) + end + + it 'dispatches expected hook' do + apply + + expect(hooks).to have_received(:run).with(:test_worker_process_start, index: index) + end + end + describe '#selections' do def apply subject.selections diff --git a/spec/unit/mutant/integration/rspec_spec.rb b/spec/unit/mutant/integration/rspec_spec.rb index 444c9d8c1..71adf77e3 100644 --- a/spec/unit/mutant/integration/rspec_spec.rb +++ b/spec/unit/mutant/integration/rspec_spec.rb @@ -13,9 +13,11 @@ let(:expected_rspec_cli) { %w[--fail-fast spec] } let(:integration_arguments) { [] } + let(:rspec_configuration) { instance_double(RSpec::Core::Configuration) } let(:rspec_options) { instance_double(RSpec::Core::ConfigurationOptions) } let(:rspec_runner) { instance_double(RSpec::Core::Runner) } let(:world) { fake_world } + let(:rspec_is_quitting) { false } let(:example_a) do double( @@ -112,7 +114,8 @@ RSpec::Core::World, example_groups: example_groups, filtered_examples: filtered_examples, - ordered_example_groups: ordered_example_groups + ordered_example_groups: ordered_example_groups, + rspec_is_quitting: rspec_is_quitting ) end @@ -162,6 +165,13 @@ expect(RSpec).to receive_messages(world: rspec_world) + allow(rspec_configuration).to receive(:start_time=) + allow(rspec_configuration).to receive(:force) + allow(rspec_configuration).to receive(:reporter) + allow(rspec_configuration).to receive(:reset_reporter) + allow(rspec_runner).to receive_messages(configuration: rspec_configuration) + allow(world.time).to receive_messages(now: Time.at(10)) + allow(world.timer).to receive(:elapsed).and_return(2.0).and_yield allow(world.timer).to receive_messages(now: 1.0) end @@ -191,12 +201,16 @@ before do expect(rspec_runner).to receive(:setup) do |error, stdout| - expect(error).to be($stderr) - expect(stdout).to be($stdout) + expect(error).to be(world.stderr) + expect(stdout).to be(world.stdout) end end it { should be(object) } + + it 'freezes object' do + expect { subject }.to change { object.frozen? }.from(false).to(true) + end end describe '#call' do @@ -206,8 +220,8 @@ before do expect(rspec_runner).to receive(:setup) do |error, stdout| - expect(error).to be($stderr) - expect(stdout).to be($stdout) + expect(error).to be(world.stderr) + expect(stdout).to be(world.stdout) end allow(rspec_runner).to receive_messages(run_specs: exit_status) end @@ -219,6 +233,18 @@ expect(rspec_runner).to have_received(:run_specs).with(ordered_example_groups) end + it 'updates rspec start time' do + subject + + expect(rspec_configuration).to have_received(:start_time=).with(Time.at(8)) + end + + it 'resets reporter' do + subject + + expect(rspec_configuration).to have_received(:reset_reporter) + end + it 'modifies filtered examples to selection' do subject @@ -237,6 +263,7 @@ it 'should return failed result' do expect(subject).to eql( Mutant::Result::Test.new( + output: '', passed: false, runtime: 0.0 ) @@ -252,11 +279,34 @@ it 'should return passed result' do expect(subject).to eql( Mutant::Result::Test.new( + output: '', passed: true, runtime: 0.0 ) ) end end + + context 'on multiple calls' do + let(:exit_status) { 0 } + + let(:tests_initial) { all_tests.take(2) } + let(:tests_followup) { all_tests.drop(1).take(2) } + + def apply + object.setup + object.call(tests_initial) + object.call(tests_followup) + end + + it 'modifies filtered examples to selection' do + apply + + expect(filtered_examples).to eql( + root_example_group => [], + leaf_example_group => [example_b, example_c] + ) + end + end end end diff --git a/spec/unit/mutant/integration_spec.rb b/spec/unit/mutant/integration_spec.rb index cf55c257a..ee5521264 100644 --- a/spec/unit/mutant/integration_spec.rb +++ b/spec/unit/mutant/integration_spec.rb @@ -157,6 +157,7 @@ def apply it 'returns test result' do should eql( Mutant::Result::Test.new( + output: '', passed: true, runtime: 0.0 ) diff --git a/spec/unit/mutant/reporter/cli/printer/test/config_spec.rb b/spec/unit/mutant/reporter/cli/printer/test/config_spec.rb new file mode 100644 index 000000000..e8a3f0123 --- /dev/null +++ b/spec/unit/mutant/reporter/cli/printer/test/config_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +RSpec.describe Mutant::Reporter::CLI::Printer::Test::Config do + setup_shared_context + + context 'on absent jobs' do + let(:reportable) { config.with(jobs: nil) } + + describe '.call' do + it_reports(<<~'REPORT') + Fail-Fast: false + Integration: null + Jobs: auto + REPORT + end + end + + context 'on present jobs' do + let(:reportable) { config.with(jobs: 10) } + + describe '.call' do + it_reports(<<~'REPORT') + Fail-Fast: false + Integration: null + Jobs: 10 + REPORT + end + end + + context 'on absent integration' do + let(:reportable) { config } + + describe '.call' do + it_reports(<<~'REPORT') + Fail-Fast: false + Integration: null + Jobs: auto + REPORT + end + end + + context 'on present integration' do + let(:reportable) { config.with(integration: Mutant::Integration::Config::DEFAULT.with(name: 'foo')) } + + describe '.call' do + it_reports(<<~'REPORT') + Fail-Fast: false + Integration: foo + Jobs: auto + REPORT + end + end +end diff --git a/spec/unit/mutant/reporter/cli/printer/test/env_result_spec.rb b/spec/unit/mutant/reporter/cli/printer/test/env_result_spec.rb new file mode 100644 index 000000000..2a9fdb010 --- /dev/null +++ b/spec/unit/mutant/reporter/cli/printer/test/env_result_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +RSpec.describe Mutant::Reporter::CLI::Printer::Test::EnvResult do + setup_shared_context + + let(:reportable) do + Mutant::Result::TestEnv.new( + env: env, + runtime: 0.8, + test_results: [test_result_a, test_result_b] + ) + end + + let(:test_result_a) do + Mutant::Result::Test.new( + output: '', + passed: false, + runtime: 0.1 + ) + end + + let(:test_result_b) do + Mutant::Result::Test.new( + output: '', + passed: true, + runtime: 0.2 + ) + end + + describe '.call' do + it_reports <<~'STR' + + Test environment: + Fail-Fast: false + Integration: null + Jobs: auto + Tests: 2 + Test-Results: 2 + Test-Failed: 1 + Test-Success: 1 + Runtime: 0.80s + Testtime: 0.30s + Efficiency: 37.50% + STR + end +end diff --git a/spec/unit/mutant/reporter/cli/printer/test/result_spec.rb b/spec/unit/mutant/reporter/cli/printer/test/result_spec.rb new file mode 100644 index 000000000..4f29c67a8 --- /dev/null +++ b/spec/unit/mutant/reporter/cli/printer/test/result_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +RSpec.describe Mutant::Reporter::CLI::Printer::Test::Result do + setup_shared_context + + let(:reportable) do + Mutant::Result::Test.new( + output: '', + passed: false, + runtime: 0.1 + ) + end + + it_reports <<~'STR' + + STR +end diff --git a/spec/unit/mutant/reporter/cli/printer/test/status_progressive_spec.rb b/spec/unit/mutant/reporter/cli/printer/test/status_progressive_spec.rb new file mode 100644 index 000000000..fad48ed54 --- /dev/null +++ b/spec/unit/mutant/reporter/cli/printer/test/status_progressive_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +RSpec.describe Mutant::Reporter::CLI::Printer::Test::StatusProgressive do + setup_shared_context + + let(:reportable) { test_status } + let(:test_results) { [] } + + let(:test_env) do + Mutant::Result::TestEnv.new( + env: env, + runtime: 0.8, + test_results: test_results + ) + end + + let(:test_status) do + Mutant::Parallel::Status.new( + active_jobs: 1, + done: false, + payload: test_env + ) + end + + describe '.call' do + context 'with empty scheduler' do + it_reports <<~REPORT + progress: 00/02 failed: 0 runtime: 0.80s testtime: 0.00s tests/s: 0.00 + REPORT + end + + context 'with test results' do + let(:test_results) do + [ + Mutant::Result::Test.new( + output: '', + passed: false, + runtime: 0.5 + ) + ] + end + + it_reports <<~REPORT + progress: 01/02 failed: 1 runtime: 0.80s testtime: 0.50s tests/s: 1.25 + REPORT + end + end +end diff --git a/spec/unit/mutant/reporter/cli_spec.rb b/spec/unit/mutant/reporter/cli_spec.rb index a0c47d177..1f56d036a 100644 --- a/spec/unit/mutant/reporter/cli_spec.rb +++ b/spec/unit/mutant/reporter/cli_spec.rb @@ -84,6 +84,18 @@ def self.it_reports(expected_content) REPORT end + describe '#test_start' do + subject { object.test_start(env) } + + it_reports(<<~REPORT) + Test environment: + Fail-Fast: false + Integration: null + Jobs: auto + Tests: 2 + REPORT + end + describe '#report' do subject { object.report(env_result) } @@ -142,4 +154,58 @@ def self.it_reports(expected_content) end end end + + describe '#test_progress' do + subject { object.test_progress(test_status) } + + let(:tty?) { true } + + let(:test_env) do + Mutant::Result::TestEnv.new( + env: env, + runtime: 1.0, + test_results: [] + ) + end + + let(:test_status) do + Mutant::Parallel::Status.new( + active_jobs: 1, + done: false, + payload: test_env + ) + end + + # rubocop:disable Layout/LineLength + # rubocop:disable Style/StringConcatenation + it_reports Unparser::Color::GREEN.format('progress: 00/02 failed: 0 runtime: 1.00s testtime: 0.00s tests/s: 0.00') + "\n" + # rubocop:enable Style/StringConcatenation + # rubocop:enable Layout/LineLength + end + + describe '#test_report' do + subject { object.test_report(test_env) } + + let(:test_env) do + Mutant::Result::TestEnv.new( + env: env, + runtime: 1.0, + test_results: [] + ) + end + + it_reports <<~'STR' + Test environment: + Fail-Fast: false + Integration: null + Jobs: auto + Tests: 2 + Test-Results: 0 + Test-Failed: 0 + Test-Success: 0 + Runtime: 1.00s + Testtime: 0.00s + Efficiency: 0.00% + STR + end end diff --git a/spec/unit/mutant/result/test_env_spec.rb b/spec/unit/mutant/result/test_env_spec.rb new file mode 100644 index 000000000..9bd80ed70 --- /dev/null +++ b/spec/unit/mutant/result/test_env_spec.rb @@ -0,0 +1,159 @@ +# frozen_string_literal: true + +RSpec.describe Mutant::Result::TestEnv do + let(:object) do + described_class.new( + runtime: instance_double(Float), + env: env, + test_results: test_results + ) + end + + let(:env) do + instance_double( + Mutant::Env, + config: instance_double(Mutant::Config, fail_fast: fail_fast), + integration: integration + ) + end + + let(:integration) do + instance_double( + Mutant::Integration, + all_tests: [test_a, test_b] + ) + end + + let(:test_result) do + Mutant::Result::Test.new( + output: '', + passed: true, + runtime: 1.0 + ) + end + + let(:test_a) do + instance_double(Mutant::Test, :a) + end + + let(:test_b) do + instance_double(Mutant::Test, :b) + end + + let(:test_c) do + instance_double(Mutant::Test, :c) + end + + let(:fail_fast) { false } + let(:test_results) { [test_result] } + + describe '#success?' do + subject { object.success? } + + context 'when no test failed' do + it { should be(true) } + end + + context 'when a test failed' do + let(:test_result) { super().with(passed: false) } + + it { should be(false) } + end + end + + describe '#stop?' do + subject { object.stop? } + + context 'without fail fast' do + context 'on empty test results' do + let(:test_results) { [] } + + it { should be(false) } + end + + context 'on failed test' do + let(:test_result) { super().with(passed: false) } + + it { should be(false) } + end + + context 'on successful subject' do + it { should be(false) } + end + end + + context 'with fail fast' do + let(:fail_fast) { true } + + context 'on empty tests results' do + let(:test_results) { [] } + + it { should be(false) } + end + + context 'on failed subject' do + let(:test_result) { super().with(passed: false) } + + it { should be(true) } + end + + context 'on successful tests' do + it { should be(false) } + end + end + end + + describe '#testttime' do + def apply + object.testtime + end + + context 'on empty test results' do + let(:test_results) { [] } + + it 'returns 0' do + expect(apply).to be(0.0) + end + end + + context 'on multiple test results' do + let(:test_results) { [test_result, test_result.with(runtime: 2.2)] } + + it 'returns 0' do + expect(apply).to be(3.2) + end + end + end + + describe '#amount_test_success' do + let(:test_results) { [test_result, test_result.with(passed: false)] } + + def apply + object.amount_tests_success + end + + it 'returns expected value' do + expect(apply).to be(1) + end + end + + describe '#amount_tests' do + def apply + object.amount_tests + end + + it 'returns expected value' do + expect(apply).to be(2) + end + end + + describe '#amount_test_results' do + def apply + object.amount_test_results + end + + it 'returns expected value' do + expect(apply).to be(1) + end + end +end diff --git a/spec/unit/mutant/result/test_spec.rb b/spec/unit/mutant/result/test_spec.rb index e5bb86151..c79208929 100644 --- a/spec/unit/mutant/result/test_spec.rb +++ b/spec/unit/mutant/result/test_spec.rb @@ -4,6 +4,7 @@ describe '.new' do it 'returns expected attributes' do expect(described_class.instance.to_h).to eql( + output: '', passed: false, runtime: 0.0 ) diff --git a/spec/unit/mutant/test/runner/sink_spec.rb b/spec/unit/mutant/test/runner/sink_spec.rb new file mode 100644 index 000000000..c6b8543bf --- /dev/null +++ b/spec/unit/mutant/test/runner/sink_spec.rb @@ -0,0 +1,251 @@ +# frozen_string_literal: true + +describe Mutant::Test::Runner::Sink do + let(:object) { described_class.new(env: env) } + let(:fail_fast) { false } + + let(:env) do + instance_double( + Mutant::Env, + config: config, + world: + instance_double( + Mutant::World, + timer: timer, + stderr: stderr + ) + ) + end + + let(:stderr) do + instance_double(IO, :stderr) + end + + let(:config) do + instance_double(Mutant::Config, fail_fast: fail_fast) + end + + let(:timer) do + instance_double(Mutant::Timer) + end + + let(:test_result_a_raw) do + Mutant::Result::Test.new( + output: '', + passed: true, + runtime: 0.1 + ) + end + + let(:test_result_b_raw) do + Mutant::Result::Test.new( + output: '', + passed: true, + runtime: 0.2 + ) + end + + let(:test_result_a) { test_result_a_raw.with(output: test_response_a.log) } + let(:test_result_b) { test_result_b_raw.with(output: test_response_b.log) } + + let(:test_response_a) do + Mutant::Parallel::Response.new( + error: nil, + result: test_result_a_raw, + log: '' + ) + end + + let(:test_response_b) do + Mutant::Parallel::Response.new( + error: nil, + result: test_result_b_raw, + log: '' + ) + end + + before do + allow(timer).to receive(:now).and_return(0.5, 2.0) + end + + shared_context 'one result' do + before do + object.response(test_response_a) + end + end + + shared_context 'two results' do + before do + object.response(test_response_a) + object.response(test_response_b) + end + end + + describe '#response' do + subject { object.response(test_response_a) } + + context 'on success' do + it 'aggregates results in #status' do + subject + object.response(test_response_b) + expect(object.status).to eql( + Mutant::Result::TestEnv.new( + env: env, + runtime: 1.5, + test_results: [test_result_a, test_result_b] + ) + ) + end + + it_should_behave_like 'a command method' + end + + context 'on error' do + before do + allow(stderr).to receive(:puts) + end + + it 'aggregates results in #status' do + subject + + expect do + object.response( + Mutant::Parallel::Response.new( + error: EOFError, + log: 'some log', + result: nil + ) + ) + end.to raise_error(EOFError) + + expect(stderr).to have_received(:puts).with('some log') + end + + it_should_behave_like 'a command method' + end + end + + describe '#status' do + subject { object.status } + + context 'no results' do + let(:expected_status) do + Mutant::Result::TestEnv.new( + env: env, + runtime: 1.5, + test_results: [] + ) + end + + it { should eql(expected_status) } + end + + context 'one result' do + include_context 'one result' + + let(:expected_status) do + Mutant::Result::TestEnv.new( + env: env, + runtime: 1.5, + test_results: [test_result_a] + ) + end + + it { should eql(expected_status) } + end + + context 'two results' do + include_context 'two results' + + let(:expected_status) do + Mutant::Result::TestEnv.new( + env: env, + runtime: 1.5, + test_results: [test_result_a, test_result_b] + ) + end + + it { should eql(expected_status) } + end + end + + describe '#stop?' do + subject { object.stop? } + + context 'without fail fast' do + context 'no results' do + it { should be(false) } + end + + context 'one result' do + include_context 'one result' + + context 'when result is successful' do + it { should be(false) } + end + + context 'when result failed' do + it { should be(false) } + end + end + + context 'two results' do + include_context 'two results' + + context 'when results are successful' do + it { should be(false) } + end + + context 'when first result is unsuccessful' do + it { should be(false) } + end + + context 'when second result is unsuccessful' do + it { should be(false) } + end + end + end + + context 'with fail fast' do + let(:fail_fast) { true } + + context 'no results' do + it { should be(false) } + end + + context 'one result' do + include_context 'one result' + + context 'when result is successful' do + it { should be(false) } + end + + context 'when result failed' do + let(:test_result_a_raw) { super().with(passed: false) } + + it { should be(true) } + end + end + + context 'two results' do + include_context 'two results' + + context 'when results are successful' do + it { should be(false) } + end + + context 'when first result is unsuccessful' do + let(:test_result_a_raw) { super().with(passed: false) } + + it { should be(true) } + end + + context 'when second result is unsuccessful' do + let(:test_result_b_raw) { super().with(passed: false) } + + it { should be(true) } + end + end + end + end +end diff --git a/spec/unit/mutant/test/runner_spec.rb b/spec/unit/mutant/test/runner_spec.rb new file mode 100644 index 000000000..e35124e48 --- /dev/null +++ b/spec/unit/mutant/test/runner_spec.rb @@ -0,0 +1,238 @@ +# frozen_string_literal: true + +RSpec.describe Mutant::Test::Runner do + describe '.call' do + let(:block) { instance_double(Proc) } + let(:delay) { instance_double(Float) } + let(:driver) { instance_double(Mutant::Parallel::Driver) } + let(:emit_test_worker_process_start) { instance_double(Proc) } + let(:env_result) { instance_double(Mutant::Result::Env) } + let(:reporter) { instance_double(Mutant::Reporter, delay: delay) } + let(:timer) { instance_double(Mutant::Timer) } + let(:world) { instance_double(Mutant::World) } + + let(:env) do + instance_double( + Mutant::Env, + config: config, + integration: integration, + mutations: [instance_double(Mutant::Mutation)], + world: world + ) + end + + let(:integration) do + instance_double( + Mutant::Integration, + all_tests: all_tests + ) + end + + let(:all_tests) do + [ + double('integration test A'), + double('integration test B') + ] + end + + let(:config) do + instance_double( + Mutant::Config, + jobs: 1, + reporter: reporter + ) + end + + let(:status_a) do + instance_double( + Mutant::Parallel::Status, + done?: false + ) + end + + let(:status_b) do + instance_double( + Mutant::Parallel::Status, + done?: true, + payload: env_result + ) + end + + let(:parallel_config) do + Mutant::Parallel::Config.new( + block: block, + jobs: 1, + on_process_start: emit_test_worker_process_start, + process_name: 'mutant-test-runner-process', + sink: described_class::Sink.new(env: env), + source: Mutant::Parallel::Source::Array.new(jobs: all_tests.each_index.to_a), + thread_name: 'mutant-test-runner-thread', + timeout: nil + ) + end + + before do + allow(world).to receive_messages(timer: timer) + allow(world.timer).to receive_messages(now: 1.0) + end + + def apply + described_class.call(env) + end + + context 'when not stopped' do + let(:raw_expectations) do + [ + { + receiver: reporter, + selector: :test_start, + arguments: [env] + }, + { + receiver: env, + selector: :record, + arguments: [:tests], + reaction: { yields: [] } + }, + { + receiver: env, + selector: :method, + arguments: [:run_test_index], + reaction: { return: block } + }, + { + receiver: env, + selector: :method, + arguments: [:emit_test_worker_process_start], + reaction: { return: emit_test_worker_process_start } + }, + { + receiver: Mutant::Parallel, + selector: :async, + arguments: [{ world: world, config: parallel_config }], + reaction: { return: driver } + }, + { + receiver: Signal, + selector: :trap, + arguments: ['INT'] + }, + { + receiver: driver, + selector: :wait_timeout, + arguments: [delay], + reaction: { return: status_a } + }, + { + receiver: reporter, + selector: :test_progress, + arguments: [status_a] + }, + { + receiver: driver, + selector: :wait_timeout, + arguments: [delay], + reaction: { return: status_b } + }, + { + receiver: env, + selector: :record, + arguments: [:report], + reaction: { yields: [] } + }, + { + receiver: reporter, + selector: :test_report, + arguments: [env_result] + } + ] + end + + before do + allow(driver).to receive_messages(stop: driver) + end + + it 'returns env result' do + verify_events { expect(apply).to eql(Mutant::Either::Right.new(env_result)) } + end + end + + context 'when stopped' do + let(:raw_expectations) do + [ + { + receiver: reporter, + selector: :test_start, + arguments: [env] + }, + { + receiver: env, + selector: :record, + arguments: [:tests], + reaction: { yields: [] } + }, + { + receiver: env, + selector: :method, + arguments: [:run_test_index], + reaction: { return: block } + }, + { + receiver: env, + selector: :method, + arguments: [:emit_test_worker_process_start], + reaction: { return: emit_test_worker_process_start } + }, + { + receiver: Mutant::Parallel, + selector: :async, + arguments: [{ world: world, config: parallel_config }], + reaction: { return: driver } + }, + { + receiver: Signal, + selector: :trap, + arguments: ['INT'], + reaction: { yields: [] } + }, + { + receiver: driver, + selector: :stop + }, + { + receiver: driver, + selector: :wait_timeout, + arguments: [delay], + reaction: { return: status_a } + }, + { + receiver: reporter, + selector: :test_progress, + arguments: [status_a] + }, + { + receiver: driver, + selector: :wait_timeout, + arguments: [delay], + reaction: { return: status_b } + }, + { + receiver: env, + selector: :record, + arguments: [:report], + reaction: { yields: [] } + }, + { + receiver: reporter, + selector: :test_report, + arguments: [env_result] + } + ] + end + + it 'returns env result' do + verify_events { expect(apply).to eql(Mutant::Either::Right.new(env_result)) } + end + end + end +end diff --git a/spec/unit/mutant/timer_spec.rb b/spec/unit/mutant/timer_spec.rb index dfe370f26..241885965 100644 --- a/spec/unit/mutant/timer_spec.rb +++ b/spec/unit/mutant/timer_spec.rb @@ -4,7 +4,7 @@ let(:events) { [] } let(:object) { described_class.new(process: process) } let(:process) { class_double(Process) } - let(:times) { [1.0, 2.0] } + let(:times) { [0.5, 2.0] } before do allow(process).to receive(:clock_gettime) do |argument| @@ -22,7 +22,7 @@ def apply end it 'returns current monotonic time' do - expect(apply).to be(1.0) + expect(apply).to be(0.5) expect(apply).to be(2.0) end @@ -33,4 +33,22 @@ def apply .to(%i[clock_gettime]) end end + + describe '#elapsed' do + let(:executions) { [] } + + def apply + object.elapsed do + executions << nil + end + end + + it 'executes the block' do + expect { apply }.to change { executions }.from([]).to([nil]) + end + + it 'returns execution time' do + expect(apply).to eql(1.5) + end + end end