diff --git a/.rubocop-bundler.yml b/.rubocop-bundler.yml index 31122b06..9ae7ace2 100644 --- a/.rubocop-bundler.yml +++ b/.rubocop-bundler.yml @@ -18,6 +18,7 @@ Lint/UnusedMethodArgument: Lint/UriEscapeUnescape: Enabled: true + # Style Layout/EndAlignment: @@ -91,10 +92,7 @@ Style/SpecialGlobalVars: Enabled: false Naming/VariableNumber: - EnforcedStyle: "snake_case" - AllowedIdentifiers: - - sha256 - - capture3 + EnforcedStyle: 'snake_case' Naming/MemoizedInstanceVariableName: Enabled: false diff --git a/Gemfile b/Gemfile index 67308ab9..7bf66dc0 100644 --- a/Gemfile +++ b/Gemfile @@ -22,6 +22,5 @@ group :linting do end group :test do - gem "gem_server_conformance", "~> 0.1.4" gem "mock_redis" end diff --git a/docs/gemstash-configuration.5.md b/docs/gemstash-configuration.5.md index 36d4ec04..a7590023 100644 --- a/docs/gemstash-configuration.5.md +++ b/docs/gemstash-configuration.5.md @@ -221,10 +221,10 @@ Boolean values `true` or `false` `:fetch_timeout` This is the number of seconds to allow for fetching a gem from upstream. -It covers establishing the connection and receiving the response. -Fetching gems over a slow connection may cause timeout errors. If you -experience timeout errors, you may want to increase this value. The -default is `20` seconds. +It covers establishing the connection and receiving the response. Fetching +gems over a slow connection may cause timeout errors. If you experience +timeout errors, you may want to increase this value. The default is `20` +seconds. ## Default value @@ -239,10 +239,10 @@ Integer value with a minimum of `1` `:open_timeout` The timeout setting for opening the connection to an upstream gem -server. On high-latency networks, even establishing the connection to an -upstream gem server can take a while. If you experience connection -failures instead of timeout errors, you may want to increase this value. -The default is `2` seconds. +server. On high-latency networks, even establishing the connection +to an upstream gem server can take a while. If you experience +connection failures instead of timeout errors, you may want to +increase this value. The default is `2` seconds. ## Default value diff --git a/gemstash.gemspec b/gemstash.gemspec index 3c7075a0..dc630fd1 100644 --- a/gemstash.gemspec +++ b/gemstash.gemspec @@ -32,7 +32,6 @@ you push your own private gems as well." spec.required_ruby_version = ">= 3.1" spec.add_runtime_dependency "activesupport", ">= 4.2", "< 8" - spec.add_runtime_dependency "compact_index", "~> 0.15.0" spec.add_runtime_dependency "dalli", ">= 3.2.3", "< 4" spec.add_runtime_dependency "faraday", ">= 1", "< 3" spec.add_runtime_dependency "faraday_middleware", "~> 1.0" diff --git a/lib/gemstash.rb b/lib/gemstash.rb index 75939d76..d56b31ab 100644 --- a/lib/gemstash.rb +++ b/lib/gemstash.rb @@ -7,7 +7,6 @@ module Gemstash autoload :DB, "gemstash/db" autoload :Cache, "gemstash/cache" autoload :CLI, "gemstash/cli" - autoload :CompactIndexBuilder, "gemstash/compact_index_builder" autoload :Configuration, "gemstash/configuration" autoload :Dependencies, "gemstash/dependencies" autoload :Env, "gemstash/env" diff --git a/lib/gemstash/cache.rb b/lib/gemstash/cache.rb index a99bf0eb..87e64b15 100644 --- a/lib/gemstash/cache.rb +++ b/lib/gemstash/cache.rb @@ -43,10 +43,7 @@ def set_dependency(scope, gem, value) def invalidate_gem(scope, gem) @client.delete("deps/v1/#{scope}/#{gem}") - if scope == "private" - Gemstash::SpecsBuilder.invalidate_stored - Gemstash::CompactIndexBuilder.invalidate_stored(gem) - end + Gemstash::SpecsBuilder.invalidate_stored if scope == "private" end end diff --git a/lib/gemstash/cli/info.rb b/lib/gemstash/cli/info.rb index 5ee921d9..100dcfe2 100644 --- a/lib/gemstash/cli/info.rb +++ b/lib/gemstash/cli/info.rb @@ -12,9 +12,6 @@ class Info < Gemstash::CLI::Base def run prepare list_config - - # Gemstash::DB - # Gemstash::Env.current.db.dump_schema_migration(same_db: true) end private diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb deleted file mode 100644 index cef36ad2..00000000 --- a/lib/gemstash/compact_index_builder.rb +++ /dev/null @@ -1,257 +0,0 @@ -# frozen_string_literal: true - -require "active_support/core_ext/string/filters" -require "compact_index" -require "gemstash" -require "stringio" -require "zlib" - -module Gemstash - # Comment - class CompactIndexBuilder - include Gemstash::Env::Helper - attr_reader :result - - def self.serve(app, ...) - app.content_type "text/plain; charset=utf-8" - body = new(app.auth, ...).serve - app.etag Digest::MD5.hexdigest(body) - sha256 = Digest::SHA256.base64digest(body) - app.headers "Accept-Ranges" => "bytes", "Digest" => "sha-256=#{sha256}", "Repr-Digest" => "sha-256=:#{sha256}:", - "Content-Length" => body.bytesize.to_s - body - end - - def self.invalidate_stored(name) - storage = Gemstash::Storage.for("private").for("compact_index") - storage.resource("names").delete(:names) - storage.resource("versions").delete(:versions) - storage.resource("info/#{name}").delete(:info) - end - - def initialize(auth) - @auth = auth - end - - def serve - check_auth if gemstash_env.config[:protected_fetch] - fetch_from_storage - return result if result - - build_result - store_result - result - end - - private - - def storage - @storage ||= Gemstash::Storage.for("private").for("compact_index") - end - - def fetch_from_storage - resource = fetch_resource - return unless resource.exist?(key) - - @result = resource.load(key).content(key) - rescue StandardError - # On the off-chance of a race condition between specs.exist? and specs.load - @result = nil - end - - def store_result - fetch_resource.save(key => @result) - end - - def check_auth - @auth.check("fetch") - end - - # Comment - class Versions < CompactIndexBuilder - def fetch_resource - storage.resource("versions") - end - - def build_result(force_rebuild: false) - resource = fetch_resource - base = !force_rebuild && resource.exist?("versions.list") && resource.content("versions.list") - Tempfile.create("versions.list") do |file| - versions_file = CompactIndex::VersionsFile.new(file.path) - if base - file.write(base) - file.close - @result = versions_file.contents( - compact_index_versions(versions_file.updated_at.to_time) - ) - else - ts = Time.now.iso8601 - versions_file.create( - compact_index_public_versions(ts), ts - ) - @result = file.read - resource.save("versions.list" => @result) - end - end - end - - private - - def compact_index_versions(date) - all_versions = Sequel::Model.db[<<~SQL.squish, date, date].to_a - SELECT r.name as name, v.created_at as date, v.info_checksum as info_checksum, v.number as number, v.platform as platform - FROM rubygems AS r, versions AS v - WHERE v.rubygem_id = r.id AND - v.created_at > ? - - UNION ALL - - SELECT r.name as name, v.yanked_at as date, v.yanked_info_checksum as info_checksum, '-'||v.number as number, v.platform as platform - FROM rubygems AS r, versions AS v - WHERE v.rubygem_id = r.id AND - v.indexed is false AND - v.yanked_at > ? - - ORDER BY date, number, platform, name - SQL - - # not ordered correctly in sqlite for some reason - all_versions.sort_by! {|v| [v[:date], v[:number], v[:platform], v[:name]] } - map_gem_versions(all_versions.map {|v| [v[:name], [v]] }) - end - - def compact_index_public_versions(date) - all_versions = Sequel::Model.db[<<~SQL.squish, date, date].to_a - SELECT r.name, v.indexed, COALESCE(v.yanked_at, v.created_at) as stamp, - COALESCE(v.yanked_info_checksum, v.info_checksum) as info_checksum, v.number, v.platform - FROM rubygems AS r, versions AS v - WHERE v.rubygem_id = r.id AND - (v.created_at <= ? OR v.yanked_at <= ?) - ORDER BY name, COALESCE(v.yanked_at, v.created_at), number, platform - SQL - - versions_by_gem = all_versions.group_by {|row| row[:name] } - versions_by_gem.each_value do |versions| - info_checksum = versions.last[:info_checksum] - versions.select! {|v| v[:indexed] == true } - # Set all versions' info_checksum to work around https://github.com/bundler/compact_index/pull/20 - versions.each {|v| v[:info_checksum] = info_checksum } - end - - map_gem_versions(versions_by_gem) - end - - def map_gem_versions(versions_by_gem) - versions_by_gem.map do |name, versions| - CompactIndex::Gem.new( - name, - versions.map do |row| - CompactIndex::GemVersion.new( - row[:number], - row[:platform], - nil, # sha256 - row[:info_checksum], - nil, # dependencies - nil, # version.required_ruby_version, - nil, # version.required_rubygems_version - ) - end - ) - end - end - - def key - :versions - end - end - - # Comment - class Info < CompactIndexBuilder - def initialize(auth, name) - super(auth) - @name = name - end - - def fetch_resource - storage.resource("info/#{@name}") - end - - def build_result - @result = CompactIndex.info(requirements_and_dependencies) - end - - private - - def requirements_and_dependencies - group_by_columns = "number, platform, sha256, info_checksum, required_ruby_version, required_rubygems_version, versions.created_at" - - dep_req_agg = "string_agg(dependencies.requirements, '@' ORDER BY dependencies.rubygem_name, dependencies.id) as dep_req_agg" - - dep_name_agg = "string_agg(dependencies.rubygem_name, ',' ORDER BY dependencies.rubygem_name) AS dep_name_agg" - - DB::Rubygem.db[<<~SQL.squish, @name]. - SELECT #{group_by_columns}, #{dep_req_agg}, #{dep_name_agg} - FROM rubygems - LEFT JOIN versions ON versions.rubygem_id = rubygems.id - LEFT JOIN dependencies ON dependencies.version_id = versions.id - WHERE rubygems.name = ? AND versions.indexed = true - GROUP BY #{group_by_columns} - ORDER BY versions.created_at, number, platform, dep_name_agg - SQL - map do |row| - reqs = row[:dep_req_agg]&.split("@") - dep_names = row[:dep_name_agg]&.split(",") - - raise "Dependencies and requirements are not the same size:\n reqs: #{reqs.inspect}\n dep_names: #{dep_names.inspect}\n row: #{row.inspect}" if dep_names&.size != reqs&.size - - deps = [] - if reqs - dep_names.zip(reqs).each do |name, req| - deps << CompactIndex::Dependency.new(name, req) - end - end - - CompactIndex::GemVersion.new( - row[:number], - row[:platform], - row[:sha256], - nil, # info_checksum - deps, - row[:required_ruby_version], - row[:required_rubygems_version] - ) - end - end - - def key - :info - end - end - - # Comment - class Names < CompactIndexBuilder - def fetch_resource - storage.resource("names") - end - - def build_result - names = DB::Rubygem.db[<<~SQL.squish].map {|row| row[:name] } - SELECT name - FROM rubygems - LEFT JOIN versions ON versions.rubygem_id = rubygems.id - WHERE versions.indexed = true - GROUP BY name - HAVING COUNT(versions.id) > 0 - ORDER BY name - SQL - @result = CompactIndex.names(names).encode("UTF-8") - end - - private - - def key - :names - end - end - end -end diff --git a/lib/gemstash/db.rb b/lib/gemstash/db.rb index 47789d73..e28743f2 100644 --- a/lib/gemstash/db.rb +++ b/lib/gemstash/db.rb @@ -10,7 +10,6 @@ module DB Sequel::Model.db = Gemstash::Env.current.db Sequel::Model.raise_on_save_failure = true Sequel::Model.plugin :timestamps, update_on_create: true - Sequel::Model.db.extension :schema_dumper autoload :Authorization, "gemstash/db/authorization" autoload :CachedRubygem, "gemstash/db/cached_rubygem" autoload :Dependency, "gemstash/db/dependency" diff --git a/lib/gemstash/db/version.rb b/lib/gemstash/db/version.rb index 032c03fa..3f42c3a5 100644 --- a/lib/gemstash/db/version.rb +++ b/lib/gemstash/db/version.rb @@ -9,11 +9,7 @@ class Version < Sequel::Model many_to_one :rubygem def deindex - info = Gemstash::CompactIndexBuilder::Info.new(nil, rubygem.name).tap(&:build_result).result - prefix = number.dup - prefix << "-#{platform}" if platform != "ruby" - info.gsub!(/^#{Regexp.escape(prefix)} .*?\n/, "") - update(indexed: false, yanked_at: Time.now.utc, yanked_info_checksum: Digest::MD5.hexdigest(info)) + update(indexed: false) end def reindex @@ -32,10 +28,8 @@ def self.slug(params) end def self.for_spec_collection(prerelease: false, latest: false) - versions = where(indexed: true, prerelease: prerelease).association_join(:rubygem). - order { [rubygem[:name], platform.desc] } - versions = select_latest(versions) if latest - order_for_spec_collection(versions) + versions = where(indexed: true, prerelease: prerelease).association_join(:rubygem) + latest ? select_latest(versions) : versions end def self.select_latest(versions) @@ -46,17 +40,6 @@ def self.select_latest(versions) map {|gem_versions| gem_versions.max_by {|version| Gem::Version.new(version.number) } } end - def self.order_for_spec_collection(versions) - versions.to_enum.group_by(&:rubygem_id).flat_map do |_, gem_versions| - versions = Hash.new {|h, k| h[k] = Gem::Version.new(k) } - numbers = gem_versions.map {|version| versions[version.number] } - numbers.sort! - gem_versions.sort_by do |version| - [-numbers.index(version.number), version.platform] - end.reverse - end - end - def self.find_by_spec(gem_id, spec) self[rubygem_id: gem_id, number: spec.version.to_s, @@ -71,33 +54,14 @@ def self.find_by_full_name(full_name) self[full_name: "#{full_name}-ruby"] end - def self.insert_by_spec(gem_id, spec, sha256:) + def self.insert_by_spec(gem_id, spec) gem_name = Gemstash::DB::Rubygem[gem_id].name - info = Gemstash::CompactIndexBuilder::Info.new(nil, gem_name).tap(&:build_result).result - info << CompactIndex::GemVersion.new( - spec.version.to_s, - spec.platform.to_s, - sha256, - nil, # info_checksum - spec.runtime_dependencies.map do |dep| - requirements = dep.requirement.requirements - requirements = requirements.map {|r| "#{r.first} #{r.last}" } - requirements = requirements.join(", ") - CompactIndex::Dependency.new(dep.name, requirements) - end, - spec.required_ruby_version&.to_s, - spec.required_rubygems_version&.to_s - ).to_line << "\n" new(rubygem_id: gem_id, number: spec.version.to_s, platform: spec.platform.to_s, full_name: "#{gem_name}-#{spec.version}-#{spec.platform}", storage_id: spec.full_name, indexed: true, - sha256: sha256, - info_checksum: Digest::MD5.hexdigest(info), - required_ruby_version: spec.required_ruby_version&.to_s, - required_rubygems_version: spec.required_rubygems_version&.to_s, prerelease: spec.version.prerelease?).tap(&:save).id end end diff --git a/lib/gemstash/gem_pusher.rb b/lib/gemstash/gem_pusher.rb index 01a1a7aa..8cc4f8a2 100644 --- a/lib/gemstash/gem_pusher.rb +++ b/lib/gemstash/gem_pusher.rb @@ -79,7 +79,7 @@ def save_to_database raise ExistingVersionError, "Cannot push to an existing version!" if existing && existing.indexed raise YankedVersionError, "Cannot push to a yanked version!" if existing && !existing.indexed - version_id = Gemstash::DB::Version.insert_by_spec(gem_id, spec, sha256: Digest::SHA256.hexdigest(@content)) + version_id = Gemstash::DB::Version.insert_by_spec(gem_id, spec) Gemstash::DB::Dependency.insert_by_spec(version_id, spec) end end diff --git a/lib/gemstash/gem_source.rb b/lib/gemstash/gem_source.rb index 48c02c07..56cce779 100644 --- a/lib/gemstash/gem_source.rb +++ b/lib/gemstash/gem_source.rb @@ -29,8 +29,7 @@ class Base include Gemstash::Logging def_delegators :@app, :cache_control, :content_type, :env, :halt, - :headers, :http_client_for, :params, :redirect, :request, - :etag + :headers, :http_client_for, :params, :redirect, :request def initialize(app) @app = app diff --git a/lib/gemstash/gem_source/private_source.rb b/lib/gemstash/gem_source/private_source.rb index 606e8cab..025312d0 100644 --- a/lib/gemstash/gem_source/private_source.rb +++ b/lib/gemstash/gem_source/private_source.rb @@ -43,17 +43,15 @@ def serve_remove_spec_json end def serve_names - protected(CompactIndexBuilder::Names) + halt 403, "Not yet supported" end def serve_versions - protected(CompactIndexBuilder::Versions) + halt 404, "Not yet supported" end def serve_info(name) - halt(404, { "Content-Type" => "text/plain; charset=utf-8" }, "This gem could not be found") unless DB::Rubygem.where(name: name).limit(1).count > 0 - - protected(CompactIndexBuilder::Info, name) + halt 403, "Not yet supported" end def serve_marshal(id) @@ -98,8 +96,8 @@ def serve_prerelease_specs private - def protected(servable, ...) - authorization.protect(self) { servable.serve(self, ...) } + def protected(servable) + authorization.protect(self) { servable.serve(self) } end def authorization diff --git a/lib/gemstash/gem_yanker.rb b/lib/gemstash/gem_yanker.rb index dd9824d4..c27ecf5f 100644 --- a/lib/gemstash/gem_yanker.rb +++ b/lib/gemstash/gem_yanker.rb @@ -66,7 +66,6 @@ def update_database def invalidate_cache gemstash_env.cache.invalidate_gem("private", @gem_name) - Gemstash::CompactIndexBuilder.invalidate_stored(@gem_name) end end end diff --git a/lib/gemstash/migrations/06_compact_index.rb b/lib/gemstash/migrations/06_compact_index.rb deleted file mode 100644 index d55e0d34..00000000 --- a/lib/gemstash/migrations/06_compact_index.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -Sequel.migration do - change do - alter_table :versions do # TODO: backfill info_checksum, sha256, required_ruby_version, required_rubygems_version - add_column :info_checksum, String, :size => 40 - add_column :yanked_info_checksum, String, :size => 40 - add_column :yanked_at, DateTime, :null => true - add_column :sha256, String, :size => 64 - add_column :required_ruby_version, String, :size => 255 - add_column :required_rubygems_version, String, :size => 255 - end - end -end diff --git a/lib/gemstash/web.rb b/lib/gemstash/web.rb index bff5e95f..ae2428c8 100644 --- a/lib/gemstash/web.rb +++ b/lib/gemstash/web.rb @@ -27,19 +27,12 @@ def http_client_for(server_url) not_found do status 404 - return body response.body if response.body && !response.body.empty? - body JSON.dump("error" => "Not found", "code" => 404) end error GemPusher::ExistingVersionError do - status 409 - body JSON.dump("error" => "Version already exists", "code" => 409) - end - - error Gemstash::GemYanker::UnknownGemError, Gemstash::GemYanker::UnknownVersionError do |e| - status 404 - body JSON.dump("error" => e.message, "code" => 404) + status 422 + body JSON.dump("error" => "Version already exists", "code" => 422) end get "/" do diff --git a/schema.rb b/schema.rb deleted file mode 100644 index 76b704fa..00000000 --- a/schema.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: true - -Sequel.migration do - change do - create_table(:authorizations) do - primary_key :id - column :auth_key, "varchar(191)", :null => false - column :permissions, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - column :name, "varchar(191)" - - index [:auth_key], :unique => true - index [:name], :unique => true - end - - create_table(:cached_rubygems) do - primary_key :id - column :upstream_id, "INTEGER", :null => false - column :name, "varchar(191)", :null => false - column :resource_type, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - - index [:name] - index %i[upstream_id resource_type name], :unique => true - end - - create_table(:dependencies) do - primary_key :id - column :version_id, "INTEGER", :null => false - column :rubygem_name, "varchar(191)", :null => false - column :requirements, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - - index [:rubygem_name] - index [:version_id] - end - - create_table(:health_tests) do - primary_key :id - column :string, "varchar(255)" - end - - create_table(:rubygems) do - primary_key :id - column :name, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - - index [:name], :unique => true - end - - create_table(:schema_info) do - column :version, "INTEGER", :default => 0, :null => false - end - - create_table(:upstreams) do - primary_key :id - column :uri, "varchar(191)", :null => false - column :host_id, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - - index [:host_id], :unique => true - index [:uri], :unique => true - end - - create_table(:versions) do - primary_key :id - column :rubygem_id, "INTEGER", :null => false - column :storage_id, "varchar(191)", :null => false - column :number, "varchar(191)", :null => false - column :platform, "varchar(191)", :null => false - column :full_name, "varchar(191)", :null => false - column :indexed, "boolean", :default => true, :null => false - column :prerelease, "boolean", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - column :info_checksum, "varchar(40)" - column :sha256, "varchar(64)" - column :required_ruby_version, "varchar(191)" - column :required_rubygems_version, "varchar(191)" - - index [:full_name], :unique => true - index [:indexed] - index %i[indexed prerelease] - index [:number] - index %i[rubygem_id number platform], :unique => true - index [:storage_id], :unique => true - end - end -end diff --git a/spec/gemstash/compact_index_builder_spec.rb b/spec/gemstash/compact_index_builder_spec.rb deleted file mode 100644 index 729c9687..00000000 --- a/spec/gemstash/compact_index_builder_spec.rb +++ /dev/null @@ -1,233 +0,0 @@ -# frozen_string_literal: true - -require "spec_helper" - -RSpec.describe Gemstash::CompactIndexBuilder do - let(:auth) { Gemstash::ApiKeyAuthorization.new(auth_key) } - let(:auth_with_invalid_auth_key) { Gemstash::ApiKeyAuthorization.new(invalid_auth_key) } - let(:auth_without_permission) { Gemstash::ApiKeyAuthorization.new(auth_key_without_permission) } - let(:auth_key) { "auth-key" } - let(:invalid_auth_key) { "invalid-auth-key" } - let(:auth_key_without_permission) { "auth-key-without-permission" } - - before do - Gemstash::Authorization.authorize(auth_key, "all") - Gemstash::Authorization.authorize(auth_key_without_permission, ["push"]) - allow(Time).to receive(:now).and_return(Time.new(1990, in: "UTC")) - end - - context "with no private gems" do - it "returns empty versions" do - result = described_class::Versions.new(auth).serve - expect(result).to eq(<<~VERSIONS) - created_at: 1990-01-01T00:00:00Z - --- - VERSIONS - end - - it "returns empty names" do - result = described_class::Names.new(auth).serve - expect(result).to eq(<<~NAMES) - --- - - NAMES - end - - it "returns 404 for info" do - result = described_class::Info.new(auth, "something").serve - expect(result).to eq(<<~INFO) - --- - INFO - end - end - - context "with some private gems" do - before do - gem_id = insert_rubygem("example") - insert_version(gem_id, "0.0.1") - insert_version(gem_id, "0.0.2") - insert_version(gem_id, "0.0.2", platform: "java") - gem_id = insert_rubygem("other-example") - insert_version(gem_id, "0.1.0") - end - - it "returns versions" do - Gemstash::CompactIndexBuilder::Versions.new(auth).serve - result = Gemstash::CompactIndexBuilder::Versions.new(auth).serve - expect(result).to eq <<~VERSIONS - created_at: 1990-01-01T00:00:00Z - --- - example 0.0.1 1e6fae87f01f5e16ef83205a1a12646c - example 0.0.2-java 02fd7dc9130d37b37fb21e7b3c870ada - example 0.0.2 be6954d4377b5262bee5bf4018e6227f - other-example 0.1.0 ff0722a59d13124677a2edd0da268bd1 - VERSIONS - end - - it "returns info" do - result = Gemstash::CompactIndexBuilder::Info.new(auth, "example").serve - expect(result).to eq <<~INFO - --- - 0.0.1 |checksum:786b0634cdc056d7fbb027802bbd6e13a6056143adc69047db6aded595754554 - 0.0.2-java |checksum:fd67cdfe89ddbd20e499efccffdc828384acf01e4a3068dbf414150ad7515f5f - 0.0.2 |checksum:bfad311d42610c3d1be9d18064f6e688152560e75c716ff63abb5cbb29673f63 - INFO - end - - it "returns names" do - result = Gemstash::CompactIndexBuilder::Names.new(auth).serve - expect(result).to eq <<~NAMES - --- - example - other-example - NAMES - end - end - - context "with some yanked gems" do - let(:expected_specs) do - [["example", Gem::Version.new("0.0.1"), "ruby"], - ["example", Gem::Version.new("0.0.2"), "ruby"], - ["example", Gem::Version.new("0.0.2"), "java"], - ["other-example", Gem::Version.new("0.1.0"), "ruby"]] - end - - let(:expected_latest_specs) do - [["example", Gem::Version.new("0.0.2"), "ruby"], - ["example", Gem::Version.new("0.0.2"), "java"], - ["other-example", Gem::Version.new("0.1.0"), "ruby"]] - end - - let(:expected_prerelease_specs) do - [["example", Gem::Version.new("0.0.2.rc1"), "ruby"], - ["example", Gem::Version.new("0.0.2.rc2"), "ruby"], - ["example", Gem::Version.new("0.0.2.rc2"), "java"], - ["other-example", Gem::Version.new("0.1.1.rc1"), "ruby"]] - end - - before do - Gemstash::CompactIndexBuilder::Versions.new(auth).serve - gem_id = insert_rubygem("example") - insert_version(gem_id, "0.0.1") - insert_version(gem_id, "0.0.2.rc1", prerelease: true) - insert_version(gem_id, "0.0.2.rc2", prerelease: true) - insert_version(gem_id, "0.0.2.rc2", platform: "java", prerelease: true) - insert_version(gem_id, "0.0.2") - insert_version(gem_id, "0.0.2", platform: "java") - insert_version(gem_id, "0.0.3.rc1", indexed: false, prerelease: true) - insert_version(gem_id, "0.0.3", indexed: false) - insert_version(gem_id, "0.0.3.rc1", indexed: false, prerelease: true, platform: "java") - insert_version(gem_id, "0.0.3", indexed: false, platform: "java") - gem_id = insert_rubygem("other-example") - insert_version(gem_id, "0.0.1", indexed: false) - insert_version(gem_id, "0.0.1.rc1", indexed: false, prerelease: true) - insert_version(gem_id, "0.1.0") - insert_version(gem_id, "0.1.1.rc1", prerelease: true) - end - - it "returns versions" do - result = Gemstash::CompactIndexBuilder::Versions.new(auth).serve - expect(result).to eq <<~VERSIONS - created_at: 1990-01-01T00:00:00Z - --- - example 0.0.1 1e6fae87f01f5e16ef83205a1a12646c - other-example 0.0.1 6105347ebb9825ac754615ca55ff3b0c - other-example 0.0.1.rc1 6105347ebb9825ac754615ca55ff3b0c - example 0.0.2-java 30b1ce74f9d06e512e354c697280c5e0 - example 0.0.2 1c60ca76f3375ac0473e16c9920a41c6 - example 0.0.2.rc1 d6f36de1e2fbebb92b6051fc6977ff0a - example 0.0.2.rc2-java 11850dde5a9df04c3fb2aba44704085d - example 0.0.2.rc2 48a1807ddf7e6a29c84d0f261cf4df64 - example 0.0.3-java 30b1ce74f9d06e512e354c697280c5e0 - example 0.0.3 30b1ce74f9d06e512e354c697280c5e0 - example 0.0.3.rc1-java 30b1ce74f9d06e512e354c697280c5e0 - example 0.0.3.rc1 30b1ce74f9d06e512e354c697280c5e0 - other-example 0.1.0 ff0722a59d13124677a2edd0da268bd1 - other-example 0.1.1.rc1 1b239fe769f037ab38a4c89ea6b37320 - VERSIONS - end - end - - context "with a new spec pushed" do - before do - Gemstash::Authorization.authorize(auth_key, "all") - gem_id = insert_rubygem("example") - insert_version(gem_id, "0.0.1") - end - - it "busts the cache" do - # before - Gemstash::GemPusher.new(auth, read_gem("example", "0.1.0")).serve - # after - end - end - - context "with a spec yanked" do - let(:initial_specs) do - [["example", Gem::Version.new("0.0.1"), "ruby"], - ["example", Gem::Version.new("0.1.0"), "ruby"]] - end - - let(:latest_specs) { [["example", Gem::Version.new("0.1.0"), "ruby"]] } - - let(:specs_after_yank) { [["example", Gem::Version.new("0.0.1"), "ruby"]] } - - before do - Gemstash::Authorization.authorize(auth_key, "all") - gem_id = insert_rubygem("example") - insert_version(gem_id, "0.0.1") - Gemstash::GemPusher.new(auth, read_gem("example", "0.1.0")).serve - end - - it "busts the cache" do - result = Gemstash::SpecsBuilder.new(auth).serve - expect(Marshal.load(gunzip(result))).to match_array(initial_specs) - Gemstash::GemYanker.new(auth, "example", "0.1.0").serve - result = Gemstash::SpecsBuilder.new(auth).serve - expect(Marshal.load(gunzip(result))).to match_array(specs_after_yank) - end - end - - context "with protected fetch disabled" do - it "serves versions without authorization" do - result = Gemstash::CompactIndexBuilder::Versions.new(auth).serve - expect(result).to eq(<<~VERSIONS) - created_at: 1990-01-01T00:00:00Z - --- - VERSIONS - end - end - - xcontext "with protected fetch enabled" do - before do - @test_env = test_env - config = Gemstash::Configuration.new(config: { protected_fetch: true }) - Gemstash::Env.current = Gemstash::Env.new(config) - end - - after do - Gemstash::Env.current = @test_env - end - - context "with valid authorization" do - it "serves specs" do - result = Gemstash::SpecsBuilder.new(auth).serve - expect(Marshal.load(gunzip(result))).to eq([]) - end - end - - context "with invalid authorization" do - it "prevents serving specs" do - expect { Gemstash::SpecsBuilder.new(auth_with_invalid_auth_key).serve }. - to raise_error(Gemstash::NotAuthorizedError) - end - end - - context "with invalid permission" do - it "prevents serving specs" do - expect { Gemstash::SpecsBuilder.new(auth_without_permission).serve }. - to raise_error(Gemstash::NotAuthorizedError) - end - end - end -end diff --git a/spec/gemstash/web_spec.rb b/spec/gemstash/web_spec.rb index f745a868..57ffe839 100644 --- a/spec/gemstash/web_spec.rb +++ b/spec/gemstash/web_spec.rb @@ -431,8 +431,8 @@ def for(server_url, timeout = 20) post "/api/v1/gems", read_gem("example", "0.1.0"), env expect(last_response).to_not be_ok - expect(last_response.status).to eq(409) - expect(JSON.parse(last_response.body)).to eq("error" => "Version already exists", "code" => 409) + expect(last_response.status).to eq(422) + expect(JSON.parse(last_response.body)).to eq("error" => "Version already exists", "code" => 422) end end end diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 4d7cbbf8..49a1de7e 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -69,14 +69,6 @@ @gemstash_empty_rubygems.start end - before(:each) do - $test_gemstash_server_current_test = self # rubocop:disable Style/GlobalVars - end - - after(:each) do - $test_gemstash_server_current_test = nil # rubocop:disable Style/GlobalVars - end - let(:platform_message) do if RUBY_PLATFORM == "java" "Java" @@ -124,15 +116,6 @@ clean_env env_name end - it "is a conformant gem server", db_transaction: false do - @gemstash.env.cache.flush - expect( - execute("gem_server_conformance", ["--fail-fast", "--format", "progress", "--tag=~content_length_header"], - env: { "UPSTREAM" => host, "GEM_HOST_API_KEY" => auth_key }) - ). - to exit_success - end - context "pushing a gem" do before do expect(deps.fetch(%w[speaker])).to match_dependencies([]) diff --git a/spec/support/db_helpers.rb b/spec/support/db_helpers.rb index ec0075d9..d5a478ca 100644 --- a/spec/support/db_helpers.rb +++ b/spec/support/db_helpers.rb @@ -31,12 +31,9 @@ def insert_version(gem_id, number, platform: "ruby", indexed: true, prerelease: :storage_id => storage_id, :indexed => indexed, :prerelease => prerelease, - :sha256 => Digest::SHA256.hexdigest(storage_id), :created_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP, :updated_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP - ).tap do |version_id| - update_info_checksum(version_id) - end + ) end def insert_dependency(version_id, gem_name, requirements) @@ -46,16 +43,6 @@ def insert_dependency(version_id, gem_name, requirements) :requirements => requirements, :created_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP, :updated_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP - ).tap do - update_info_checksum(version_id) - end - end - - def update_info_checksum(version_id) - gem_id = Gemstash::Env.current.db[:versions][id: version_id][:rubygem_id] - gem_name = Gemstash::Env.current.db[:rubygems][id: gem_id][:name] - info = Gemstash::CompactIndexBuilder::Info.new(nil, gem_name).tap(&:build_result).result - Gemstash::DB::Version.where(id: version_id).update(info_checksum: Digest::MD5.hexdigest(info)) - raise "Failed to update info checksum for version #{version_id}" unless Gemstash::Env.current.db[:versions][id: version_id][:info_checksum] + ) end end diff --git a/spec/support/exec_helpers.rb b/spec/support/exec_helpers.rb index 1838a688..3820bad8 100644 --- a/spec/support/exec_helpers.rb +++ b/spec/support/exec_helpers.rb @@ -76,10 +76,7 @@ def clear_ruby_env "BUNDLER_SETUP" => nil, "GEM_PATH" => original_gem_path, "RUBYLIB" => nil, - "RUBYOPT" => nil, - "XDG_CONFIG_HOME" => nil, - "XDG_CACHE_HOME" => nil, - "XDG_DATA_HOME" => nil + "RUBYOPT" => nil } end diff --git a/spec/support/test_gemstash_server.ru b/spec/support/test_gemstash_server.ru index c7317dcb..358a5878 100644 --- a/spec/support/test_gemstash_server.ru +++ b/spec/support/test_gemstash_server.ru @@ -5,18 +5,4 @@ use Rack::Deflater use Gemstash::Env::RackMiddleware, $test_gemstash_server_env use Gemstash::GemSource::RackMiddleware use Gemstash::Health::RackMiddleware -map "/set_time" do - run lambda {|env| - now = Time.iso8601(Rack::Request.new(env).body.read) - $test_gemstash_server_current_test.allow(Time). - to $test_gemstash_server_current_test.receive(:now).and_return(now) - [200, {}, ["OK"]] - } -end -map "/rebuild_versions_list" do - run lambda {|_env| - Gemstash::CompactIndexBuilder::Versions.new(nil).build_result(force_rebuild: true) - [200, {}, ["OK"]] - } -end run Gemstash::Web.new(gemstash_env: $test_gemstash_server_env)