REFACTOR Use changelog uri from rubygems metadata (#171)

.github/workflows/ruby.yml

@@ -9,15 +9,17 @@ jobs:
       matrix:
         ruby: ['3.0', '3.1', '3.2']
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Ruby ${{ matrix.ruby }}
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Install dependencies
       run: bundle install
     - name: Run tests
-      run: make tests
+      # Exclude acceptance specs as github action fails due to:
+      # It is a security vulnerability to allow your home directory to be world-writable, and bundler cannot continue.
+      run: bundle exec rspec --exclude-pattern "spec/acceptance/**/*_spec.rb"
     - name: Run linter
       run: make lint
 

CHANGELOG.md

@@ -1,5 +1,12 @@
 # master (unreleased)
 
+Changed:
+* know rely on rubygems metadata to get the changelog uri.
+  It means that for any gem not hosted on rubygems, the changelog won’t be found.
+
+Deprecated:
+* [rails-assets](https://rails-assets.org/#/) support
+
 Development tools:
 * add `.ruby-version``
 * add `Makefile`

Gemfile.lock

@@ -1,10 +1,9 @@
 PATH
   remote: .
   specs:
-    gem_updater (6.1.0)
+    gem_updater (7.0.0)
       bundler (< 3)
       json (~> 2.6)
-      memoist (~> 0.16.2)
       nokogiri (~> 1.13)
 
 GEM
@@ -21,7 +20,6 @@ GEM
     hashdiff (1.0.1)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
-    memoist (0.16.2)
     method_source (1.0.0)
     mini_portile2 (2.8.4)
     nokogiri (1.15.3)
@@ -102,4 +100,4 @@ DEPENDENCIES
   webmock (~> 3.18)
 
 BUNDLED WITH
-   2.4.17
+   2.5.0.dev

README.md

@@ -59,6 +59,14 @@ By default, diff for your gems will look like the following:
 You can change it if you like by writing you own template `.gem_updater_template.erb` in your home directory.
 [Look at default template](lib/gem_updater_template.erb) for an example on how to do it.
 
+## Troubleshooting
+
+### Changelog not found?
+
+This project relies on the gem’s metadata to find the changelog url.
+If a changelog was not found, check if the gem’s authors declared its uri in its gemspec,
+like [here](https://github.com/thoughtbot/factory_bot/blob/8f4f899305be5a09cee206876eb8d346cf6a0dcb/factory_bot.gemspec#L26).
+
 ## Contributing
 
 PRs are always welcome!

gem_updater.gemspec

@@ -26,7 +26,6 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'bundler',  '< 3'
   s.add_runtime_dependency 'json',     '~> 2.6'
-  s.add_runtime_dependency 'memoist',  '~> 0.16.2'
   s.add_runtime_dependency 'nokogiri', '~> 1.13'
 
   s.add_development_dependency 'pry', '~> 0.14'

lib/gem_updater.rb

@@ -1,22 +1,21 @@
 # frozen_string_literal: true
 
 require 'erb'
-require 'memoist'
-require 'gem_updater/gem_file'
+require 'gem_updater/changelog_parser'
+require 'gem_updater/gemfile'
 require 'gem_updater/ruby_gems_fetcher'
-require 'gem_updater/source_page_parser'
 
 # Base lib.
 module GemUpdater
   # Updater's main responsability is to fill changes
   # happened before and after update of `Gemfile`, and then format them.
   class Updater
-    extend Memoist
+    RUBYGEMS_SOURCE_NAME = 'rubygems repository https://rubygems.org/'
 
     attr_accessor :gemfile
 
     def initialize
-      @gemfile = GemUpdater::GemFile.new
+      @gemfile = GemUpdater::Gemfile.new
     end
 
     # Update process.
@@ -52,35 +51,23 @@ def format_diff
     # For each gem, retrieve its changelog
     def fill_changelogs
       [].tap do |threads|
-        gemfile.changes.each do |gem_name, details|
-          threads << Thread.new { retrieve_gem_changes(gem_name, details) }
+        gemfile.changes.each do |gem_changes, details|
+          threads << Thread.new { retrieve_changelog(gem_changes, details) }
         end
       end.each(&:join)
     end
 
-    # Find where is hosted the source of a gem
-    #
-    # @param gem [String] the name of the gem
-    # @param source [Bundler::Source] gem's source
-    # @return [String] url where gem is hosted
-    def find_source(gem, source)
-      case source
-      when Bundler::Source::Rubygems
-        GemUpdater::RubyGemsFetcher.new(gem, source).source_uri
-      when Bundler::Source::Git
-        source.uri.gsub(/^git/, 'http').chomp('.git')
-      end
-    end
+    # Get the changelog URL.
+    def retrieve_changelog(gem_name, details)
+      return unless details[:source].name == RUBYGEMS_SOURCE_NAME
 
-    def retrieve_gem_changes(gem_name, details)
-      source_uri = find_source(gem_name, details[:source])
-      return unless source_uri
+      changelog_uri = RubyGemsFetcher.new(gem_name).changelog_uri
 
-      source_page = GemUpdater::SourcePageParser.new(
-        url: source_uri, version: details[:versions][:new]
-      )
+      return unless changelog_uri
 
-      gemfile.changes[gem_name][:changelog] = source_page.changelog if source_page.changelog
+      changelog = ChangelogParser
+                  .new(uri: changelog_uri, version: details.dig(:versions, :new)).changelog
+      gemfile.changes[gem_name][:changelog] = changelog&.to_s
     end
 
     # Get the template for gem's diff.
@@ -92,6 +79,5 @@ def template
     rescue Errno::ENOENT
       File.read(File.expand_path('../lib/gem_updater_template.erb', __dir__))
     end
-    memoize :template
   end
 end

lib/gem_updater/changelog_parser.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'open-uri'
+require 'gem_updater/changelog_parser/github_parser'
+
+module GemUpdater
+  # ChangelogParser is responsible for parsing a source page where
+  # the gem code is hosted.
+  class ChangelogParser
+    MARKUP_FILES = %w[.md .rdoc .textile].freeze
+
+    attr_reader :uri, :version
+
+    # @param uri [String] uri of changelog
+    # @param version [String] version of gem
+    def initialize(uri:, version:)
+      @uri     = uri
+      @version = version
+    end
+
+    # Get the changelog in an uri.
+    #
+    # @return [String, nil] URL of changelog
+    def changelog
+      return uri unless changelog_may_contain_anchor?
+
+      parse_changelog
+    rescue OpenURI::HTTPError # Uri points to nothing
+      log_error_and_return_uri("Cannot find #{uri}")
+    rescue Errno::ETIMEDOUT # timeout
+      log_error_and_return_uri("#{URI.parse(uri).host} is down")
+    rescue ArgumentError => e # x-oauth-basic raises userinfo not supported. [RFC3986]
+      log_error_and_return_uri(e)
+    end
+
+    private
+
+    # Try to find where changelog might be.
+    #
+    # @param doc [Nokogiri::XML::Element] document of source page
+    def parse_changelog
+      case URI.parse(uri).host
+      when 'github.com'
+        GithubParser.new(uri: uri, version: version).changelog
+      else
+        uri
+      end
+    end
+
+    # Some documents like the one written in markdown may contain
+    # a direct anchor to specific version.
+    #
+    # @return [Boolean] true if file may contain an anchor
+    def changelog_may_contain_anchor?
+      MARKUP_FILES.include?(File.extname(uri.to_s))
+    end
+
+    def log_error_and_return_uri(error_message)
+      Bundler.ui.error error_message
+      uri
+    end
+  end
+end

lib/gem_updater/changelog_parser/github_parser.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'open-uri'
+
+module GemUpdater
+  class ChangelogParser
+    # ChangelogParser is responsible for parsing a changelog hosted on github.
+    class GithubParser
+      attr_reader :uri, :version
+
+      # @param uri [String] changelog uri
+      # @param version [String] version of gem
+      def initialize(uri:, version:)
+        @uri     = uri
+        @version = version
+      end
+
+      # Finds anchor in changelog, otherwise return the base uri.
+      #
+      # @return [String] the URL of changelog
+      def changelog
+        uri + find_anchor(document).to_s
+      end
+
+      private
+
+      # Opens changelog url and parses it.
+      #
+      # @return [Nokogiri::HTML4::Document] the changelog
+      def document
+        Nokogiri::HTML(URI.parse(uri).open, nil, Encoding::UTF_8.to_s)
+      end
+
+      # Looks into document to find it there is an anchor to new gem version.
+      #
+      # @param doc [Nokogiri::HTML4::Document] document
+      # @return [String, nil] anchor's href
+      def find_anchor(doc)
+        anchor = doc.xpath('//a[contains(@class, "anchor")]').find do |element|
+          element.attr('href').match(version.delete('.'))
+        end
+        return unless anchor
+
+        anchor.attr('href').gsub(%(\\"), '')
+      end
+    end
+  end
+end

lib/gem_updater/gem_file.rb → lib/gem_updater/gemfile.rb

@@ -4,8 +4,8 @@
 require 'bundler/cli'
 
 module GemUpdater
-  # GemFile is responsible for handling `Gemfile`
-  class GemFile
+  # Gemfile is responsible for handling `Gemfile`
+  class Gemfile
     attr_accessor :changes
     attr_reader :old_spec_set, :new_spec_set
 

lib/gem_updater/ruby_gems_fetcher.rb

@@ -8,23 +8,22 @@ module GemUpdater
   # RubyGemsFetcher is a wrapper around rubygems API.
   class RubyGemsFetcher
     HTTP_TOO_MANY_REQUESTS = '429'
-    GEM_HOMEPAGES = %w[source_code_uri homepage_uri].freeze
 
-    attr_reader :gem_name, :source
+    attr_reader :gem_name
 
     # @param gem_name [String] name of the gem
-    # @param source [Bundler::Source] source of gem
-    def initialize(gem_name, source)
+    def initialize(gem_name)
       @gem_name = gem_name
-      @source   = source
     end
 
-    # Finds where code is hosted.
-    # Most likely in will be on rubygems, else look in other sources.
+    # Finds the changelog uri.
+    # It asks rubygems.org for changelog_uri of gem.
+    # See API: http://guides.rubygems.org/rubygems-org-api/#gem-methods
     #
-    # @return [String|nil] url of gem source code
-    def source_uri
-      uri_from_rubygems || uri_from_other_sources
+    # @return [String|nil] uri of changelog
+    def changelog_uri
+      response = query_rubygems
+      response.to_h['changelog_uri']
     end
 
     private
@@ -37,19 +36,6 @@ def parse_remote_json(url)
       JSON.parse(URI.parse(url).open.read)
     end
 
-    # Ask rubygems.org for source uri of gem.
-    # See API: http://guides.rubygems.org/rubygems-org-api/#gem-methods
-    #
-    # @return [String|nil] uri of source code
-    def uri_from_rubygems
-      return unless source.remotes.map(&:host).include?('rubygems.org')
-
-      response = query_rubygems
-      return unless response
-
-      response[GEM_HOMEPAGES.find { |key| response[key] && !response[key].empty? }]
-    end
-
     # Make the real query to rubygems
     # It may fail in case we trigger too many requests
     #
@@ -63,43 +49,5 @@ def query_rubygems(tries = 0)
         sleep 1 && retry if tries < 2
       end
     end
-
-    # Look if gem can be found in another remote
-    #
-    # @return [String|nil] uri of source code
-    def uri_from_other_sources
-      source.remotes.find do |remote|
-        case remote.host
-        when 'rubygems.org' then next # already checked
-        when 'rails-assets.org'
-          return uri_from_railsassets
-        else
-          Bundler.ui.error "Source #{remote} is not supported, ' \
-            'feel free to open a PR or an issue on https://github.com/MaximeD/gem_updater"
-        end
-      end
-    end
-
-    # Ask rails-assets.org for source uri of gem.
-    # API is at : https://rails-assets.org/packages/package_name
-    #
-    # @return [String|nil] uri of source code
-    def uri_from_railsassets
-      response = query_railsassets
-      return unless response
-
-      response['url'].gsub(/^git/, 'http')
-    end
-
-    # Make the real query to railsassets
-    # rubocop:disable Lint/SuppressedException
-    def query_railsassets
-      parse_remote_json("https://rails-assets.org/packages/#{gem_name.gsub('rails-assets-', '')}")
-    rescue JSON::ParserError
-      # if gem is not found, rails-assets returns a 200
-      # with html (instead of json) containing a 500...
-    rescue OpenURI::HTTPError
-    end
-    # rubocop:enable Lint/SuppressedException
   end
 end