Fetch maintainers at a given date. (#71)

Signed-off-by: dblock <[email protected]>

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2023-08-16 20:13:00 UTC using RuboCop version 1.36.0.
+# on 2023-08-16 20:46:04 UTC using RuboCop version 1.36.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -18,6 +18,11 @@ Lint/EmptyBlock:
     - 'spec/project/commands/members_spec.rb'
     - 'spec/project/commands/repos_spec.rb'
 
+# Offense count: 1
+Lint/NoReturnInBeginEndBlocks:
+  Exclude:
+    - 'lib/github/repo.rb'
+
 # Offense count: 1
 # This cop supports unsafe autocorrection (--autocorrect-all).
 Lint/NonDeterministicRequireOrder:
@@ -36,7 +41,7 @@ Lint/UnderscorePrefixedVariableName:
   Exclude:
     - 'lib/github/maintainers.rb'
 
-# Offense count: 4
+# Offense count: 3
 Lint/UselessAssignment:
   Exclude:
     - 'lib/github/repo.rb'
@@ -45,7 +50,7 @@ Lint/UselessAssignment:
 # Offense count: 1
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 124
+  Max: 105
 
 # Offense count: 2
 # Configuration parameters: AllowedMethods, AllowedPatterns, IgnoredMethods.
@@ -57,12 +62,14 @@ Metrics/CyclomaticComplexity:
 Metrics/PerceivedComplexity:
   Max: 10
 
-# Offense count: 1
+# Offense count: 4
 # Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames.
 # AllowedNames: as, at, by, db, id, in, io, ip, of, on, os, pp, to
 Naming/MethodParameterName:
   Exclude:
     - 'lib/github/item.rb'
+    - 'lib/github/repo.rb'
+    - 'lib/github/repos.rb'
 
 # Offense count: 21
 # Configuration parameters: Prefixes, AllowedPatterns.
@@ -180,12 +187,6 @@ Style/FrozenStringLiteralComment:
     - 'spec/support/github.rb'
     - 'spec/support/without_data_files.rb'
 
-# Offense count: 2
-# This cop supports unsafe autocorrection (--autocorrect-all).
-Style/GlobalStdStream:
-  Exclude:
-    - 'bin/commands/maintainers.rb'
-
 # Offense count: 1
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: AllowedReceivers.
@@ -199,18 +200,12 @@ Style/IdenticalConditionalBranches:
   Exclude:
     - 'lib/github/repo.rb'
 
-# Offense count: 2
+# Offense count: 1
 # This cop supports unsafe autocorrection (--autocorrect-all).
 Style/MapToHash:
   Exclude:
-    - 'bin/commands/maintainers.rb'
     - 'lib/github/buckets.rb'
 
-# Offense count: 1
-Style/MultilineBlockChain:
-  Exclude:
-    - 'bin/commands/maintainers.rb'
-
 # Offense count: 1
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: EnforcedStyle, AllowedMethods, AllowedPatterns, IgnoredMethods.

README.md

@@ -114,6 +114,14 @@ Shows maintainer stats.
 As of 2023-05-17, 98 repos have 198 maintainers, including 17% (17/98) of repos with at least one of 15 external maintainers.
 ```
 
+You can pass a date to find out stats for a given point in time.
+
+```
+./bin/project maintainers stats --date=2023-06-01
+
+As of 2023-06-01, 103 repos have 202 maintainers, including 18% (19/103) of repos with at least one of 17 external maintainers.
+```
+
 Shows missing MAINTAINERS.md.
 
 ```

bin/commands/maintainers.rb

@@ -6,27 +6,29 @@ class Commands
     command 'maintainers' do |g|
       g.desc 'Show MAINTAINERS.md stats.'
       g.command 'stats' do |c|
+        c.flag %i[date], desc: 'Date at.', default_value: nil
         c.action do |_global_options, options, _args|
           org = GitHub::Organization.new(options.merge(org: options['org'] || 'opensearch-project'))
-          maintainers = org.repos.maintainers
-          puts "As of #{Date.today}, #{org.repos.count} repos have #{maintainers.unique_count} maintainers, including #{org.repos.external_maintainers_percent}% (#{org.repos.maintained[:external].size + org.repos.maintained[:students].size}/#{org.repos.count}) of repos with at least one of #{maintainers.external_unique_count} external maintainers."
+          dt = options[:date] ? Chronic.parse(options[:date]).to_date : nil
+          maintainers = org.repos.maintainers(dt)
+          puts "As of #{dt || Date.today}, #{org.repos.count} repos have #{maintainers.unique_count} maintainers, including #{org.repos.external_maintainers_percent}% (#{org.repos.external_maintained_size}/#{org.repos.count}) of repos with at least one of #{maintainers.external_unique_count} external maintainers."
           puts "\n# Maintainers\n"
           puts "unique: #{maintainers.unique_count}"
           maintainers.each_pair do |bucket, logins|
             puts "#{bucket}: #{logins.size} (#{logins.map(&:to_s).join(', ')})"
           end
           puts "\n# External Maintainers\n"
-          org.repos.maintained[:external].sort_by(&:name).each do |repo|
+          org.repos.maintained[:external]&.sort_by(&:name)&.each do |repo|
             puts "#{repo.html_url}: #{repo.maintainers[:external]}"
           end
 
           puts "\n# Student Maintainers\n"
-          org.repos.maintained[:students].sort_by(&:name).each do |repo|
+          org.repos.maintained[:students]&.sort_by(&:name)&.each do |repo|
             puts "#{repo.html_url}: #{repo.maintainers[:students]}"
           end
 
           puts "\n# Unknown Maintainers\n"
-          org.repos.maintained[:unknown].sort_by(&:name).each do |repo|
+          org.repos.maintained[:unknown]&.sort_by(&:name)&.each do |repo|
             puts "#{repo.html_url}: #{repo.maintainers[:unknown]}"
           end
         end
@@ -80,7 +82,7 @@ class Commands
           total_users = 0
           total_repos = 0
           unique_users = Set.new
-          all = repos.map do |repo|
+          repos.each do |repo|
             users = repo.maintainers&.map do |user|
               commits = $github.commits(repo.full_name, author: user)
               next if commits.any?
@@ -92,13 +94,9 @@ class Commands
             total_users += users.count
             total_repos += 1
             unique_users.add(users)
-            STDOUT.print('.')
-            [repo, users]
-          end.compact.to_h
-          puts "\nThere are #{unique_users.count} unique names in #{total_users} instances of users listed in MAINTAINERS.md that have never contributed across #{total_repos}/#{repos.count} repos.\n\n"
-          all.sort_by { |_k, v| -v.size }.each do |repo, users|
             puts "#{repo.html_url}: #{users}" if users&.any?
           end
+          puts "\nThere are #{unique_users.count} unique names in #{total_users} instances of users listed in MAINTAINERS.md that have never contributed across #{total_repos}/#{repos.count} repos."
         end
       end
 
@@ -119,24 +117,6 @@ class Commands
           end
         end
       end
-
-      g.desc 'Find MAINTAINERS.md emails.'
-      g.command 'emails' do |c|
-        c.action do |_global_options, options, _args|
-          org = GitHub::Organization.new(options.merge(org: options['org'] || 'opensearch-project'))
-          repos = org.repos.sort_by(&:name)
-          all = repos.map do |repo|
-            users_and_emails = repo.maintainers&.map do |user|
-              [user, $github.commits(repo.full_name, author: user).map do |commit|
-                commit.commit.author.email
-              end.uniq.compact.reject { |e| e.ends_with?('@users.noreply.github.com') }.take(1)]
-            end&.compact.to_h
-            STDOUT.print '.'
-            [repo, users_and_emails]
-          end.to_h.values.inject(:merge)
-          puts all.values
-        end
-      end
     end
   end
 end

lib/github/maintainers.rb

@@ -21,7 +21,7 @@ def unique_count
     end
 
     def external_unique_count
-      buckets[:external].size + buckets[:students].size
+      (buckets[:external]&.size || 0) + (buckets[:students]&.size || 0)
     end
 
     def each_pair(&_block)

lib/github/repo.rb

@@ -12,19 +12,30 @@ def initialize(id_or_obj)
       raise "Invalid repo: #{id_or_obj}: #{e.message}"
     end
 
-    def maintainers_md
+    def maintainers_md(dt = nil)
       @maintainers_md ||= begin
-        data = $github.contents(full_name, path: 'MAINTAINERS.md')
-        md = Base64.decode64(data.content)
+        if dt
+          $github.commits(full_name, path: 'MAINTAINERS.md').each do |commit|
+            next unless commit.commit.author[:date] < dt
+
+            # puts "Fetched #{full_name}, path=MAINTAINERS.md, ref=#{commit.sha}"
+            data = $github.contents(full_name, path: 'MAINTAINERS.md', query: { ref: commit.sha })
+            return Base64.decode64(data.content)
+          end
+          nil
+        else
+          data = $github.contents(full_name, path: 'MAINTAINERS.md')
+          Base64.decode64(data.content)
+        end
       rescue Octokit::NotFound
         nil
       end
     end
 
-    def maintainers
+    def maintainers(dt = nil)
       @maintainers ||= begin
         data = {}
-        content = maintainers_md
+        content = maintainers_md(dt)
         parsed = Redcarpet::Markdown.new(MaintainersExtractor, tables: true, target: data).render(content) if content
         GitHub::Maintainers.new(data[:maintainers]) if data && data.key?(:maintainers)
       rescue Octokit::NotFound

lib/github/repos.rb

@@ -17,11 +17,11 @@ def unarchived
       GitHub::Repos.new(reject { |repo| repo.archived })
     end
 
-    def maintainers
+    def maintainers(dt = nil)
       @maintainers ||= begin
         all = Set.new
         each do |repo|
-          maintainers = repo.maintainers
+          maintainers = repo.maintainers(dt)
           maintainers&.each do |user|
             all.add(user)
           end
@@ -43,10 +43,14 @@ def maintained
       end
     end
 
+    def external_maintained_size
+      (maintained[:external]&.size || 0) + (maintained[:students]&.size || 0)
+    end
+
     def external_maintainers_percent
       return 0 unless any?
 
-      (((maintained[:external].size.to_f + maintained[:students].size.to_f) / size) * 100).to_i
+      (((maintained[:external]&.size.to_f + maintained[:students]&.size.to_f) / size) * 100).to_i
     end
   end
 end