First commit, can analyse words using smart stoplist

.github/workflows/main.yml

@@ -0,0 +1,27 @@
+name: Ruby
+
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Ruby ${{ matrix.ruby }}
+    strategy:
+      matrix:
+        ruby:
+          - '3.2.2'
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Run the default task
+      run: bundle exec rake

.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in keyphrase.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rspec", "~> 3.0"
+gem "pry"

Gemfile.lock

@@ -0,0 +1,40 @@
+PATH
+  remote: .
+  specs:
+    keyphrase (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.3)
+    diff-lcs (1.5.0)
+    method_source (1.0.0)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.1.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  keyphrase!
+  pry
+  rake (~> 13.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.4.22

LICENSE.txt

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Ben D'Angelo
+Copyright (c) 2013, Darius Morawiec
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,74 @@
+# Keyphrase
+
+Extracts keywords from texts using a stoplist and some magic. This is forked
+from the original gem [rake_text](https://github.com/voidplus/rake-text-ruby)
+and has some added improvements.
+
+* Multi-language support.
+* Better split word support, handles - and ' characters better.
+* Stop word is not removed if deemed important.
+
+## Installation
+
+Install [Keyphrase](https://rubygems.org/gems/keyphrase) via RubyGem:
+
+```
+gem install keyphrase
+```
+
+## Usage
+
+Import the library and create an instance:
+
+```
+require 'keyphrase'
+keyphrase = Keyphrase.new
+```
+
+Use the Smart Stoplist:
+
+```
+keyphrase.analyse "your text", stoplist: Keyphrase.stopwords[:en]
+# → {"compatibility"=>1.0, "systems"=>1.0, "linear constraints"=>4.5, "set"=>2.0, "natural numbers"=>4.0, "criteria"=>1.0, "system"=>1.0, "linear diophantine equations"=>8.5, "strict inequations"=>4.0, "nonstrict inequations"=>4.0, "considered"=>1.5, "upper bounds"=>4.0, "components"=>1.0, "minimal set"=>4.666666666666666, "solutions"=>1.0, "algorithms"=>1.0, "construction"=>1.0, "minimal generating sets"=>8.666666666666666, "types"=>1.6666666666666667, "constructing"=>1.0, "minimal supporting set"=>7.666666666666666, "solving"=>1.0, "considered types"=>3.166666666666667, "mixed types"=>3.666666666666667}
+```
+
+Use a custom stopword list:
+
+```
+keyphrase.analyse "your text", ["custom","stopword","list"]
+```
+
+Shorthand usage:
+
+```
+Keyphrase.analyse "your text"
+```
+
+Show sorted results:
+
+```
+keyphrase.analyse "your text", verbose: true
+# 8.67 - minimal generating sets
+# 8.50 - linear diophantine equations
+# 7.67 - minimal supporting set
+# 4.67 - minimal set
+# 4.50 - linear constraints
+# 4.00 - upper bounds
+# 4.00 - strict inequations
+# [...]
+# → {"compatibility"=>1.0, "systems"=>1.0, "linear constraints"=>4.5, "set"=>2.0, "natural numbers"=>4.0, "criteria"=>1.0, "system"=>1.0, "linear diophantine equations"=>8.5, "strict inequations"=>4.0, "nonstrict inequations"=>4.0, "considered"=>1.5, "upper bounds"=>4.0, "components"=>1.0, "minimal set"=>4.666666666666666, "solutions"=>1.0, "algorithms"=>1.0, "construction"=>1.0, "minimal generating sets"=>8.666666666666666, "types"=>1.6666666666666667, "constructing"=>1.0, "minimal supporting set"=>7.666666666666666, "solving"=>1.0, "considered types"=>3.166666666666667, "mixed types"=>3.666666666666667}
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/bendangelo/keyphrase.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "keyphrase"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "irb"
+IRB.start(__FILE__)

bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

keyphrase.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "lib/keyphrase/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "keyphrase"
+  spec.version = Keyphrase::VERSION
+  spec.authors = ["Ben D'Angelo"]
+  spec.email = ["[email protected]"]
+
+  spec.summary = "Extracts keywords from texts using a stoplist and some magic."
+  spec.description = "Implementation of the Rapid Automatic Keyword Extraction (RAKE) algorithm in Ruby. Forked from the original rake_text gem."
+  spec.homepage = "https://github.com/bendangelo/keyphrase"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/bendangelo/keyphrase"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .github appveyor Gemfile])
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+end

lib/keyphrase.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require_relative "keyphrase/version"
+
+class Keyphrase
+
+  autoload :Stoplist, "keyphrase/stoplist"
+
+  CLEAN_REGEX = /([^a-zA-Z0-9'\- \.]|(?<!\w)'|(?<!\w)\.)/
+  SENTENCES_REGEX = /[!?,;:\n\t\\"\\(\\)\u2019\u2013\|]|-(?!\w)|(?<!\w)'(?!\w)|(?<!\s)\.[^a-zA-Z0-9]/u
+
+  def self.analyse text, options={}
+    @@keyphrase ||= Keyphrase.new
+    @@keyphrase.analyse text, options
+  end
+
+  def analyse text, options={}
+    stoplist = options[:stoplist] || :smart
+    clean_regex = options[:clean] || CLEAN_REGEX
+
+    pattern    = buildStopwordRegExPattern stoplist
+    sentences  = text.split SENTENCES_REGEX
+    phrases    = generateCandidateKeywords sentences, pattern, clean_regex
+    wordscores = calculateWordScores phrases
+    candidates = generateCandidateKeywordScores phrases, wordscores
+
+    if options[:verbose]
+      result = candidates.sort_by{|k,v| v}.reverse
+      result.each do |word, score|
+        puts sprintf '%.2f - %s', score, word
+      end
+    end
+
+    return candidates
+  end
+
+  private
+
+  # create stopword pattern
+  # 1
+  def buildStopwordRegExPattern stopwords
+
+    if stopwords.is_a? Symbol
+      # use caching
+      return Keyphrase::Stoplist::Eng.smart_regex
+    end
+
+    stop_regex = /(?:^|\s)(?:#{stopwords.join('|')})(?:$|\s)/io
+
+    return stop_regex
+  end
+
+  # generate candidate keywords
+  # 2
+  def generateCandidateKeywords sentences, stopwords_regex, clean_regex
+    phrases = Array.new
+
+    filtered_sentences = sentences.map { |sentence| sentence.gsub(clean_regex, "").gsub(stopwords_regex, "|") }
+
+    filtered_sentences.each do |parts|
+      parts.split("|").each do |part|
+        part = part.strip
+
+        if !part.empty?
+          phrases.push part
+        end
+      end
+    end
+
+    return phrases
+  end
+
+  # calculate individual word scores
+  # 3
+  def calculateWordScores phrases
+    word_freq = Hash.new 0
+    word_degree = Hash.new 0
+    word_score = Hash.new 0
+
+    phrases.each do |phrase|
+      words = seperateWords phrase
+
+      length = words.length
+      degree = length-1
+
+      words.each do |word|
+        word_freq[word] += 1
+        word_degree[word] += degree
+      end
+    end
+
+    word_freq.each do |word, counter|
+      word_degree[word] = word_degree[word] + word_freq[word]
+    end
+
+    word_freq.each do |word, counter|
+      word_score[word] = word_degree[word]/(word_freq[word] * 1.0)
+    end
+
+    return word_score
+  end
+
+  # generate candidate keyword scores
+  # 4
+  def generateCandidateKeywordScores phrases, scores
+    candidates = Hash.new 0
+
+    phrases.each do |phrase|
+      words = seperateWords(phrase)
+      score = 0
+      words.each do |word|
+        score += scores[word]
+      end
+      candidates[phrase] = score
+    end
+
+    return candidates
+  end
+
+  def seperateWords text
+    words = Array.new
+
+    text.split(/[^a-zA-Z0-9_\\+\\-\\']/).each do |word|
+      word = word.strip.downcase
+      if !word.empty? && !(true if Float(word) rescue false)
+        words.push word
+      end
+    end
+
+    return words
+  end
+
+end