From 6f05adc547a378c72dbfceca1f73a043444491cf Mon Sep 17 00:00:00 2001 From: Jill Klang Date: Fri, 5 Jan 2024 12:01:38 -0500 Subject: [PATCH] DataTaster: MVP Features (#214) Co-authored-by: Guilherme Moretti --- packages/data_taster/.rubocop.yml | 21 +- packages/data_taster/Appraisals | 13 + packages/data_taster/Gemfile | 6 +- packages/data_taster/Gemfile.lock | 247 +++++++++++++---- packages/data_taster/bin/console | 15 - packages/data_taster/bin/setup | 8 - packages/data_taster/data_taster.gemspec | 23 +- .../data_taster/doc/dependency_decisions.yml | 6 + packages/data_taster/docs/CHANGELOG.md | 8 + packages/data_taster/docs/README.md | 74 ++++- .../data_taster/gemfiles/rails_6_0.gemfile | 8 + .../gemfiles/rails_6_0.gemfile.lock | 258 +++++++++++++++++ .../data_taster/gemfiles/rails_6_1.gemfile | 8 + .../gemfiles/rails_6_1.gemfile.lock | 261 ++++++++++++++++++ .../data_taster/gemfiles/rails_7_0.gemfile | 8 + .../gemfiles/rails_7_0.gemfile.lock | 260 +++++++++++++++++ packages/data_taster/lib/data_taster.rb | 57 +++- .../data_taster/lib/data_taster/collection.rb | 74 +++++ .../data_taster/lib/data_taster/confection.rb | 38 +++ .../data_taster/lib/data_taster/detergent.rb | 98 +++++++ .../data_taster/lib/data_taster/flavors.rb | 72 +++++ .../data_taster/lib/data_taster/helper.rb | 28 ++ .../data_taster/lib/data_taster/sample.rb | 45 +++ .../data_taster/lib/data_taster/sanitizer.rb | 100 +++++++ .../data_taster/lib/data_taster/version.rb | 2 +- packages/data_taster/mkdocs.yml | 1 + packages/data_taster/spec/data_taster_spec.rb | 4 +- packages/data_taster/spec/spec_helper.rb | 53 +++- 28 files changed, 1694 insertions(+), 102 deletions(-) create mode 100644 packages/data_taster/Appraisals delete mode 100755 packages/data_taster/bin/console delete mode 100755 packages/data_taster/bin/setup create mode 100644 packages/data_taster/gemfiles/rails_6_0.gemfile create mode 100644 packages/data_taster/gemfiles/rails_6_0.gemfile.lock create mode 100644 packages/data_taster/gemfiles/rails_6_1.gemfile create mode 100644 packages/data_taster/gemfiles/rails_6_1.gemfile.lock create mode 100644 packages/data_taster/gemfiles/rails_7_0.gemfile create mode 100644 packages/data_taster/gemfiles/rails_7_0.gemfile.lock create mode 100644 packages/data_taster/lib/data_taster/collection.rb create mode 100644 packages/data_taster/lib/data_taster/confection.rb create mode 100644 packages/data_taster/lib/data_taster/detergent.rb create mode 100644 packages/data_taster/lib/data_taster/flavors.rb create mode 100644 packages/data_taster/lib/data_taster/helper.rb create mode 100644 packages/data_taster/lib/data_taster/sample.rb create mode 100644 packages/data_taster/lib/data_taster/sanitizer.rb diff --git a/packages/data_taster/.rubocop.yml b/packages/data_taster/.rubocop.yml index e3462a74..e9fe2e8e 100644 --- a/packages/data_taster/.rubocop.yml +++ b/packages/data_taster/.rubocop.yml @@ -1,13 +1,16 @@ +require: + - rubocop-powerhome + AllCops: - TargetRubyVersion: 2.6 + TargetRubyVersion: 2.7 -Style/StringLiterals: - Enabled: true - EnforcedStyle: double_quotes +Style/FrozenStringLiteralComment: + Exclude: + - 'gemfiles/*' -Style/StringLiteralsInInterpolation: - Enabled: true - EnforcedStyle: double_quotes +Bundler/OrderedGems: + Exclude: + - 'gemfiles/*' -Layout/LineLength: - Max: 120 +Rails: + Enabled: false diff --git a/packages/data_taster/Appraisals b/packages/data_taster/Appraisals new file mode 100644 index 00000000..895c313c --- /dev/null +++ b/packages/data_taster/Appraisals @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +appraise "rails-6-0" do + gem "rails", "6.0.6.1" +end + +appraise "rails-6-1" do + gem "rails", "6.1.7.4" +end + +appraise "rails-7-0" do + gem "rails", "7.0.6" +end diff --git a/packages/data_taster/Gemfile b/packages/data_taster/Gemfile index 5dfa6a1f..993fc496 100644 --- a/packages/data_taster/Gemfile +++ b/packages/data_taster/Gemfile @@ -2,9 +2,7 @@ source "https://rubygems.org" -# Specify your gem's dependencies in data_taster.gemspec gemspec -gem "rake", "~> 13.0" -gem "rspec", "~> 3.0" -gem "rubocop", "~> 1.21" +gem "nokogiri", "< 1.16" +gem "rubocop-powerhome", path: "../rubocop-powerhome" diff --git a/packages/data_taster/Gemfile.lock b/packages/data_taster/Gemfile.lock index a7a335a0..b8158fa5 100644 --- a/packages/data_taster/Gemfile.lock +++ b/packages/data_taster/Gemfile.lock @@ -1,33 +1,94 @@ +PATH + remote: ../rubocop-powerhome + specs: + rubocop-powerhome (0.5.2) + rubocop (~> 1.52.0) + rubocop-performance + rubocop-rails + rubocop-rake + rubocop-rspec + PATH remote: . specs: - data_taster (0.1.0) + data_taster (0.2.0) + rails (>= 6.0) GEM remote: https://rubygems.org/ specs: - activesupport (7.1.2) - base64 - bigdecimal + actioncable (6.0.6.1) + actionpack (= 6.0.6.1) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + actionmailbox (6.0.6.1) + actionpack (= 6.0.6.1) + activejob (= 6.0.6.1) + activerecord (= 6.0.6.1) + activestorage (= 6.0.6.1) + activesupport (= 6.0.6.1) + mail (>= 2.7.1) + actionmailer (6.0.6.1) + actionpack (= 6.0.6.1) + actionview (= 6.0.6.1) + activejob (= 6.0.6.1) + mail (~> 2.5, >= 2.5.4) + rails-dom-testing (~> 2.0) + actionpack (6.0.6.1) + actionview (= 6.0.6.1) + activesupport (= 6.0.6.1) + rack (~> 2.0, >= 2.0.8) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (6.0.6.1) + actionpack (= 6.0.6.1) + activerecord (= 6.0.6.1) + activestorage (= 6.0.6.1) + activesupport (= 6.0.6.1) + nokogiri (>= 1.8.5) + actionview (6.0.6.1) + activesupport (= 6.0.6.1) + builder (~> 3.1) + erubi (~> 1.4) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activejob (6.0.6.1) + activesupport (= 6.0.6.1) + globalid (>= 0.3.6) + activemodel (6.0.6.1) + activesupport (= 6.0.6.1) + activerecord (6.0.6.1) + activemodel (= 6.0.6.1) + activesupport (= 6.0.6.1) + activestorage (6.0.6.1) + actionpack (= 6.0.6.1) + activejob (= 6.0.6.1) + activerecord (= 6.0.6.1) + marcel (~> 1.0) + activesupport (6.0.6.1) concurrent-ruby (~> 1.0, >= 1.0.2) - connection_pool (>= 2.2.5) - drb - i18n (>= 1.6, < 2) - minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + zeitwerk (~> 2.2, >= 2.2.2) + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) ast (2.4.2) - base64 (0.2.0) - bigdecimal (3.1.5) + builder (3.2.4) concurrent-ruby (1.2.2) - connection_pool (2.4.1) + crass (1.0.6) + date (3.3.4) diff-lcs (1.5.0) - drb (2.2.0) - ruby2_keywords + docile (1.1.5) + erubi (1.12.0) + globalid (1.1.0) + activesupport (>= 5.0) i18n (1.14.1) concurrent-ruby (~> 1.0) - json (2.7.1) - language_server-protocol (3.17.0.3) + json (2.6.3) license_finder (7.1.0) bundler rubyzip (>= 1, < 3) @@ -35,16 +96,78 @@ GEM tomlrb (>= 1.3, < 2.1) with_env (= 1.1.0) xml-simple (~> 1.1.9) - minitest (5.20.0) - mutex_m (0.2.0) - parallel (1.24.0) - parser (3.2.2.4) + loofah (2.22.0) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + mail (2.8.1) + mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp + marcel (1.0.2) + method_source (1.0.0) + mini_mime (1.1.5) + mini_portile2 (2.8.5) + minitest (5.18.0) + net-imap (0.4.9) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.4.0) + net-protocol + nio4r (2.7.0) + nokogiri (1.15.5) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.15.5-arm64-darwin) + racc (~> 1.4) + nokogiri (1.15.5-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.15.5-x86_64-linux) + racc (~> 1.4) + parallel (1.23.0) + parser (3.2.2.3) ast (~> 2.4.1) racc - racc (1.7.3) - rack (3.0.8) - rainbow (3.1.1) - rake (13.1.0) + power_assert (2.0.3) + racc (1.7.1) + rack (2.2.7) + rack-test (2.1.0) + rack (>= 1.3) + rails (6.0.6.1) + actioncable (= 6.0.6.1) + actionmailbox (= 6.0.6.1) + actionmailer (= 6.0.6.1) + actionpack (= 6.0.6.1) + actiontext (= 6.0.6.1) + actionview (= 6.0.6.1) + activejob (= 6.0.6.1) + activemodel (= 6.0.6.1) + activerecord (= 6.0.6.1) + activestorage (= 6.0.6.1) + activesupport (= 6.0.6.1) + bundler (>= 1.3.0) + railties (= 6.0.6.1) + sprockets-rails (>= 2.0.0) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) + railties (6.0.6.1) + actionpack (= 6.0.6.1) + activesupport (= 6.0.6.1) + method_source + rake (>= 0.8.7) + thor (>= 0.20.3, < 2.0) + rainbow (2.2.2) + rake + rake (13.0.6) regexp_parser (2.8.3) rexml (3.2.6) rspec (3.12.0) @@ -56,19 +179,18 @@ GEM rspec-expectations (3.12.3) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) - rspec-mocks (3.12.6) + rspec-mocks (3.12.5) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) - rspec-support (3.12.1) - rubocop (1.59.0) + rspec-support (3.12.0) + rubocop (1.52.1) json (~> 2.3) - language_server-protocol (>= 3.17.0) parallel (~> 1.10) - parser (>= 3.2.2.4) + parser (>= 3.2.2.3) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.28.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) rubocop-ast (1.30.0) @@ -77,16 +199,10 @@ GEM rubocop (~> 1.41) rubocop-factory_bot (2.24.0) rubocop (~> 1.33) - rubocop-performance (1.20.1) - rubocop (>= 1.48.1, < 2.0) - rubocop-ast (>= 1.30.0, < 2.0) - rubocop-powerhome (0.5.0) - rubocop - rubocop-performance - rubocop-rails - rubocop-rake - rubocop-rspec - rubocop-rails (2.23.1) + rubocop-performance (1.19.1) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.22.2) activesupport (>= 4.2.0) rack (>= 1.1) rubocop (>= 1.33.0, < 2.0) @@ -98,28 +214,63 @@ GEM rubocop-capybara (~> 2.17) rubocop-factory_bot (~> 2.22) ruby-progressbar (1.13.0) - ruby2_keywords (0.0.5) rubyzip (2.3.2) - thor (1.3.0) - tomlrb (2.0.3) - tzinfo (2.0.6) + simplecov (0.15.1) + docile (~> 1.1.0) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.2) + sprockets (4.2.1) concurrent-ruby (~> 1.0) + rack (>= 2.2.4, < 4) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) + sprockets (>= 3.0.0) + test-unit (3.1.5) + power_assert + thor (1.2.2) + thread_safe (0.3.6) + timeout (0.4.1) + tomlrb (2.0.3) + tzinfo (1.2.11) + thread_safe (~> 0.1) unicode-display_width (2.5.0) + websocket-driver (0.7.6) + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) with_env (1.1.0) xml-simple (1.1.9) rexml + yard (0.9.34) + zeitwerk (2.6.8) PLATFORMS + arm64-darwin-20 + arm64-darwin-21 arm64-darwin-22 + arm64-darwin-23 + ruby + x86_64-darwin-20 + x86_64-darwin-21 + x86_64-darwin-22 + x86_64-darwin-23 x86_64-linux DEPENDENCIES + appraisal (= 2.5.0) + bundler (~> 2.1) data_taster! license_finder (~> 7.0) + nokogiri (< 1.16) + parser (>= 2.5, != 2.5.1.1) + rainbow (= 2.2.2) rake (~> 13.0) rspec (~> 3.0) - rubocop (~> 1.21) - rubocop-powerhome (= 0.5.0) + rubocop-powerhome! + simplecov (= 0.15.1) + test-unit (= 3.1.5) + yard (= 0.9.34) BUNDLED WITH - 2.4.10 + 2.4.22 diff --git a/packages/data_taster/bin/console b/packages/data_taster/bin/console deleted file mode 100755 index 7f0bf70c..00000000 --- a/packages/data_taster/bin/console +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require "bundler/setup" -require "data_taster" - -# You can add fixtures and/or initialization code here to make experimenting -# with your gem easier. You can also use a different console, if you like. - -# (If you use this, don't forget to add pry to your Gemfile!) -# require "pry" -# Pry.start - -require "irb" -IRB.start(__FILE__) diff --git a/packages/data_taster/bin/setup b/packages/data_taster/bin/setup deleted file mode 100755 index dce67d86..00000000 --- a/packages/data_taster/bin/setup +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -IFS=$'\n\t' -set -vx - -bundle install - -# Do any other automated setup that you need to do here diff --git a/packages/data_taster/data_taster.gemspec b/packages/data_taster/data_taster.gemspec index b7f0b9a2..8cdfbeb3 100644 --- a/packages/data_taster/data_taster.gemspec +++ b/packages/data_taster/data_taster.gemspec @@ -3,16 +3,16 @@ require_relative "lib/data_taster/version" Gem::Specification.new do |spec| - spec.name = "data_taster" - spec.version = DataTaster::VERSION - spec.authors = ["Jill Klang"] - spec.email = ["jillian.emilie@gmail.com"] + spec.name = "data_taster" + spec.version = DataTaster::VERSION + spec.authors = ["Jill Klang"] + spec.email = ["jillian.emilie@gmail.com"] spec.summary = "Delicious and sanitized data samples for development and testing." spec.description = "Export, sanitize, and import data to help develop better apps." spec.homepage = "https://github.com/powerhome/power-tools" spec.license = "MIT" - spec.required_ruby_version = ">= 2.6.0" + spec.required_ruby_version = ">= 2.7" spec.metadata["rubygems_mfa_required"] = "true" spec.metadata["homepage_uri"] = spec.homepage @@ -30,7 +30,18 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] - spec.add_development_dependency "license_finder", "~> 7.0" + spec.add_dependency "rails", ">= 6.0" + spec.add_development_dependency "license_finder", "~> 7.0" spec.add_development_dependency "rubocop-powerhome", "0.5.0" + + spec.add_development_dependency "appraisal", "2.5.0" + spec.add_development_dependency "bundler", "~> 2.1" + spec.add_development_dependency "parser", ">= 2.5", "!= 2.5.1.1" + spec.add_development_dependency "rainbow", "2.2.2" + spec.add_development_dependency "rake", "~> 13.0" + spec.add_development_dependency "rspec", "~> 3.0" + spec.add_development_dependency "simplecov", "0.15.1" + spec.add_development_dependency "test-unit", "3.1.5" + spec.add_development_dependency "yard", "0.9.34" end diff --git a/packages/data_taster/doc/dependency_decisions.yml b/packages/data_taster/doc/dependency_decisions.yml index f734baa9..12b1b0cb 100644 --- a/packages/data_taster/doc/dependency_decisions.yml +++ b/packages/data_taster/doc/dependency_decisions.yml @@ -1,3 +1,9 @@ --- - - :inherit_from - https://raw.githubusercontent.com/powerhome/oss-guide/master/license_rules.yml +- - :permit + - MIT AND (BSD-2-Clause OR GPL-2.0-or-later) + - :who: + :why: + :versions: [] + :when: 2024-01-04 20:18:22.141453000 Z diff --git a/packages/data_taster/docs/CHANGELOG.md b/packages/data_taster/docs/CHANGELOG.md index 701cd4c9..8ce68de8 100644 --- a/packages/data_taster/docs/CHANGELOG.md +++ b/packages/data_taster/docs/CHANGELOG.md @@ -1 +1,9 @@ ## [Unreleased] + +## [0.2.0] - 2023-12-24 + +- Rollout of basic features. See README for details. + +## [0.1.0] - 2023-05-30 + +- Initial release diff --git a/packages/data_taster/docs/README.md b/packages/data_taster/docs/README.md index 997c2dc9..acf2b755 100644 --- a/packages/data_taster/docs/README.md +++ b/packages/data_taster/docs/README.md @@ -1,3 +1,75 @@ # 🍭 Data Taster -More info coming soon +Database exporting tool. + +## Usage + +### `data_taster_export_tables.yml` + +With DataTaster, you can configure certain tables and rows to be pulled from the source_db. Look for (or create) a `data_taster_export_tables.yml` file anywhere in your repo (for example, this could live under `config/` or `db/`). + +By default, any table that has nothing specified in the files will not have any data selected from the source client. + +Each key should represent the table name that you are defining rules for. If you don't have any custom rows to sanitize, the value should be a clause that will be wrapped in a MySQL WHERE clause. DataTaster will automatically look for any files with that name and include them in the export. + +```ruby +# Example, simple usage + +sales_summaries: quarter_start_date >= '<%= Date.current.last_year.beginning_of_quarter %>' +``` + +If you have columns that you need to sanitize, you'll need to specify the column names and values along with the selections. Note that many things get sanitized by default. + +```ruby +# Example, with sanitization specified + +sales_summaries: + select: "quarter_start_date >= '<%= Date.current.last_year.beginning_of_quarter %>'" + sanitize: + comp_total: 9999.99 +``` + +### Default Sanitization + +To help protect data, DataTaster scans columns that it's importing against of blocklist of terms that it identifies as high risk (you can see the full list of default sanitizations at `lib/data_taster/sanitizer.rb`.). If the term is a match against its list and there is no custom sanitization defined, it will attempt to sanitize the column. If you do not need to sanitize that column, you can always skip it: + +```ruby +users: + select: "activated_at IS NOT NULL" + sanitize: + retain_email: "<%= skip_sanitization %>" +``` + +### `DataTaster.sample!` + +DataTaster uses its [Sample](https://github.com/powerhome/nitro-web/blob/master/components/data_taster/lib/data_taster/sample.rb) class to load the yml files, filtered through erb methods provided through its [Flavors](https://github.com/powerhome/nitro-web/blob/master/components/data_taster/lib/data_taster/flavors.rb) class. + +Before doing so, you'll need to specify a client to use to connect to mysql and make queries. Doing so can be done by calling `DataTaster.config`, like so: + +```ruby +# Example, using all defaults + + DataTaster.config( + source_client: Mysql2::Client.new(db_config), + working_client: Mysql2::Client.new(db_config), + ) +``` + +This method takes several optional arguments for further configuration of data selection. The list of arguments and their defaults can be found at `components/data_taster/lib/data_taster.rb`. + +When you are ready to populate your database with the selected data, you can pass `include_insert: true` to this method call so that it returns INSERT statements. Otherwise, it will only return SELECT queries, useful for testing and debugging purposes. + +Asking for samples without previously configuring db clients will result in errors. However, once you have done so, sampling is easy: + +```ruby +# Example, using all defaults + +DataTaster.sample! +``` + +This will select data from the source_client and insert it using the working_client. + + + #### Deprecating Tables + + DataTaster queries directly on the source_client to retrieve the list of tables to work on, which can cause issues when tables are removed -- migrations that have run in one environment may not have yet run in another. In order to maintain smoother transitions, use the `deprecated_table` method found in the DataTaster#Flavors file to mark the table as one that should include neither the schema nor the data. diff --git a/packages/data_taster/gemfiles/rails_6_0.gemfile b/packages/data_taster/gemfiles/rails_6_0.gemfile new file mode 100644 index 00000000..fc00cb5a --- /dev/null +++ b/packages/data_taster/gemfiles/rails_6_0.gemfile @@ -0,0 +1,8 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "rubocop-powerhome", path: "../../rubocop-powerhome" +gem "rails", "6.0.6.1" + +gemspec path: "../" diff --git a/packages/data_taster/gemfiles/rails_6_0.gemfile.lock b/packages/data_taster/gemfiles/rails_6_0.gemfile.lock new file mode 100644 index 00000000..8b367e3d --- /dev/null +++ b/packages/data_taster/gemfiles/rails_6_0.gemfile.lock @@ -0,0 +1,258 @@ +PATH + remote: ../../rubocop-powerhome + specs: + rubocop-powerhome (0.5.2) + rubocop (~> 1.52.0) + rubocop-performance + rubocop-rails + rubocop-rake + rubocop-rspec + +PATH + remote: .. + specs: + data_taster (0.2.0) + +GEM + remote: https://rubygems.org/ + specs: + actioncable (6.0.6.1) + actionpack (= 6.0.6.1) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + actionmailbox (6.0.6.1) + actionpack (= 6.0.6.1) + activejob (= 6.0.6.1) + activerecord (= 6.0.6.1) + activestorage (= 6.0.6.1) + activesupport (= 6.0.6.1) + mail (>= 2.7.1) + actionmailer (6.0.6.1) + actionpack (= 6.0.6.1) + actionview (= 6.0.6.1) + activejob (= 6.0.6.1) + mail (~> 2.5, >= 2.5.4) + rails-dom-testing (~> 2.0) + actionpack (6.0.6.1) + actionview (= 6.0.6.1) + activesupport (= 6.0.6.1) + rack (~> 2.0, >= 2.0.8) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (6.0.6.1) + actionpack (= 6.0.6.1) + activerecord (= 6.0.6.1) + activestorage (= 6.0.6.1) + activesupport (= 6.0.6.1) + nokogiri (>= 1.8.5) + actionview (6.0.6.1) + activesupport (= 6.0.6.1) + builder (~> 3.1) + erubi (~> 1.4) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activejob (6.0.6.1) + activesupport (= 6.0.6.1) + globalid (>= 0.3.6) + activemodel (6.0.6.1) + activesupport (= 6.0.6.1) + activerecord (6.0.6.1) + activemodel (= 6.0.6.1) + activesupport (= 6.0.6.1) + activestorage (6.0.6.1) + actionpack (= 6.0.6.1) + activejob (= 6.0.6.1) + activerecord (= 6.0.6.1) + marcel (~> 1.0) + activesupport (6.0.6.1) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + zeitwerk (~> 2.2, >= 2.2.2) + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + builder (3.2.4) + concurrent-ruby (1.2.2) + crass (1.0.6) + date (3.3.4) + diff-lcs (1.5.0) + docile (1.1.5) + erubi (1.12.0) + globalid (1.1.0) + activesupport (>= 5.0) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.7.1) + license_finder (7.1.0) + bundler + rubyzip (>= 1, < 3) + thor (~> 1.2) + tomlrb (>= 1.3, < 2.1) + with_env (= 1.1.0) + xml-simple (~> 1.1.9) + loofah (2.22.0) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + mail (2.8.1) + mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp + marcel (1.0.2) + method_source (1.0.0) + mini_mime (1.1.5) + minitest (5.20.0) + net-imap (0.4.9) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.4.0) + net-protocol + nio4r (2.7.0) + nokogiri (1.16.0-arm64-darwin) + racc (~> 1.4) + parallel (1.24.0) + parser (3.2.2.4) + ast (~> 2.4.1) + racc + power_assert (2.0.3) + racc (1.7.3) + rack (2.2.8) + rack-test (2.1.0) + rack (>= 1.3) + rails (6.0.6.1) + actioncable (= 6.0.6.1) + actionmailbox (= 6.0.6.1) + actionmailer (= 6.0.6.1) + actionpack (= 6.0.6.1) + actiontext (= 6.0.6.1) + actionview (= 6.0.6.1) + activejob (= 6.0.6.1) + activemodel (= 6.0.6.1) + activerecord (= 6.0.6.1) + activestorage (= 6.0.6.1) + activesupport (= 6.0.6.1) + bundler (>= 1.3.0) + railties (= 6.0.6.1) + sprockets-rails (>= 2.0.0) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) + railties (6.0.6.1) + actionpack (= 6.0.6.1) + activesupport (= 6.0.6.1) + method_source + rake (>= 0.8.7) + thor (>= 0.20.3, < 2.0) + rainbow (2.2.2) + rake + rake (13.1.0) + regexp_parser (2.8.3) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rubocop (1.52.1) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.2.3) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.20.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.24.0) + rubocop (~> 1.33) + rubocop-performance (1.20.1) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + rubocop-rails (2.23.1) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + rubocop-rake (0.6.0) + rubocop (~> 1.0) + rubocop-rspec (2.25.0) + rubocop (~> 1.40) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + rubyzip (2.3.2) + simplecov (0.15.1) + docile (~> 1.1.0) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.2) + sprockets (4.2.1) + concurrent-ruby (~> 1.0) + rack (>= 2.2.4, < 4) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) + sprockets (>= 3.0.0) + test-unit (3.1.5) + power_assert + thor (1.3.0) + thread_safe (0.3.6) + timeout (0.4.1) + tomlrb (2.0.3) + tzinfo (1.2.11) + thread_safe (~> 0.1) + unicode-display_width (2.5.0) + websocket-driver (0.7.6) + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) + with_env (1.1.0) + xml-simple (1.1.9) + rexml + yard (0.9.34) + zeitwerk (2.6.12) + +PLATFORMS + arm64-darwin-22 + +DEPENDENCIES + appraisal (= 2.5.0) + bundler (~> 2.1) + data_taster! + license_finder (~> 7.0) + parser (>= 2.5, != 2.5.1.1) + rails (= 6.0.6.1) + rainbow (= 2.2.2) + rake (~> 13.0) + rspec (~> 3.0) + rubocop-powerhome! + simplecov (= 0.15.1) + test-unit (= 3.1.5) + yard (= 0.9.34) + +BUNDLED WITH + 2.4.22 diff --git a/packages/data_taster/gemfiles/rails_6_1.gemfile b/packages/data_taster/gemfiles/rails_6_1.gemfile new file mode 100644 index 00000000..c060fe7e --- /dev/null +++ b/packages/data_taster/gemfiles/rails_6_1.gemfile @@ -0,0 +1,8 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "rubocop-powerhome", path: "../../rubocop-powerhome" +gem "rails", "6.1.7.4" + +gemspec path: "../" diff --git a/packages/data_taster/gemfiles/rails_6_1.gemfile.lock b/packages/data_taster/gemfiles/rails_6_1.gemfile.lock new file mode 100644 index 00000000..df6636d0 --- /dev/null +++ b/packages/data_taster/gemfiles/rails_6_1.gemfile.lock @@ -0,0 +1,261 @@ +PATH + remote: ../../rubocop-powerhome + specs: + rubocop-powerhome (0.5.2) + rubocop (~> 1.52.0) + rubocop-performance + rubocop-rails + rubocop-rake + rubocop-rspec + +PATH + remote: .. + specs: + data_taster (0.2.0) + +GEM + remote: https://rubygems.org/ + specs: + actioncable (6.1.7.4) + actionpack (= 6.1.7.4) + activesupport (= 6.1.7.4) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + actionmailbox (6.1.7.4) + actionpack (= 6.1.7.4) + activejob (= 6.1.7.4) + activerecord (= 6.1.7.4) + activestorage (= 6.1.7.4) + activesupport (= 6.1.7.4) + mail (>= 2.7.1) + actionmailer (6.1.7.4) + actionpack (= 6.1.7.4) + actionview (= 6.1.7.4) + activejob (= 6.1.7.4) + activesupport (= 6.1.7.4) + mail (~> 2.5, >= 2.5.4) + rails-dom-testing (~> 2.0) + actionpack (6.1.7.4) + actionview (= 6.1.7.4) + activesupport (= 6.1.7.4) + rack (~> 2.0, >= 2.0.9) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (6.1.7.4) + actionpack (= 6.1.7.4) + activerecord (= 6.1.7.4) + activestorage (= 6.1.7.4) + activesupport (= 6.1.7.4) + nokogiri (>= 1.8.5) + actionview (6.1.7.4) + activesupport (= 6.1.7.4) + builder (~> 3.1) + erubi (~> 1.4) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activejob (6.1.7.4) + activesupport (= 6.1.7.4) + globalid (>= 0.3.6) + activemodel (6.1.7.4) + activesupport (= 6.1.7.4) + activerecord (6.1.7.4) + activemodel (= 6.1.7.4) + activesupport (= 6.1.7.4) + activestorage (6.1.7.4) + actionpack (= 6.1.7.4) + activejob (= 6.1.7.4) + activerecord (= 6.1.7.4) + activesupport (= 6.1.7.4) + marcel (~> 1.0) + mini_mime (>= 1.1.0) + activesupport (6.1.7.4) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + builder (3.2.4) + concurrent-ruby (1.2.2) + crass (1.0.6) + date (3.3.4) + diff-lcs (1.5.0) + docile (1.1.5) + erubi (1.12.0) + globalid (1.2.1) + activesupport (>= 6.1) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.7.1) + license_finder (7.1.0) + bundler + rubyzip (>= 1, < 3) + thor (~> 1.2) + tomlrb (>= 1.3, < 2.1) + with_env (= 1.1.0) + xml-simple (~> 1.1.9) + loofah (2.22.0) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + mail (2.8.1) + mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp + marcel (1.0.2) + method_source (1.0.0) + mini_mime (1.1.5) + minitest (5.20.0) + net-imap (0.4.9) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.4.0) + net-protocol + nio4r (2.7.0) + nokogiri (1.16.0-arm64-darwin) + racc (~> 1.4) + parallel (1.24.0) + parser (3.2.2.4) + ast (~> 2.4.1) + racc + power_assert (2.0.3) + racc (1.7.3) + rack (2.2.8) + rack-test (2.1.0) + rack (>= 1.3) + rails (6.1.7.4) + actioncable (= 6.1.7.4) + actionmailbox (= 6.1.7.4) + actionmailer (= 6.1.7.4) + actionpack (= 6.1.7.4) + actiontext (= 6.1.7.4) + actionview (= 6.1.7.4) + activejob (= 6.1.7.4) + activemodel (= 6.1.7.4) + activerecord (= 6.1.7.4) + activestorage (= 6.1.7.4) + activesupport (= 6.1.7.4) + bundler (>= 1.15.0) + railties (= 6.1.7.4) + sprockets-rails (>= 2.0.0) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) + railties (6.1.7.4) + actionpack (= 6.1.7.4) + activesupport (= 6.1.7.4) + method_source + rake (>= 12.2) + thor (~> 1.0) + rainbow (2.2.2) + rake + rake (13.1.0) + regexp_parser (2.8.3) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rubocop (1.52.1) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.2.3) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.20.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.24.0) + rubocop (~> 1.33) + rubocop-performance (1.20.1) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + rubocop-rails (2.23.1) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + rubocop-rake (0.6.0) + rubocop (~> 1.0) + rubocop-rspec (2.25.0) + rubocop (~> 1.40) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + rubyzip (2.3.2) + simplecov (0.15.1) + docile (~> 1.1.0) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.2) + sprockets (4.2.1) + concurrent-ruby (~> 1.0) + rack (>= 2.2.4, < 4) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) + sprockets (>= 3.0.0) + test-unit (3.1.5) + power_assert + thor (1.3.0) + timeout (0.4.1) + tomlrb (2.0.3) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.5.0) + websocket-driver (0.7.6) + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) + with_env (1.1.0) + xml-simple (1.1.9) + rexml + yard (0.9.34) + zeitwerk (2.6.12) + +PLATFORMS + arm64-darwin-22 + +DEPENDENCIES + appraisal (= 2.5.0) + bundler (~> 2.1) + data_taster! + license_finder (~> 7.0) + parser (>= 2.5, != 2.5.1.1) + rails (= 6.1.7.4) + rainbow (= 2.2.2) + rake (~> 13.0) + rspec (~> 3.0) + rubocop-powerhome! + simplecov (= 0.15.1) + test-unit (= 3.1.5) + yard (= 0.9.34) + +BUNDLED WITH + 2.4.22 diff --git a/packages/data_taster/gemfiles/rails_7_0.gemfile b/packages/data_taster/gemfiles/rails_7_0.gemfile new file mode 100644 index 00000000..9d400f3f --- /dev/null +++ b/packages/data_taster/gemfiles/rails_7_0.gemfile @@ -0,0 +1,8 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "rubocop-powerhome", path: "../../rubocop-powerhome" +gem "rails", "7.0.6" + +gemspec path: "../" diff --git a/packages/data_taster/gemfiles/rails_7_0.gemfile.lock b/packages/data_taster/gemfiles/rails_7_0.gemfile.lock new file mode 100644 index 00000000..7fc6294c --- /dev/null +++ b/packages/data_taster/gemfiles/rails_7_0.gemfile.lock @@ -0,0 +1,260 @@ +PATH + remote: ../../rubocop-powerhome + specs: + rubocop-powerhome (0.5.2) + rubocop (~> 1.52.0) + rubocop-performance + rubocop-rails + rubocop-rake + rubocop-rspec + +PATH + remote: .. + specs: + data_taster (0.2.0) + +GEM + remote: https://rubygems.org/ + specs: + actioncable (7.0.6) + actionpack (= 7.0.6) + activesupport (= 7.0.6) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + actionmailbox (7.0.6) + actionpack (= 7.0.6) + activejob (= 7.0.6) + activerecord (= 7.0.6) + activestorage (= 7.0.6) + activesupport (= 7.0.6) + mail (>= 2.7.1) + net-imap + net-pop + net-smtp + actionmailer (7.0.6) + actionpack (= 7.0.6) + actionview (= 7.0.6) + activejob (= 7.0.6) + activesupport (= 7.0.6) + mail (~> 2.5, >= 2.5.4) + net-imap + net-pop + net-smtp + rails-dom-testing (~> 2.0) + actionpack (7.0.6) + actionview (= 7.0.6) + activesupport (= 7.0.6) + rack (~> 2.0, >= 2.2.4) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (7.0.6) + actionpack (= 7.0.6) + activerecord (= 7.0.6) + activestorage (= 7.0.6) + activesupport (= 7.0.6) + globalid (>= 0.6.0) + nokogiri (>= 1.8.5) + actionview (7.0.6) + activesupport (= 7.0.6) + builder (~> 3.1) + erubi (~> 1.4) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activejob (7.0.6) + activesupport (= 7.0.6) + globalid (>= 0.3.6) + activemodel (7.0.6) + activesupport (= 7.0.6) + activerecord (7.0.6) + activemodel (= 7.0.6) + activesupport (= 7.0.6) + activestorage (7.0.6) + actionpack (= 7.0.6) + activejob (= 7.0.6) + activerecord (= 7.0.6) + activesupport (= 7.0.6) + marcel (~> 1.0) + mini_mime (>= 1.1.0) + activesupport (7.0.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + builder (3.2.4) + concurrent-ruby (1.2.2) + crass (1.0.6) + date (3.3.4) + diff-lcs (1.5.0) + docile (1.1.5) + erubi (1.12.0) + globalid (1.2.1) + activesupport (>= 6.1) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.7.1) + license_finder (7.1.0) + bundler + rubyzip (>= 1, < 3) + thor (~> 1.2) + tomlrb (>= 1.3, < 2.1) + with_env (= 1.1.0) + xml-simple (~> 1.1.9) + loofah (2.22.0) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + mail (2.8.1) + mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp + marcel (1.0.2) + method_source (1.0.0) + mini_mime (1.1.5) + minitest (5.20.0) + net-imap (0.4.9) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.4.0) + net-protocol + nio4r (2.7.0) + nokogiri (1.16.0-arm64-darwin) + racc (~> 1.4) + parallel (1.24.0) + parser (3.2.2.4) + ast (~> 2.4.1) + racc + power_assert (2.0.3) + racc (1.7.3) + rack (2.2.8) + rack-test (2.1.0) + rack (>= 1.3) + rails (7.0.6) + actioncable (= 7.0.6) + actionmailbox (= 7.0.6) + actionmailer (= 7.0.6) + actionpack (= 7.0.6) + actiontext (= 7.0.6) + actionview (= 7.0.6) + activejob (= 7.0.6) + activemodel (= 7.0.6) + activerecord (= 7.0.6) + activestorage (= 7.0.6) + activesupport (= 7.0.6) + bundler (>= 1.15.0) + railties (= 7.0.6) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) + railties (7.0.6) + actionpack (= 7.0.6) + activesupport (= 7.0.6) + method_source + rake (>= 12.2) + thor (~> 1.0) + zeitwerk (~> 2.5) + rainbow (2.2.2) + rake + rake (13.1.0) + regexp_parser (2.8.3) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rubocop (1.52.1) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.2.3) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.20.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.24.0) + rubocop (~> 1.33) + rubocop-performance (1.20.1) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + rubocop-rails (2.23.1) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + rubocop-rake (0.6.0) + rubocop (~> 1.0) + rubocop-rspec (2.25.0) + rubocop (~> 1.40) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + rubyzip (2.3.2) + simplecov (0.15.1) + docile (~> 1.1.0) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.2) + test-unit (3.1.5) + power_assert + thor (1.3.0) + timeout (0.4.1) + tomlrb (2.0.3) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.5.0) + websocket-driver (0.7.6) + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) + with_env (1.1.0) + xml-simple (1.1.9) + rexml + yard (0.9.34) + zeitwerk (2.6.12) + +PLATFORMS + arm64-darwin-22 + +DEPENDENCIES + appraisal (= 2.5.0) + bundler (~> 2.1) + data_taster! + license_finder (~> 7.0) + parser (>= 2.5, != 2.5.1.1) + rails (= 7.0.6) + rainbow (= 2.2.2) + rake (~> 13.0) + rspec (~> 3.0) + rubocop-powerhome! + simplecov (= 0.15.1) + test-unit (= 3.1.5) + yard (= 0.9.34) + +BUNDLED WITH + 2.4.22 diff --git a/packages/data_taster/lib/data_taster.rb b/packages/data_taster/lib/data_taster.rb index 63c133d4..d6cf7064 100644 --- a/packages/data_taster/lib/data_taster.rb +++ b/packages/data_taster/lib/data_taster.rb @@ -1,8 +1,59 @@ # frozen_string_literal: true -require_relative "data_taster/version" +require "logger" module DataTaster - class Error < StandardError; end - # Your code goes here... + autoload :Collection, "data_taster/collection" + autoload :Confection, "data_taster/confection" + autoload :Detergent, "data_taster/detergent" + autoload :Helper, "data_taster/helper" + autoload :Sample, "data_taster/sample" + autoload :Sanitizer, "data_taster/sanitizer" + + SKIP_CODE = "skip_processing" + + def self.logger=(logger) + @logger = logger + end + + def self.logger + @logger ||= Logger.new($stdout) + end + + def self.config(**args) + @config ||= Config.new( + args[:months], + Array.wrap(args[:list] || Rails.root.glob("**/data_taster_export_tables.yml")), + args[:source_client] || raise(ArgumentError, "DataTaster.config missing source_client"), + args[:working_client] || raise(ArgumentError, "DataTaster.config missing working_client"), + args[:include_insert] || false + ) + end + + def self.confection + @confection ||= DataTaster::Confection.new.assemble + end + + def self.sample! + DataTaster + .config + .source_client + .query("SHOW tables").collect { |t| t[t.keys.first] } + .each do |table_name| + DataTaster::Sample.new(table_name).serve! + end + end + + def self.safe_execute(sql, client = DataTaster.config.working_client) + foreign_key_check = client.query("SELECT @@FOREIGN_KEY_CHECKS").first["@@FOREIGN_KEY_CHECKS"] + + begin + client.query("SET FOREIGN_KEY_CHECKS=0") + client.query(sql) + ensure + client.query("SET FOREIGN_KEY_CHECKS=#{foreign_key_check};") + end + end + + Config = Struct.new(:months, :list, :source_client, :working_client, :include_insert) end diff --git a/packages/data_taster/lib/data_taster/collection.rb b/packages/data_taster/lib/data_taster/collection.rb new file mode 100644 index 00000000..c4fafa3e --- /dev/null +++ b/packages/data_taster/lib/data_taster/collection.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module DataTaster + # Ingests the processed yml file and returns either + # an empty hash (for cases that are skippable) + # or a hash that contains: + # select: + # (whatever is configured to go in the SELECT .. WHERE clause) + # sanitize: + # the columns and values that need custom sanitization + class Collection + def initialize(table_name) + @table_name = table_name + @ingredients = DataTaster.confection[table_name] + @include_insert = DataTaster.config.include_insert + end + + def assemble + DataTaster.logger.info("#{table_name}...") + + if skippable? + DataTaster.logger.info("configured to skip both schema and data") + {} + else + { select: selection, sanitize: sanitization } + end + end + + private + + attr_reader :table_name, :ingredients, :include_insert + + def skippable? + table_name.downcase.match(/^_/) || + ingredients == DataTaster::SKIP_CODE + end + + def selection + insert = include_insert ? "INSERT INTO #{working_db}.#{table_name}" : "" + + sql = <<-SQL.squish + #{insert} + SELECT * FROM #{source_db}.#{table_name} + WHERE #{where_clause} + SQL + + DataTaster.logger.info(sql) + sql + end + + # The yml file allows you to define either a simple clause + # or some more fine-grained sanitization. If neither is + # defined, we pass a clause that selects nothing. + def where_clause + clause = ingredients.is_a?(Hash) ? ingredients["select"] : ingredients + + clause || "1 = 0" + end + + def sanitization + return unless ingredients.is_a?(Hash) + + ingredients["sanitize"] + end + + def source_db + @source_db ||= DataTaster.config.source_client.query_options[:database] + end + + def working_db + @working_db ||= DataTaster.config.working_client.query_options[:database] + end + end +end diff --git a/packages/data_taster/lib/data_taster/confection.rb b/packages/data_taster/lib/data_taster/confection.rb new file mode 100644 index 00000000..2fee275f --- /dev/null +++ b/packages/data_taster/lib/data_taster/confection.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "erb" +require "socket" +require "yaml" +require "data_taster/flavors" + +module DataTaster + # Ingests the list of data_taster_export_tables.yml files + # and processes them through an erb template + # returns a ruby hash of the data + class Confection + def assemble + DataTaster.config.list.each_with_object(default_data) do |path, merged_list| + merged_list.merge!(load_yml(path.to_s)) + end + end + + def load_yml(filename) + return {} unless File.exist?(filename) + + erb = ::ERB.new(File.read(filename)) + erb.filename = filename + flavored_erb = erb.def_class(DataTaster::Flavors, "render()") + erb_result = flavored_erb.new.render + + YAML.safe_load(erb_result.gsub(/((.|\n)*---)/, "\n---")) || {} + end + + private + + def default_data + { + "schema_migrations" => "1 = 1", + } + end + end +end diff --git a/packages/data_taster/lib/data_taster/detergent.rb b/packages/data_taster/lib/data_taster/detergent.rb new file mode 100644 index 00000000..b99fef7a --- /dev/null +++ b/packages/data_taster/lib/data_taster/detergent.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module DataTaster + # Returns SQL for given data, based on types. Used for sanitizing inputs. + class Detergent + SANITIZE_FUNCTIONS = [ + /CONCAT/, + /DATE/, + /DAY/, + /FORMAT/, + /LOWER/, + /REPLACE/, + /TRIM/, + /UCASE/, + /UPPER/, + ].freeze + + def initialize(table_name, column_name, given_value) + @table_name = table_name + @column_name = column_name + @value = parse_value(given_value) + end + + def deliver + return value if value == DataTaster::SKIP_CODE + + sql = sql_for(value) + + DataTaster.logger.info("--> #{sql}") + sql + end + + private + + attr_reader :table_name, :column_name, :value + + def parse_value(given_value) + # yml files can't hold custom-set dates, they have to be converted to strings + return given_value unless given_value.is_a?(String) && given_value.match?(/\d{4}-\d{2}-\d{2}/) + + Date.parse(given_value) + end + + def sql_for(value) + if value.is_a?(Date) + sql_for_date_value + elsif value.is_a?(Numeric) || sanitize_function? + sql_for_uncast_value + elsif value.blank? + sql_for_nil_value + else + sql_for_cast_value + end + end + + def sanitize_function? + SANITIZE_FUNCTIONS.any? { |fun| value.to_s.match(fun) } + end + + def sql_for_uncast_value + <<-SQL.squish + UPDATE #{working_db}.#{table_name} + SET #{column_name} = #{value} + WHERE #{column_name} IS NOT NULL + AND #{column_name} <> #{value} + SQL + end + + def sql_for_date_value + <<-SQL.squish + UPDATE #{working_db}.#{table_name} + SET #{column_name} = '#{value}' + WHERE #{column_name} IS NOT NULL + SQL + end + + def sql_for_nil_value + <<-SQL.squish + UPDATE #{working_db}.#{table_name} + SET #{column_name} = NULL + WHERE #{column_name} IS NOT NULL + SQL + end + + def sql_for_cast_value + <<-SQL.squish + UPDATE #{working_db}.#{table_name} + SET #{column_name} = '#{value}' + WHERE #{column_name} IS NOT NULL + AND #{column_name} <> '' + SQL + end + + def working_db + DataTaster.config.working_client.query_options[:database] + end + end +end diff --git a/packages/data_taster/lib/data_taster/flavors.rb b/packages/data_taster/lib/data_taster/flavors.rb new file mode 100644 index 00000000..b15f8bfa --- /dev/null +++ b/packages/data_taster/lib/data_taster/flavors.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module DataTaster + # helper methods made to make data_taster_export_tables.yml + # files more user-friendly + class Flavors + include DataTaster::Helper + + def current_date + @current_date ||= Date.current + end + + def date + @date ||= if DataTaster.config.months + (current_date - DataTaster.config.months.to_i.months).beginning_of_day.to_s(:db) + else + (current_date - 1.week).beginning_of_day.to_s(:db) + end + end + + # skips dumping both schema and data + def deprecated_table + DataTaster::SKIP_CODE + end + + def skip_sanitization + DataTaster::SKIP_CODE + end + + def encrypt(klass, column, value = nil) + value_to_encrypt = value || default_value_for(column) + + klass.new.encrypt(column, value_to_encrypt) + end + + def default_value_for(column) + case column + when /date_of_birth/, /dob/ + (Date.current - 25.years).strftime("%m/%d/%Y") + when /ssn/, /license/ + "111111111" + when /compensation/ + 1 + else + "1" + end + end + + def full_table_dump + "1 = 1" + end + + def recent_table_updates + "created_at >= '#{date}' OR updated_at >= '#{date}'" + end + + def recent_ids(table_name, col_name) + <<~SQL.squish + (SELECT DISTINCT(#{col_name}) + FROM #{source_db}.#{table_name} + WHERE + created_at >= '#{date}' + OR + updated_at >= '#{date}') + SQL + end + + def source_db + @source_db ||= db_config["database"] + end + end +end diff --git a/packages/data_taster/lib/data_taster/helper.rb b/packages/data_taster/lib/data_taster/helper.rb new file mode 100644 index 00000000..dbaf8a59 --- /dev/null +++ b/packages/data_taster/lib/data_taster/helper.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module DataTaster + # helpers used globally in DataTaster + module Helper + def sanitize_command(command, params = nil) + sanitized_command = command.gsub(Shellwords.escape(ENV.fetch("DEV_DUMP_USER", nil)), "") + .gsub(Shellwords.escape(ENV.fetch("DEV_DUMP_PASSWORD", nil)), "") + .gsub(ENV.fetch("DEV_DUMP_PASSWORD", nil), "") + + sanitized_command = sanitized_command.gsub(Shellwords.escape(params["password"]), "") if params + + sanitized_command + end + + def db_yml + @db_yml ||= YAML.safe_load(ERB.new(Rails.root.join("config", "database.yml").read).result, aliases: true) + end + + def db_config + @db_config ||= db_yml[Rails.env] + end + + def logg(message) + DataTaster.logger.debug { "[#{Time.current}] #{message}" } + end + end +end diff --git a/packages/data_taster/lib/data_taster/sample.rb b/packages/data_taster/lib/data_taster/sample.rb new file mode 100644 index 00000000..3b89f064 --- /dev/null +++ b/packages/data_taster/lib/data_taster/sample.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module DataTaster + # Selects and processes tables from the source_db + # to insert (or query) into the working_db + class Sample + def initialize(table_name) + @table_name = table_name + @include_insert = DataTaster.config.include_insert + @collection = DataTaster::Collection.new( + table_name + ).assemble + end + + def serve! + # Any table that does not return SQL is considered deprecated and we should fully skip it + if collection.empty? && include_insert + DataTaster.safe_execute("DROP TABLE IF EXISTS #{table_name}") + else + ensure_empty_table + process_select(collection[:select]) + DataTaster::Sanitizer.new(table_name, collection[:sanitize]).clean! + end + end + + private + + attr_reader :table_name, :include_insert, :collection + + def ensure_empty_table + DataTaster.safe_execute("TRUNCATE TABLE #{working_db}.#{table_name}") + end + + def process_select(sql) + DataTaster.safe_execute(sql) + rescue => e + e.message << " executing SQL statement for #{table_name}: #{sql}" + raise e + end + + def working_db + @working_db ||= DataTaster.config.working_client.query_options[:database] + end + end +end diff --git a/packages/data_taster/lib/data_taster/sanitizer.rb b/packages/data_taster/lib/data_taster/sanitizer.rb new file mode 100644 index 00000000..b49c150e --- /dev/null +++ b/packages/data_taster/lib/data_taster/sanitizer.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +module DataTaster + class Sanitizer + # Ensures the given tables are cleaned of + # information deemed sensitive + def initialize(table_name, custom_selections) + @table_name = table_name + @custom_selections = custom_selections || {} + @include_insert = DataTaster.config.include_insert + end + + def clean! + return if skippable_table? + + # custom selections should ALWAYS override defaults + default_selections.merge(custom_selections).filter_map do |column_name, sanitized_value| + sql = DataTaster::Detergent.new( + table_name, column_name, sanitized_value + ).deliver + + process(sql) + end + end + + private + + attr_reader :table_name, :custom_selections, :include_insert + + def process(sql) + return if sql == DataTaster::SKIP_CODE + return sql unless include_insert + + DataTaster.safe_execute(sql) + rescue => e + e.message << context_warning + + raise e + end + + def context_warning + <<~WARNING + + ***** + + DATA TASTER WARNING: Many columns are sanitized by default for safety. Please check DataTaster documentation for more details. + + ***** + + WARNING + end + + def skippable_table? + DataTaster.confection[table_name].blank? || + DataTaster.confection[table_name] == DataTaster::SKIP_CODE + end + + def default_selections + table_columns.each_with_object({}) do |table_col, selections| + sanitized_value = defaults.select do |default_col| + table_col.match(default_col) + end&.first&.last + + next unless sanitized_value + + selections[table_col] = sanitized_value + end + end + + def table_columns + @table_columns ||= ActiveRecord::Base + .connection + .schema_cache + .columns(table_name.to_s) + .map(&:name) + end + + def defaults # rubocop:disable Metrics/MethodLength + { + # `encrypted` should be removed if it is not custom-sanitized + /encrypted/ => "", + /#{exceptions}.*(ssn|passport|license)/ => "111111111", + /#{exceptions}.*(dob|birth)/ => Date.current - 29.years, + /#{exceptions}.*(note|body)/ => "Redacted for privacy", + /#{exceptions}.*(compensation|income)/ => 999_999, + /#{exceptions(extra: ['2'])}.*email.*/ => "CONCAT('#{table_name}_', id, '@nitrophrg.com')", + /email.*2$/ => "CONCAT('#{table_name}_', id, '_2', '@nitrophrg.com')", + /#{exceptions(extra: %w[2 email])}.*address.*/ => "CONCAT(id, ' Disneyland Dr')", + /#{exceptions(extra: ['email'])}.*address.*2/ => "Unit M", + }.freeze + end + + def exceptions(extra: []) + default = %w[_id _at type subject ip mac remote count encrypted voice count] + all_exceptions = default + extra + + "^(?!.*(?:#{all_exceptions.join('|')}))" + end + end +end diff --git a/packages/data_taster/lib/data_taster/version.rb b/packages/data_taster/lib/data_taster/version.rb index e0f3c27e..bdd88549 100644 --- a/packages/data_taster/lib/data_taster/version.rb +++ b/packages/data_taster/lib/data_taster/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module DataTaster - VERSION = "0.1.0" + VERSION = "0.2.0" end diff --git a/packages/data_taster/mkdocs.yml b/packages/data_taster/mkdocs.yml index ab75fa2e..825a94e4 100644 --- a/packages/data_taster/mkdocs.yml +++ b/packages/data_taster/mkdocs.yml @@ -1,4 +1,5 @@ site_name: Data Taster + nav: - "Home": "README.md" - "Changelog": "CHANGELOG.md" diff --git a/packages/data_taster/spec/data_taster_spec.rb b/packages/data_taster/spec/data_taster_spec.rb index 7773b938..847c2602 100644 --- a/packages/data_taster/spec/data_taster_spec.rb +++ b/packages/data_taster/spec/data_taster_spec.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true -RSpec.describe DataTaster do +require "spec_helper" + +describe DataTaster do it "has a version number" do expect(DataTaster::VERSION).not_to be nil end diff --git a/packages/data_taster/spec/spec_helper.rb b/packages/data_taster/spec/spec_helper.rb index 152a66ea..f957e920 100644 --- a/packages/data_taster/spec/spec_helper.rb +++ b/packages/data_taster/spec/spec_helper.rb @@ -1,15 +1,56 @@ # frozen_string_literal: true +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) require "data_taster" RSpec.configure do |config| - # Enable flags like --only-failures and --next-failure - config.example_status_persistence_file_path = ".rspec_status" + if ENV["CI"] + config.before(:example, :focus) { raise "Should not commit focused specs" } + else + config.filter_run :focus + config.run_all_when_everything_filtered = true + end + config.warnings = false + + config.default_formatter = "doc" if config.files_to_run.one? + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed + + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # Enable only the newer, non-monkey-patching expect syntax. + # For more details, see: + # - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax + expectations.syntax = :expect + end - # Disable RSpec exposing methods globally on `Module` and `main` - config.disable_monkey_patching! + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Enable only the newer, non-monkey-patching expect syntax. + # For more details, see: + # - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ + mocks.syntax = :expect - config.expect_with :rspec do |c| - c.syntax = :expect + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended. + mocks.verify_partial_doubles = false end end