Merge pull request #1 from aswinanand/fix_107_deadlock

Fix 107 deadlock

.rubocop.yml

@@ -5,36 +5,9 @@
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 2
-Lint/AmbiguousRegexpLiteral:
- Enabled: false
-
-# Offense count: 2
-# Configuration parameters: AllowSafeAssignment.
-Lint/AssignmentInCondition:
- Enabled: false
-
-# Offense count: 1
-Lint/ConditionPosition:
- Enabled: false
-
-# Offense count: 1
-Lint/HandleExceptions:
- Enabled: false
-
-# Offense count: 6
-# Cop supports --auto-correct.
-Lint/UnusedBlockArgument:
- Enabled: false
-
-# Offense count: 3
-# Cop supports --auto-correct.
-Lint/UnusedMethodArgument:
- Enabled: false
-
-# Offense count: 4
-Lint/UselessAssignment:
- Enabled: false
+AllCops:
+ Exclude:
+ - 'gemfiles/**/*'
 
 # Offense count: 1
 # Configuration parameters: CountComments.
@@ -59,18 +32,6 @@ Metrics/MethodLength:
 Metrics/PerceivedComplexity:
  Max: 15
 
-# Offense count: 10
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-Style/AccessModifierIndentation:
- Enabled: false
-
-# Offense count: 4
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-Style/AlignParameters:
- Enabled: false
-
 # Offense count: 5
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles.
@@ -83,11 +44,6 @@ Style/AndOr:
 Style/BarePercentLiterals:
  Enabled: false
 
-# Offense count: 1
-# Cop supports --auto-correct.
-Style/Blocks:
- Enabled: false
-
 # Offense count: 11
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles.
@@ -159,16 +115,6 @@ Style/IfUnlessModifier:
 Style/IndentHash:
  Enabled: false
 
-# Offense count: 1
-# Cop supports --auto-correct.
-Style/IndentationConsistency:
- Enabled: false
-
-# Offense count: 1
-# Cop supports --auto-correct.
-Style/IndentationWidth:
- Enabled: false
-
 # Offense count: 2
 Style/ModuleFunction:
  Enabled: false
@@ -177,11 +123,6 @@ Style/ModuleFunction:
 Style/MultilineBlockChain:
  Enabled: false
 
-# Offense count: 1
-# Cop supports --auto-correct.
-Style/NegatedIf:
- Enabled: false
-
 # Offense count: 4
 # Cop supports --auto-correct.
 Style/NumericLiterals:
@@ -198,17 +139,6 @@ Style/PercentLiteralDelimiters:
 Style/RaiseArgs:
  Enabled: false
 
-# Offense count: 1
-# Cop supports --auto-correct.
-# Configuration parameters: AllowMultipleReturnValues.
-Style/RedundantReturn:
- Enabled: false
-
-# Offense count: 2
-# Cop supports --auto-correct.
-Style/RedundantSelf:
- Enabled: false
-
 # Offense count: 1
 Style/RescueModifier:
  Enabled: false
@@ -217,11 +147,10 @@ Style/RescueModifier:
 Style/SelfAssignment:
  Enabled: false
 
-# Offense count: 7
-# Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles.
 Style/SignalException:
- Enabled: false
+ EnforcedStyle: only_raise
+ Enabled: true
 
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SupportedStyles.
@@ -249,8 +178,6 @@ Style/UnneededPercentQ:
 Style/WordArray:
  MinSize: 2
 
-# Offense count: 2
-# Cop supports --auto-correct.
-# Configuration parameters: AllowSafeAssignment.
-Style/ParenthesesAroundCondition:
- Enabled: false
+# Offense count: 7
+Metrics/AbcSize:
+ Max: 31

.travis.yml

@@ -2,14 +2,20 @@ language: ruby
 before_script:
  - "mysql -e 'create database lhm;'"
 rvm:
- - 1.9.3
  - 2.0.0
  - 2.1
+ - 2.2
 sudo: false
-matrix:
- fast_finish: true
 gemfile:
  - gemfiles/ar-2.3_mysql.gemfile
  - gemfiles/ar-3.2_mysql.gemfile
  - gemfiles/ar-3.2_mysql2.gemfile
- - gemfiles/dm_mysql.gemfile
+ - gemfiles/ar-4.0_mysql2.gemfile
+ - gemfiles/ar-4.1_mysql2.gemfile
+ - gemfiles/ar-4.2_mysql2.gemfile
+matrix:
+ exclude:
+ - rvm: 2.2
+ gemfile: gemfiles/ar-3.2_mysql.gemfile
+ - rvm: 2.2
+ gemfile: gemfiles/ar-2.3_mysql.gemfile

CHANGELOG.md

@@ -1,10 +1,25 @@
-# 2.2.0 (not released)
+# 3.0.0
+
+* Drop support for throttle and stride options. Use `throttler`, instead:
+```
+Lhm.change_table :users, throttler: [:time_throttler, {stride: x}] do
+end
+```
+* #118 - Truncate long trigger names. (@sj26)
+* #114 - Update chunker requirements (@bjk-soundcloud)
+* #98 - Add slave lag throttler. (@camilo, @jasonhl)
+* #92 - Fix check for table requirement before starting a lhm.(@hannestyden)
+* #93 - Makes the atomic switcher retry on metadata locks (@camilo)
+* #63 - Sets the LHM's session lock wait timeout variables (@camilo)
+* #75 - Remove DataMapper and ActiveRecord 2.x support (@camilo)
+
+# 2.2.0 (Jan 16, 2015)
 
 * #84 - Require index names to be strings or symbols (Thibaut)
 * #39 - Adding the ability to rename columns (erikogan)
 * #67 - Allow for optional time filter on .cleanup (joelr)
 
-# 2.1.0
+# 2.1.0 (July 31, 2014)
 
 * #48 - Add percentage output for migrations (@arthurnn)
 * #60 - Quote table names (@spickermann)
@@ -15,7 +30,7 @@
 * #51 - Ensure Lhm.cleanup removes temporary triggers (@edmundsalvacion)
 * #46 - Allow custom throttler (@arthurnn)
 
-# 2.0.0
+# 2.0.0 (July 10, 2013)
 
 * #44 - Conditional migrations (@durran)
 

README.md

@@ -22,7 +22,7 @@ is great if you are using this engine, but only solves half the problem.
 At SoundCloud we started having migration pains quite a while ago, and after
 looking around for third party solutions, we decided to create our
 own. We called it Large Hadron Migrator, and it is a gem for online
-ActiveRecord and DataMapper migrations.
+ActiveRecord migrations.
 
 ![LHC](http://farm4.static.flickr.com/3093/2844971993_17f2ddf2a8_z.jpg)
 
@@ -35,28 +35,27 @@ without locking the table. In contrast to [OAK][0] and the
 [facebook tool][1], we only use a copy table and triggers.
 
 The Large Hadron is a test driven Ruby solution which can easily be dropped
-into an ActiveRecord or DataMapper migration. It presumes a single auto
+into an ActiveRecord migration. It presumes a single auto
 incremented numerical primary key called id as per the Rails convention. Unlike
 the [twitter solution][2], it does not require the presence of an indexed
 `updated_at` column.
 
 ## Requirements
 
 Lhm currently only works with MySQL databases and requires an established
-ActiveRecord or DataMapper connection.
-
-It is compatible and [continuously tested][4] with MRI 1.9.x, 2.0.x, 2.1.x,
-ActiveRecord 2.3.x and 3.x (mysql and mysql2 adapters), as well as DataMapper
-1.2 (dm-mysql-adapter).
-
-Lhm also works with dm-master-slave-adapter, it'll bind to the master before
-running the migrations.
+ActiveRecord connection.
 
+It is compatible and [continuously tested][4] with MRI 2.0.x, 2.1.x,
+ActiveRecord 3.2.x and 4.x (mysql and mysql2 adapters).
 
 ## Limitations
 
-Lhm requires a monotonically increasing numeric Primary Key on the table, due to how
-the Chunker works.
+Due to the Chunker implementation, Lhm requires that the table to migrate has
+a single integer numeric key column called `id`.
+
+Another note about the Chunker, it performs static sized row copies against the `id`
+column. Therefore sparse assignment of `id` can cause performance problems for the
+backfills. Typically LHM assumes that `id` is an `auto_increment` style column.
 
 ## Installation
 
@@ -76,9 +75,6 @@ ActiveRecord::Base.establish_connection(
  :database => 'lhm'
 )
 
-# or with DataMapper
-Lhm.setup(DataMapper.setup(:default, 'mysql://127.0.0.1/lhm'))
-
 # and migrate
 Lhm.change_table :users do |m|
  m.add_column :arbitrary, "INT(12)"
@@ -111,41 +107,16 @@ class MigrateUsers < ActiveRecord::Migration
 end
 ```
 
-Using dm-migrations, you'd define all your migrations as follows, and then call
-`migrate_up!` or `migrate_down!` as normal.
-
-```ruby
-require 'dm-migrations/migration_runner'
-require 'lhm'
-
-migration 1, :migrate_users do
- up do
- Lhm.change_table :users do |m|
- m.add_column :arbitrary, "INT(12)"
- m.add_index [:arbitrary_id, :created_at]
- m.ddl("alter table %s add column flag tinyint(1)" % m.name)
- end
- end
-
- down do
- Lhm.change_table :users do |m|
- m.remove_index [:arbitrary_id, :created_at]
- m.remove_column :arbitrary
- end
- end
-end
-```
-
 **Note:** Lhm won't delete the old, leftover table. This is on purpose, in order
 to prevent accidental data loss.
 
 ## Throttler
 
-Lhm is using a throttle mecanism to read data in your original table.
+Lhm is using a throttle mechanism to read data in your original table.
 
 By default, 40000 rows are read each 0.1 second.
 
-If you want to change that behiavour, you can pass an instance of a throttler with the `throttler` option.
+If you want to change that behaviour, you can pass an instance of a throttler with the `throttler` option.
 
 In this example, 1000 rows will be read with a 10 seconds delay between each processing:
 ```ruby
@@ -156,6 +127,21 @@ Lhm.change_table :users, throttler: my_throttler do |m|
 end
 ```
 
+### SlaveLag Throttler
+
+Lhm uses by default the time throttler, however a better solution is to throttle the copy of the data
+depending on the time that the slaves are behind. To use the SlaveLag throttler:
+```ruby
+Lhm.change_table :users, throttler: :slave_lag_throttler do |m|
+ #
+end
+```
+
+Or to set that as default throttler, use the following (for instance in a Rails initializer):
+```ruby
+Lhm.setup_throttler(:slave_lag_throttler)
+```
+
 ## Table rename strategies
 
 There are two different table rename strategies available: LockedSwitcher and
@@ -210,19 +196,36 @@ Lhm.cleanup
 
 To remove any Lhm tables/triggers found:
 ```ruby
-Lhm.cleanup(true)
+Lhm.cleanup(:run)
 ```
 
 Optionally only remove tables up to a specific Time, if you want to retain previous migrations.
 
 Rails:
 ```ruby
-Lhm.cleanup(true, until: 1.day.ago)
+Lhm.cleanup(:run, until: 1.day.ago)
 ```
 
 Ruby:
 ```ruby
-Lhm.cleanup(true, until: Time.now - 86400)
+Lhm.cleanup(:run, until: Time.now - 86400)
+```
+
+## Retry on deadlock
+
+LHM supports automatic retry on deadlock. By default it retries 10 times with
+a 10 second delay between each retry. It can be customized using the following
+options:
+
+ 1. `retry_on_deadlock` - true / false (true by default)
+ 2. `retry_attempts` - Number of retries on deadlock (10 by default)
+ 3. `retry_wait_time` - Number of seconds to wait between each retry
+ (10 seconds by default)
+
+```ruby
+Lhm.change_table :users, :retry_on_deadlock => true, :retry_attempts => 5 do |m|
+ m.add_column :arbitrary, "INT(12)"
+end
 ```
 
 ## Contributing

Rakefile

@@ -6,14 +6,14 @@ Bundler::GemHelper.install_tasks
 Rake::TestTask.new('unit') do |t|
  t.libs << 'lib'
  t.libs << 'spec'
- t.test_files = FileList['spec/unit/*_spec.rb']
+ t.test_files = FileList['spec/unit/**/*_spec.rb']
  t.verbose = true
 end
 
 Rake::TestTask.new('integration') do |t|
  t.libs << 'lib'
  t.libs << 'spec'
- t.test_files = FileList['spec/integration/*_spec.rb']
+ t.test_files = FileList['spec/integration/**/*_spec.rb']
  t.verbose = true
 end