Freeze layers for transfer learning

training/deepspeech_training/train.py

@@ -29,7 +29,7 @@
 from .evaluate import evaluate
 from six.moves import zip, range
 from .util.config import Config, initialize_globals
-from .util.checkpoints import load_or_init_graph_for_training, load_graph_for_evaluation, reload_best_checkpoint
+from .util.checkpoints import drop_freeze_number_to_layers, load_or_init_graph_for_training, load_graph_for_evaluation, reload_best_checkpoint
 from .util.evaluate_tools import save_samples_json
 from .util.feeding import create_dataset, audio_to_features, audiofile_to_features
 from .util.flags import create_flags, FLAGS
@@ -322,8 +322,24 @@ def get_tower_results(iterator, optimizer, dropout_rates):
  # Retain tower's avg losses
  tower_avg_losses.append(avg_loss)
 
+ train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
+
+ # Filter out layers if we want to freeze some
+ if FLAGS.freeze_source_layers > 0:
+ filter_vars = drop_freeze_number_to_layers(FLAGS.freeze_source_layers, "freeze")
+ new_train_vars = list(train_vars)
+ for fv in filter_vars:
+ for tv in train_vars:
+ if fv in tv.name:
+ new_train_vars.remove(tv)
+ train_vars = new_train_vars
+ msg = "Tower {} - Training only variables: {}"
+ print(msg.format(i, [v.name for v in train_vars]))
+ else:
+ print("Tower {} - Training all layers".format(i))
+
  # Compute gradients for model parameters using tower's mini-batch
- gradients = optimizer.compute_gradients(avg_loss)
+ gradients = optimizer.compute_gradients(avg_loss, var_list=train_vars)
 
  # Retain tower's gradients
  tower_gradients.append(gradients)
@@ -671,7 +687,6 @@ def __call__(self, progress, data, **kwargs):
 
  print('-' * 80)
 
-
  except KeyboardInterrupt:
  pass
  log_info('FINISHED optimization in {}'.format(datetime.utcnow() - train_start_time))

training/deepspeech_training/util/checkpoints.py

@@ -1,9 +1,9 @@
 import sys
-import tensorflow as tf
+
 import tensorflow.compat.v1 as tfv1
 
 from .flags import FLAGS
-from .logging import log_info, log_error, log_warn
+from .logging import log_error, log_info, log_warn
 
 
 def _load_checkpoint(session, checkpoint_path, allow_drop_layers, allow_lr_init=True):
@@ -19,32 +19,33 @@ def _load_checkpoint(session, checkpoint_path, allow_drop_layers, allow_lr_init=
  # compatibility with older checkpoints.
  lr_var = set(v for v in load_vars if v.op.name == 'learning_rate')
  if lr_var and ('learning_rate' not in vars_in_ckpt or
-  (FLAGS.force_initialize_learning_rate and allow_lr_init)):
+ (FLAGS.force_initialize_learning_rate and allow_lr_init)):
  assert len(lr_var) <= 1
  load_vars -= lr_var
  init_vars |= lr_var
 
- if FLAGS.load_cudnn:
- # Initialize training from a CuDNN RNN checkpoint
- # Identify the variables which we cannot load, and set them
- # for initialization
- missing_vars = set()
- for v in load_vars:
- if v.op.name not in vars_in_ckpt:
- log_warn('CUDNN variable not found: %s' % (v.op.name))
- missing_vars.add(v)
+ # After training with "freeze_source_layers" the Adam moment tensors for the frozen layers
+ # are missing because they were not used. This might also occur when loading a cudnn checkpoint
+ # Therefore we have to initialize them again to continue training on such checkpoints
+ print_msg = False
+ for v in load_vars:
+ if v.op.name not in vars_in_ckpt:
+ if 'Adam' in v.name:
  init_vars.add(v)
+ print_msg = True
+ if print_msg:
+ msg = "Some Adam tensors are missing, they will be initialized automatically."
+ log_info(msg)
+ load_vars -= init_vars
 
- load_vars -= init_vars
-
- # Check that the only missing variables (i.e. those to be initialised)
- # are the Adam moment tensors, if they aren't then we have an issue
- missing_var_names = [v.op.name for v in missing_vars]
- if any('Adam' not in v for v in missing_var_names):
- log_error('Tried to load a CuDNN RNN checkpoint but there were '
- 'more missing variables than just the Adam moment '
- 'tensors. Missing variables: {}'.format(missing_var_names))
- sys.exit(1)
+ if FLAGS.load_cudnn:
+ # Check all required tensors are included in the cudnn checkpoint we want to load
+ for v in load_vars:
+ if v.op.name not in vars_in_ckpt and 'Adam' not in v.op.name:
+ msg = 'Tried to load a CuDNN RNN checkpoint but there was a missing' \
+ ' variable other than an Adam moment tensor: {}'
+ log_error(msg.format(v.op.name))
+ sys.exit(1)
 
  if allow_drop_layers and FLAGS.drop_source_layers > 0:
  # This transfer learning approach requires supplying
@@ -59,7 +60,7 @@ def _load_checkpoint(session, checkpoint_path, allow_drop_layers, allow_lr_init=
  'dropping only 5 layers.')
  FLAGS.drop_source_layers = 5
 
- dropped_layers = ['2', '3', 'lstm', '5', '6'][-1 * int(FLAGS.drop_source_layers):]
+ dropped_layers = drop_freeze_number_to_layers(FLAGS.drop_source_layers, "drop")
  # Initialize all variables needed for DS, but not loaded from ckpt
  for v in load_vars:
  if any(layer in v.op.name for layer in dropped_layers):
@@ -75,6 +76,24 @@ def _load_checkpoint(session, checkpoint_path, allow_drop_layers, allow_lr_init=
  session.run(v.initializer)
 
 
+def drop_freeze_number_to_layers(drop_freeze_number, mode):
+ """ Convert number of layers to drop or freeze into layer names """
+
+ if drop_freeze_number >= 6:
+ log_warn('The checkpoint only has 6 layers, but you are trying '
+ 'to drop or freeze all of them or more. Continuing with 5 layers.')
+ drop_freeze_number = 5
+
+ layer_keys = ["layer_1", "layer_2", "layer_3", "lstm", "layer_5", "layer_6"]
+ if mode == "drop":
+ layer_keys = layer_keys[-1 * int(drop_freeze_number):]
+ elif mode == "freeze":
+ layer_keys = layer_keys[:-1 * int(drop_freeze_number)]
+ else:
+ raise ValueError
+ return layer_keys
+
+
 def _checkpoint_path_or_none(checkpoint_filename):
  checkpoint = tfv1.train.get_checkpoint_state(FLAGS.load_checkpoint_dir, checkpoint_filename)
  if not checkpoint:

training/deepspeech_training/util/flags.py

@@ -92,7 +92,8 @@ def create_flags():
 
  # Transfer Learning
 
- f.DEFINE_integer('drop_source_layers', 0, 'single integer for how many layers to drop from source model (to drop just output == 1, drop penultimate and output ==2, etc)')
+ f.DEFINE_integer('drop_source_layers', 0, 'single integer for how many layers to drop from source model (to drop just output == 1, drop penultimate and output == 2, etc)')
+ f.DEFINE_integer('freeze_source_layers', 0, 'freeze layer weights (to freeze all but output == 1, freeze all but penultimate and output == 2, etc). Normally used in combination with "drop_source_layers" flag and should be used in a two step training (first drop and freeze layers and train a few epochs, second continue without both flags)')
 
  # Exporting