Update configs to work with a filtered raw + add debug script with pdb

cfgs/default/4m/data/video/mix_mod3_rgb_tok_to_all_a0.5.yaml

@@ -21,7 +21,7 @@ train:
  # Data can either be local or on cloud storage (e.g. S3), see data docs for more info
  # Use braceexpand notation to indicate shard range (e.g. shard-{0000..9999}.tar)
  # Use brackets to indicate multiple modalities (e.g. [modality1,modality2,modality3]) 
- data_path: '/store/swissai/a08/data/4m/splits2/train/[video_rgb,video_rgb_tok]/00000{00000..00100}.tar' # TODO: need to reformat the data correctly here. 
+ data_path: '/store/swissai/a08/data/4m/cleaned/train/[video_rgb,video_tok_rgb]/0000000000.tar' # TODO: need to reformat the data correctly here. 
  use_wds: True # Use webdataset
  wds_n_repeats: 4 # Number of repeats for webdataset loader to improve efficiency
  wds_shuffle_buffer_tar: 1_000 # Webdatasets shuffle buffer after loading tar files
@@ -41,4 +41,4 @@ train:
 # val:
 # datasets:
 # my_video_dataset: 
-# data_path: '/store/swissai/a08/data/4m/val/[video_rgb,video_rgb_tok]/00000{00175..00199}.tar'
+# data_path: '/store/swissai/a08/data/4m/val/[video_rgb,video_tok_rgb]/00000{00175..00199}.tar'

fourm/data/unified_datasets.py

@@ -136,6 +136,7 @@ def build_fm_transfer_dataset(
 def _keyless_map(data, f, handler=reraise_exception):
  """Map samples without adding __key__."""
  for sample in data:
+ import pdb; pdb.set_trace()
  try:
  result = f(sample)
  except Exception as exn:
@@ -390,7 +391,7 @@ def build_wds_fm_pretraining_dataloader(
 
  if batch_size is not None:
  # Perform multi-threaded dataloading
- return wds.WebLoader(datapipe, num_workers=num_workers, batch_size=None)
+ return wds.WebLoader(datapipe, num_workers=0, batch_size=None)
  else:
  return datapipe
 
@@ -552,6 +553,6 @@ def build_mixture_dataloader(data_iters, weights, modality_info, batch_size, num
  wds.batched(batch_size, collation_fn=default_collate, partial=False),
  ).with_epoch(epoch_size // (num_gpus * num_workers * batch_size)) # Pre-define iterator length
 
- mixture_loader = wds.WebLoader(mixture_pipe, num_workers=num_workers, batch_size=None)
+ mixture_loader = wds.WebLoader(mixture_pipe, num_workers=0, batch_size=None)
 
  return mixture_loader

run_training_4m.py

@@ -12,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# python run_training_4m.py --data_config cfgs/default/4m/data/video/mix_mod3_rgb_tok_to_all_a0.5.yaml
-import argparse
+# FOR DEBUGGING with pdb:
+# 1. make sure the wds.WebLoader(num_workers=0) in a) build_wds_fm_pretraining_dataloader and b) build_mixture_dataloader
+# python -m torch.distributed.launch --nproc_per_node 1 --use-env run_training_4m.py --config /store/swissai/a08/kdu/ml-4m/cfgs/default/4m/models/video/4m-b_mod3.yamlimport argparse
 import datetime
 import json
 import math