This might be a bit crazy

feedstock/recipe.py

@@ -54,7 +54,7 @@ def transfer(self, source_store) -> str:
  target_bucket = f"m2lines-test"
  # source_prefix = "data-library/feedstocks/GODAS"
  source_prefix = ""
- target_prefix = "test-transfer-500GB-crazy-town"
+ target_prefix = "test-transfer-500GB-crazy-town-in-the-morning"
 
  # construct full valid rclone 'paths'
  source = os.path.join(source_bucket, source_prefix, store_name).strip('/')
@@ -63,7 +63,7 @@ def transfer(self, source_store) -> str:
  logger.warning(f"Copying from {source} to {target}")
 
  copy_proc = subprocess.run(
- f'rclone copy --transfers 256 --checkers 512 -vv -P "{gcs_remote}{source}/" "{osn_remote}{target}/"',
+ f'rclone copy --s3-chunk-size 100M --s3-upload-concurrency 128 --transfers 128 --checkers 256  -vv -P "{gcs_remote}{source}/" "{osn_remote}{target}/"',
  shell=True,
  capture_output=False, # will expose secrets if true
  text=True,