diff --git a/gluoncv/auto/data/data_zoo.py b/gluoncv/auto/data/data_zoo.py index c95d31b067..c20019203a 100644 --- a/gluoncv/auto/data/data_zoo.py +++ b/gluoncv/auto/data/data_zoo.py @@ -68,7 +68,26 @@ def _pascal_0712_detection(download=True): filename = _download('https://s3.amazonaws.com/fast-ai-imagelocal/pascal-voc.tgz', path=root, overwrite=True) with tarfile.open(filename) as tar: - tar.extractall(path=root) + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + + safe_extract(tar, path=root) shutil.move(os.path.join(root, 'pascal-voc', 'VOC2007'), os.path.join(root, 'VOC2007')) shutil.move(os.path.join(root, 'pascal-voc', 'VOC2012'), os.path.join(root, 'VOC2012')) shutil.rmtree(os.path.join(root, 'pascal-voc')) diff --git a/scripts/datasets/ilsvrc_det.py b/scripts/datasets/ilsvrc_det.py index 065db749a6..364bebe42a 100644 --- a/scripts/datasets/ilsvrc_det.py +++ b/scripts/datasets/ilsvrc_det.py @@ -41,7 +41,26 @@ def download_det(args, overwrite=False): filename = download(url, path=args.download_dir, overwrite=overwrite, sha1_hash=checksum) print(' dataset has already download completed') with tarfile.open(filename) as tar: - tar.extractall(path=args.download_dir) + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + + safe_extract(tar, path=args.download_dir) if os.path.isdir(os.path.join(args.download_dir, 'ILSVRC2015')): os.rename(os.path.join(args.download_dir, 'ILSVRC2015'), os.path.join(args.download_dir, 'ILSVRC')) diff --git a/scripts/datasets/ilsvrc_vid.py b/scripts/datasets/ilsvrc_vid.py index 4c6b019f49..57ec2299b3 100644 --- a/scripts/datasets/ilsvrc_vid.py +++ b/scripts/datasets/ilsvrc_vid.py @@ -44,7 +44,26 @@ def download_VID(args, overwrite=False): filename = download(url, path=args.download_dir, overwrite=overwrite, sha1_hash=checksum) print('dataset is unziping') with tarfile.open(filename) as tar: - tar.extractall(path=args.download_dir) + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + + safe_extract(tar, path=args.download_dir) def parse_vid(ann_base_path, args): """ diff --git a/scripts/datasets/imagenet.py b/scripts/datasets/imagenet.py index c9506669b9..d1731b1450 100644 --- a/scripts/datasets/imagenet.py +++ b/scripts/datasets/imagenet.py @@ -85,7 +85,26 @@ def extract_train(tar_fname, target_dir, with_rec=False, num_thread=1): class_dir = os.path.splitext(class_fname)[0] os.mkdir(class_dir) with tarfile.open(class_fname) as f: - f.extractall(class_dir) + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + + safe_extract(f, class_dir) os.remove(class_fname) pbar.update(1) pbar.close() @@ -96,7 +115,26 @@ def extract_val(tar_fname, target_dir, with_rec=False, num_thread=1): os.makedirs(target_dir) print('Extracting ' + tar_fname) with tarfile.open(tar_fname) as tar: - tar.extractall(target_dir) + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + + safe_extract(tar, target_dir) # build rec file before images are moved into subfolders if with_rec: build_rec_process(target_dir, False, num_thread) diff --git a/scripts/datasets/pascal_voc.py b/scripts/datasets/pascal_voc.py index 65afdabbc0..19c084fa78 100644 --- a/scripts/datasets/pascal_voc.py +++ b/scripts/datasets/pascal_voc.py @@ -35,7 +35,26 @@ def download_voc(path, overwrite=False): filename = download(url, path=path, overwrite=overwrite, sha1_hash=checksum) # extract with tarfile.open(filename) as tar: - tar.extractall(path=path) + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + + safe_extract(tar, path=path) ##################################################################################### @@ -49,7 +68,26 @@ def download_aug(path, overwrite=False): filename = download(url, path=path, overwrite=overwrite, sha1_hash=checksum) # extract with tarfile.open(filename) as tar: - tar.extractall(path=path) + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + + safe_extract(tar, path=path) shutil.move(os.path.join(path, 'benchmark_RELEASE'), os.path.join(path, 'VOCaug')) filenames = ['VOCaug/dataset/train.txt', 'VOCaug/dataset/val.txt']