feat: implement manufact_80 works and finetune indoor_80

.gitignore

@@ -7,6 +7,7 @@ run_scripts/finetune/polyformer_b_aihub_manufact_checkpoints
 run_scripts/finetune/polyformer_b_aihub_manufact_logs
 run_scripts/finetune/polyformer_l_checkpoints
 run_scripts/finetune/polyformer_l_logs
+run_scripts/pretrain/polyformer_b_pretrain_aihub_indoor_80_logs_resume_2/
 results_polyformer_b
 weights
 

data/create_aihub_data.py

@@ -16,13 +16,13 @@
 
 data_root = './refer/data'
 # datasets = ['refcoco', 'refcoco+', 'refcocog']
-datasets = ['aihub_indoor']
-# datasets = ['aihub_manufact']
+# datasets = ['aihub_indoor']
+datasets = ['aihub_manufact']
 
 if datasets[0] == 'aihub_indoor':
     image_dir = './refer/data/aihub_refcoco_format/indoor_80/images'
 elif datasets[0] == 'aihub_manufact':
-    image_dir = './refer/data/aihub_refcoco_format/manufact/images'
+    image_dir = './refer/data/aihub_refcoco_format/manufact_80/images'
 else:
     image_dir = './datasets/images/mscoco/train2014'
 val_test_files = pickle.load(open("data/val_test_files.p", "rb"))
@@ -46,6 +46,7 @@
         splits = ['train', 'val']
         splitBy = None
     elif dataset == 'aihub_manufact':
+        # splits = ['val', 'test']
         splits = ['train', 'val', 'test']
         splitBy = None
 
@@ -66,7 +67,10 @@
         writer = open(file_name, 'w')
         refer = REFER(data_root, dataset, splitBy)
 
-        ref_ids = refer.getRefIds(split=split)
+        if dataset == 'aihub_manufact' and split == 'val':
+            ref_ids = refer.getRefIds(split='validation')
+        else:
+            ref_ids = refer.getRefIds(split=split)
 
         for this_ref_id in tqdm(ref_ids):
             this_img_id = refer.getImgIds(this_ref_id)

data/create_pretraining_aihub_data.py

@@ -5,13 +5,13 @@
 import pickle
 
 
-img_path = 'refer/data/aihub_refcoco_format/indoor_80/images'
-# img_path = 'refer/data/aihub_refcoco_format/indoor/images'
+# img_path = 'refer/data/aihub_refcoco_format/indoor_80/images'
+img_path = 'refer/data/aihub_refcoco_format/manufact_80/images'
 
 # load annotation files
 # f = open("datasets/annotations/instances.json")
-f = open("refer/data/aihub_refcoco_format/indoor_80/instances.json")
-# f = open("refer/data/aihub_refcoco_format/manufact/instances.json")
+# f = open("refer/data/aihub_refcoco_format/indoor_80/instances.json")
+f = open("refer/data/aihub_refcoco_format/manufact_80/instances.json")
 print("Loading annotation file")
 data = json.load(f)
 f.close()
@@ -35,8 +35,8 @@
 print(len(data['images']))
 print(len(data['annotations']))
 
-ref_file = 'refer/data/aihub_refcoco_format/indoor_80/refs.p'
-# ref_file = 'refer/data/aihub_refcoco_format/manufact/refs.p'
+# ref_file = 'refer/data/aihub_refcoco_format/indoor_80/refs.p'
+ref_file = 'refer/data/aihub_refcoco_format/manufact_80/refs.p'
 ref_ann = pickle.load(open(ref_file, 'rb'))
 print(ref_ann[10])
 print(ref_ann[1])
@@ -51,7 +51,8 @@
 # exit()
 
 
-tsv_filename = "datasets/pretrain/train_aihub_indoor_80_test.tsv"
+# tsv_filename = "datasets/pretrain/train_aihub_indoor_80_test.tsv"
+tsv_filename = "datasets/pretrain/train_aihub_manufact_80.tsv"
 writer = open(tsv_filename, 'w')
 print("generating ", tsv_filename)
 
@@ -80,6 +81,7 @@
         # print("train!!")
         pass
     else:
+        # print(ref_ann_i['split'])
         # print("It's validation or test data")
         continue
 
@@ -117,7 +119,8 @@
 
 #####################################
 # generate validation tsv files
-tsv_filename = f"datasets/pretrain/val_aihub_indoor_80.tsv"
+# tsv_filename = f"datasets/pretrain/val_aihub_indoor_80.tsv"
+tsv_filename = f"datasets/pretrain/val_aihub_manufact_80.tsv"
 writer = open(tsv_filename, 'w')
 print("generating ", tsv_filename)
 
@@ -132,7 +135,7 @@
 
     # ref_ann_i = next((d for d in ref_ann if d["ref_id"] == str(i)), None)
     # ref_ann_i = ref_ann[i]
-    if ref_ann_i['split'] == 'val':
+    if ref_ann_i['split'] == 'validation':
         # print("val!!")
         pass
     else:

data/refcoco_dataset.py

@@ -19,7 +19,7 @@
 import utils.transforms as T
 import math
 # from PIL import ImageFile
-from PIL import Image, ImageFile
+from PIL import Image, ImageFile, UnidentifiedImageError
 # import Image
 
 from data import data_utils
@@ -92,7 +92,19 @@ def __getitem__(self, index):
         # print(BytesIO(base64.urlsafe_b64decode(base64_str)))
 
         # image = Image.open(BytesIO(base64.urlsafe_b64decode(base64_str))).convert("RGB")
-        image = Image.open(BytesIO(base64.b64decode(base64_str))).convert("RGB")
+        try:
+            image = Image.open(BytesIO(base64.b64decode(base64_str))).convert("RGB")
+        # except UnidentifiedImageError:
+        except:
+            data = self.dataset[index+1]
+            if len(data) == 7:
+                uniq_id, base64_str, seg64_str, text, poly_original, region_coord, poly_interpolated = data
+                train = True
+            else:
+                uniq_id, base64_str, seg64_str, text, poly, region_coord = data
+                train = False
+            image = Image.open(BytesIO(base64.b64decode(base64_str))).convert("RGB")
+
 
         label = Image.open(BytesIO(base64.urlsafe_b64decode(seg64_str)))
         label = np.asarray(label)

refer/refer.py

@@ -66,8 +66,8 @@ def __init__(self, data_root, dataset='refcoco', splitBy='unc'):
 			# print(os.listdir(self.IMAGE_DIR))
 		elif dataset == 'aihub_manufact':
 			# self.DATA_DIR = osp.join(data_root, 'refcocog')
-			self.DATA_DIR = "refer/data/aihub_refcoco_format/manufact"
-			self.IMAGE_DIR = "refer/data/aihub_refcoco_format/manufact/images"
+			self.DATA_DIR = "refer/data/aihub_refcoco_format/manufact_80"
+			self.IMAGE_DIR = "refer/data/aihub_refcoco_format/manufact_80/images"
 			print('Dataset preprocessing...')
 			print('Print list of AIHub Manufact dataset...')
 			# print(os.listdir(self.IMAGE_DIR))
@@ -142,7 +142,7 @@ def createIndex(self):
 			for sent in ref['sentences']:
 				Sents[sent['sent_id']] = sent
 				sentToRef[sent['sent_id']] = ref
-				sentToTokens[sent['sent_id']] = sent['tokens']
+				# sentToTokens[sent['sent_id']] = sent['tokens']
 
 		# create class members
 		self.Refs = Refs
@@ -184,6 +184,8 @@ def getRefIds(self, image_ids=[], cat_ids=[], ref_ids=[], split=''):
 					refs = [ref for ref in refs if 'test' in ref['split']]
 				elif split == 'train' or split == 'val':
 					refs = [ref for ref in refs if ref['split'] == split]
+				elif split == 'validation':
+					refs = [ref for ref in refs if ref['split'] == split]
 				else:
 					print ('No such split [%s]' % split)
 					sys.exit()

run_scripts/evaluation/evaluate_polyformer_b_aihub_indoor.sh

@@ -18,12 +18,13 @@ num_bins=64
 batch_size=16
 
 dataset='aihub_indoor'
-ckpt_path=../finetune/polyformer_b_aihub_indoor_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
+ckpt_path=../finetune/polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt
 # dataset='refcocog'
 # ckpt_path=../../weights/polyformer_b_refcocog.pt
 
 # for split in 'refcocog_val' 'refcocog_test'
-for split in 'aihub_indoor_val' 'aihub_indoor_test'
+# for split in 'aihub_indoor_val' 'aihub_indoor_test'
+for split in 'aihub_indoor_val'
 do
 data=../../datasets/finetune/${dataset}/${split}.tsv
 result_path=../../results_${model}/${dataset}/

run_scripts/finetune/train_polyformer_b_aihub.sh

@@ -22,7 +22,7 @@ data=${data_dir}/aihub_indoor/aihub_indoor_train.tsv,${data_dir}/aihub_indoor/ai
 selected_cols=0,5,6,2,4,3,7
 # restore_file=../../weights/polyformer_b_pretrain.pt
 # restore_file=../pretrain/polyformer_b_pretrain_aihub_indoor_checkpoints/20_5e-5_512/checkpoint_20_1000.pt
-restore_file=../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume/20_5e-5_512/checkpoint.best_score_0.5110.pt
+restore_file=../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume_2/20_5e-5_512/checkpoint.best_score_0.5220.pt
 
 
 

run_scripts/pretrain/pretrain_polyformer_b_aihub_indoor.sh

@@ -7,8 +7,8 @@ export MASTER_PORT=6061
 det_weight=1
 cls_weight=0
 num_bins=64
-log_dir=./polyformer_b_pretrain_aihub_indoor_80_logs_resume
-save_dir=./polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume
+log_dir=./polyformer_b_pretrain_aihub_indoor_80_logs_resume_2
+save_dir=./polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume_2
 mkdir -p $log_dir $save_dir
 
 bpe_dir=../../utils/BPE
@@ -39,8 +39,7 @@ max_tgt_length=420
 
 patch_image_size=512
 
-restore_file=../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints/20_5e-5_512/checkpoint_16_5000.pt
-
+restore_file=../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume/20_5e-5_512/checkpoint.best_score_0.5110.pt
 
 for max_epoch in 20; do
   echo "max_epoch "${max_epoch}