clean up repo

README.md

@@ -6,14 +6,12 @@ Song Wang, Liyan Tang, Mingquan Lin, George Shih, Ying Ding, Yifan Peng
 
 ## Overview
 
-The proposed model is defined in 'sentgcn.py'. 
-To train a model, first train a gcnclassifier (multi-label classifier) as shown in run_gcnclassifier.sh, then train a sentgcn (report generation decoder) as shown in run_sentgcn.sh.
+The proposed model is defined in `sentgcn.py`. 
 
-To train the gcnclassifier, you would need a DenseNet-121 model pretrained on ChexPert: model_ones_3epoch_densenet.tar.
+1. To train a model, first train a gcnclassifier (multi-label classifier) model as shown in `run_gcnclassifier.sh`;
+2. Then train a sentgcn (report generation decoder) model as shown in `run_sentgcn.sh` using the best gcnclassifier model from last step.
 
-The dataset splits, class keywords, embeddings and vocabs can be found in the data folder.
-
-## Usage
+To train the gcnclassifier, you would need a DenseNet-121 model pretrained on ChexPert `model_ones_3epoch_densenet.tar`. The dataset splits, class keywords, embeddings and vocabs can be found in the `/data` folder. Image paths can be modified inside `/data/fold{}.txt` files.
 
 
 ## Citation

constants.py

@@ -0,0 +1 @@
+FOLDER = '/data/'

mlclassifier.py

@@ -11,7 +11,6 @@ def __init__(self, num_classes):
         super().__init__()
         self.densenet121 = models.densenet121(pretrained=True)
         num_ftrs = self.densenet121.classifier.in_features
-        # self.backbone = nn.Sequential(*list(densenet.children())[:-1])
         self.densenet121.classifier = nn.Linear(num_ftrs, num_classes)
 
     def forward(self, img1, img2):
@@ -140,8 +139,7 @@ def forward(self, x, fw_A, bw_A):
 
         return x
 
-# our new GCN
-class GCN_new(nn.Module):
+class GCN(nn.Module):
 
     def __init__(self, in_size, state_size):
         super(GCN_new, self).__init__()
@@ -161,27 +159,6 @@ def forward(self, states, fw_A, bw_A):
 
         return states.permute(0,2,1)
 
-# original GCN
-class GCN(nn.Module):
-
-    def __init__(self, in_size, state_size, steps=3):
-        super().__init__()
-        self.in_size = in_size
-        self.state_size = state_size
-        self.steps = steps
-
-        self.layer1 = GCLayer(in_size, state_size)
-        self.layer2 = GCLayer(in_size, state_size)
-        self.layer3 = GCLayer(in_size, state_size)
-
-    def forward(self, states, fw_A, bw_A):
-        states = states.permute(0, 2, 1)
-        states = self.layer1(states, fw_A, bw_A)
-        states = self.layer2(states, fw_A, bw_A)
-        states = self.layer3(states, fw_A, bw_A)
-        return states.permute(0, 2, 1)
-
-
 class GCNClassifier(nn.Module):
 
     def __init__(self, num_classes, fw_adj, bw_adj):
@@ -193,11 +170,7 @@ def __init__(self, num_classes, fw_adj, bw_adj):
         self.densenet121.classifier = nn.Linear(feat_size, num_classes)
         self.cls_atten = ClsAttention(feat_size, num_classes)
 
-        # original GCN
-        #self.gcn = GCN(feat_size, 256)
-
-        # our new GCN
-        self.gcn = GCN_new(feat_size, 256)
+        self.gcn = GCN(feat_size, 256)
 
         self.fc2 = nn.Linear(feat_size, num_classes)
 

my_build_vocab.py

@@ -3,6 +3,7 @@
 import pickle
 from collections import Counter
 import nltk
+from constants import FOLDER
 
 class Vocabulary(object):
     """Simple vocabulary wrapper."""
@@ -27,10 +28,8 @@ def __len__(self):
 
 
 if __name__ == '__main__':
-
-    folder = '/home/sw37643/ReportGenerationMeetsGraph/data/'
 
-    with open( folder + 'reports.json') as f:
+    with open( FOLDER + 'reports.json') as f:
         reports = json.load(f)
 
     counter = Counter()
@@ -46,12 +45,6 @@ def __len__(self):
         text = text.replace('.', ' .')
         tokens = text.strip().split()
 
-        # tokens = []
-        # if report['findings'] is not None:
-        #     tokens = nltk.tokenize.word_tokenize(report['findings'].lower())
-        # if report['impression'] is not None:
-        #     tokens.extend(nltk.tokenize.word_tokenize(report['impression'].lower()))
-
         counter.update(tokens)
     words = [word for word, cnt in counter.items() if cnt >= 3]
 
@@ -62,10 +55,9 @@ def __len__(self):
     vocab.add_word('<unk>')
 
     for word in words:
-        #print(word)
         vocab.add_word(word)
 
-    with open(folder + 'vocab.pkl', 'wb') as f:
+    with open( FOLDER + 'vocab.pkl', 'wb') as f:
         pickle.dump(vocab, f)
 
     print('Total vocabulary size {}. Saved to {}'.format(len(vocab), 'vocab.pkl'))

process_emb.py

@@ -42,8 +42,6 @@ def read_word_embeddings(embeddings_file: str) -> WordEmbeddings:
     word_indexer.add_and_get_index("<pad>")
     word_indexer.add_and_get_index("<start>")
     word_indexer.add_and_get_index("<end>")
-    #word_indexer.add_and_get_index("<unk>")
-    #vectors.append(np.random.randn(200))
     vectors.append(np.random.randn(200))
     vectors.append(np.random.randn(200))
     vectors.append(np.random.randn(200))

run_gcnclassifier.sh

@@ -1,3 +1,3 @@
 #! /usr/bin/env bash
 
-python /home/sw37643/ReportGenerationMeetsGraph/train_gcnclassifier.py --name gcnclassifier_30keywords_t401v2t3 --pretrained /home/sw37643/ReportGenerationMeetsGraph/models/pretrained/model_ones_3epoch_densenet.tar --dataset-dir /home/sw37643/ReportGenerationMeetsGraph/data/NLMCXR_png --train-folds 401 --val-folds 2 --test-folds 3 --lr 1e-6 --batch-size 8 --gpus 0 --num-epochs 150
+python train_gcnclassifier.py --name gcnclassifier_30keywords_train401val2test3 --pretrained /models/pretrained/model_ones_3epoch_densenet.tar --dataset-dir /data/openi --train-folds 401 --val-folds 2 --test-folds 3 --lr 1e-6 --batch-size 8 --gpus 0 --num-epochs 150

run_sentgcn.sh

@@ -1,2 +1,2 @@
 #! /usr/bin/env bash
-python /home/sw37643/ReportGenerationMeetsGraph/train_sentgcn.py --name sentgcn_30keywords_t401v2t3_ctx --pretrained /home/sw37643/ReportGenerationMeetsGraph/models/gcnclassifier_30keywords_t401v2t3_e100.pth --train-folds 401 --val-folds 2 --test-folds 3 --gpus 0 --batch-size 8 --decoder-lr 1e-4 --vocab-path /home/sw37643/ReportGenerationMeetsGraph/data/vocab.pkl
+python train_sentgcn.py --name sentgcn_30keywords_train401val2test3 --pretrained models/gcnclassifier_30keywords_train401val2test3_e100.pth --train-folds 401 --val-folds 2 --test-folds 3 --gpus 0 --batch-size 8 --decoder-lr 1e-6 --vocab-path /data/vocab.pkl

sentgcn.py

@@ -62,15 +62,12 @@ def __init__(self, in_features, out_features, bias=True):
         super(GraphConvolution, self).__init__()
         self.in_features = in_features
         self.out_features = out_features
-        #self.weight = Parameter(torch.FloatTensor(8, in_features, out_features))
 
         if bias:
             self.bias = Parameter(torch.FloatTensor(out_features))
         else:
             self.register_parameter('bias', None)
 
-        #self.reset_parameters_xavier()
-
     def reset_parameters_xavier(self):
         nn.init.xavier_normal_(self.weight.data, gain=0.02) # Implement Xavier Uniform
         if self.bias is not None:
@@ -126,7 +123,7 @@ def forward(self, x, fw_A, bw_A):
         return x
 
 
-class GCN_new(nn.Module):
+class GCN(nn.Module):
 
     def __init__(self, in_size, state_size):
         super(GCN_new, self).__init__()
@@ -167,26 +164,6 @@ def forward(self, states, fw_A, bw_A):
         return updated
 
 
-class GCN(nn.Module):
-
-    def __init__(self, in_size, state_size, steps=3):
-        super().__init__()
-        self.in_size = in_size
-        self.state_size = state_size
-        self.steps = steps
-
-        self.layer1 = GCLayer(in_size, state_size)
-        self.layer2 = GCLayer(in_size, state_size)
-        self.layer3 = GCLayer(in_size, state_size)
-
-    def forward(self, states, fw_A, bw_A):
-        states = states.permute(0, 2, 1)
-        states = self.layer1(states, fw_A, bw_A)
-        states = self.layer2(states, fw_A, bw_A)
-        states = self.layer3(states, fw_A, bw_A)
-        return states.permute(0, 2, 1)
-
-
 class SentGCN(nn.Module):
 
     # random embeddings with dimension 256
@@ -205,10 +182,7 @@ def __init__(self, num_classes, fw_adj, bw_adj, vocab_size, embds, feat_size=102
         self.densenet121.classifier = nn.Linear(feat_size, num_classes)
         self.cls_atten = ClsAttention(feat_size, num_classes)
 
-        # original GCN
-        #self.gcn = GCN(feat_size, feat_size // 4, steps=3)
-        # our new GCN
-        self.gcn = GCN_new(feat_size, feat_size//4)
+        self.gcn = GCN(feat_size, feat_size//4)
         self.atten = Attention(hidden_size, feat_size)
 
         # random embeddings

train_gcnclassifier.py

@@ -14,36 +14,31 @@
 from torch.utils.data import DataLoader
 from torch.utils.tensorboard.writer import SummaryWriter
 from sklearn.metrics import precision_recall_fscore_support, roc_auc_score
-
-os.environ["CUDA_VISIBLE_DEVICES"]= "3"
-folder = '/home/sw37643/ReportGenerationMeetsGraph/'
+from constants import FOLDER
 
 def get_args():
     parser = argparse.ArgumentParser()
 
     parser.add_argument('--name', type=str, required=True)
-    parser.add_argument('--model-path', type=str, default= folder + 'models')
-    parser.add_argument('--pretrained', type=str, default= folder + 'models/pretrained/model_ones_3epoch_densenet.tar')
+    parser.add_argument('--model-path', type=str, default= FOLDER + 'models')
+    parser.add_argument('--pretrained', type=str, default= FOLDER + 'models/pretrained/model_ones_3epoch_densenet.tar')
     parser.add_argument('--checkpoint', type=str, default='')
-    parser.add_argument('--dataset-dir', type=str, default= folder + 'data/NLMCXR_png')
+    parser.add_argument('--dataset-dir', type=str, default= FOLDER + 'data/openi')
     parser.add_argument('--train-folds', type=str, default='012')
     parser.add_argument('--val-folds', type=str, default='3')
     parser.add_argument('--test-folds', type=str, default='4')
-    parser.add_argument('--report-path', type=str, default= folder + 'data/reports.json')
-    parser.add_argument('--vocab-path', type=str, default= folder + 'data/vocab.pkl')
-    parser.add_argument('--label-path', type=str, default= folder + 'data/label_dict.json')
-    parser.add_argument('--log-path', type=str, default= folder + 'logs')
+    parser.add_argument('--report-path', type=str, default= FOLDER + 'data/reports.json')
+    parser.add_argument('--vocab-path', type=str, default= FOLDER + 'data/vocab.pkl')
+    parser.add_argument('--label-path', type=str, default= FOLDER + 'data/label_dict.json')
+    parser.add_argument('--log-path', type=str, default= FOLDER + 'logs')
     parser.add_argument('--log-freq', type=int, default=1)
     parser.add_argument('--num-epochs', type=int, default=100)
     parser.add_argument('--seed', type=int, default=123)
     parser.add_argument('--lr', type=float, default=1e-6)
     parser.add_argument('--batch-size', type=int, default=8)
     parser.add_argument('--gpus', type=str, default='0')
     parser.add_argument('--clip-value', type=float, default=5.0)
-
-    #parser.add_argument('--num-classes', type=int, default=20)
     parser.add_argument('--num-classes', type=int, default=30)
-    #parser.add_argument('--num-classes', type=int, default=40)
 
     args = parser.parse_args()
 
@@ -62,63 +57,26 @@ def get_args():
     for k, v in vars(args).items():
         logging.info('{}: {}'.format(k, v))
 
-    writer = SummaryWriter(log_dir=os.path.join( folder + 'runs/openi_top30', args.name))
-    #writer = SummaryWriter(log_dir=os.path.join( folder + 'runs/openi_top40', args.name))
+    writer = SummaryWriter(log_dir=os.path.join( FOLDER + 'runs/openi_top30', args.name))
 
     device = torch.device('cuda:{}'.format(args.gpus[0]) if torch.cuda.is_available() else 'cpu')
     gpus = [int(_) for _ in list(args.gpus)]
     torch.manual_seed(args.seed)
 
-    #with open( folder + 'data/19class_keywords.txt') as f:
-    #    keywords = f.read().splitlines()
-
-    with open( folder + 'data/openi_top30/openi_30keywords.txt') as f:
+    with open( FOLDER + 'data/openi_top30/openi_30keywords.txt') as f:
         keywords = f.read().splitlines()
     keywords.append('others')
 
-    #with open( folder + 'data/openi_top40/openi_40keywords.txt') as f:
-    #    keywords = f.read().splitlines()
-
     train_set = Biview_Classification('train', args.dataset_dir, args.train_folds, args.label_path)
     train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=8)
     val_set = Biview_Classification('val', args.dataset_dir, args.val_folds, args.label_path)
     val_loader = DataLoader(val_set, batch_size=1, shuffle=False, num_workers=1)
     test_set = Biview_Classification('test', args.dataset_dir, args.test_folds, args.label_path)
     test_loader = DataLoader(test_set, batch_size=1, shuffle=False, num_workers=1)
 
-    with open( folder + 'data/openi_top30/auxillary_openi_matrix_30nodes.txt','r') as matrix_file:
+    with open( FOLDER + 'data/openi_top30/auxillary_openi_matrix_30nodes.txt','r') as matrix_file:
         adjacency_matrix = [[int(num) for num in line.split(', ')] for line in matrix_file]
 
-    #with open( folder + 'data/openi_top40/openi_matrix_40nodes_binary.txt','r') as matrix_file:
-    #    adjacency_matrix = [[int(num) for num in line.split(', ')] for line in matrix_file]
-
-    # original GCN
-    #fw_adj = torch.tensor([
-    #    [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    #], dtype=torch.float, device=device)
-    #bw_adj = fw_adj.t()
-
-    # our new GCN
     fw_adj = torch.tensor(adjacency_matrix, dtype=torch.float, device=device)
     bw_adj = fw_adj.t()
     identity_matrix = torch.eye(args.num_classes+1, device=device)
@@ -193,8 +151,6 @@ def get_args():
             model.eval()
             y = torch.zeros((len(val_set), 20), dtype=torch.int)
             y_score = torch.zeros((len(val_set), 20), dtype=torch.float)
-            #y = torch.zeros((len(val_set), args.num_classes), dtype=torch.int)
-            #y_score = torch.zeros((len(val_set), args.num_classes), dtype=torch.float)
             with torch.no_grad():
                 for i, (images1, images2, labels) in enumerate(val_loader):
                     images1, images2 = images1.to(device), images2.to(device)
@@ -216,8 +172,6 @@ def get_args():
             # test
             y = torch.zeros((len(test_set), 20), dtype=torch.int)
             y_score = torch.zeros((len(test_set), 20), dtype=torch.float)
-            #y = torch.zeros((len(test_set), args.num_classes), dtype=torch.int)
-            #y_score = torch.zeros((len(test_set), args.num_classes), dtype=torch.float)
             with torch.no_grad():
                 for i, (images1, images2, labels) in enumerate(test_loader):
                     images1, images2 = images1.to(device), images2.to(device)
@@ -239,11 +193,8 @@ def get_args():
             df = np.stack([p, r, f, roc_auc], axis=1)
             df = pd.DataFrame(df, columns=['precision', 'recall', 'f1', 'auc'])
 
-            #df_keywords = keywords[:19].append('others')
             df_keywords = keywords[:20]
-            #df.insert(0, 'name', keywords)
             df.insert(0, 'name', df_keywords)
-            df.to_csv(os.path.join( folder + 'output/openi_top30', args.name + '_e{}.csv'.format(epoch)))
-            #df.to_csv(os.path.join( folder + 'output/openi_top40', args.name + '_e{}.csv'.format(epoch)))
+            df.to_csv(os.path.join( FOLDER + 'output/openi_top30', args.name + '_e{}.csv'.format(epoch)))
 
     writer.close()