Ans #1 +Coding
+ += +Friday, +May 15, 2020
+ += +1:12 +AM
+ +![](3D"Ans1Coding_files/image001.png")
def to_input_tensor_char(self, s= +ents: List[List[str]], device: torch.device) -> torch.Tens= +or:
+ +&n= +bsp; """ Convert l= +ist of sentences (words) into tensor with&nbs= +p;necessary padding for
+ +shorter sentences.
+ +@param sents (Li= +st[List[str]]): list of sentences (words)
+ +@param device: d= +evice on which to load the tensor, i.e.&= +nbsp;CPU or GPU
+ +@returns sents_var:&n= +bsp;tensor of (max_sentence_length, batch_size, max_wor= +d_length)
+ +"""
+ +&n= +bsp; ### YOUR CODE HERE = +;for part 1e
+ +&n= +bsp; ### TODO:
+ +&n= +bsp; ### -&nbs= +p;Use `words2charindices()` from this file, which&= +nbsp;converts each character to its corresponding&= +nbsp;index in the
+ +&n= +bsp; ### = +; character-vocabulary.
+ +&n= +bsp; ### -&nbs= +p;Use `pad_sents_char()` from utils.py, which pads= + all words to max_word_length of all wor= +ds in the batch,
+ +&n= +bsp; ### = +; and pads all sentences to max length&n= +bsp;of all sentences in the batch. Read = +__init__ to see how to get
+ +&n= +bsp; ### = +; index of character-padding token
+ +&n= +bsp; ### -&nbs= +p;Connect these two parts to convert the = +;resulting padded sentences to a torch tensor= +.
+ +&n= +bsp; ### HINT:
+ +&n= +bsp; ### -&nbs= +p;You may find .contiguous() useful after res= +haping. Check the following links for more&nb= +sp;details:
+ +&n= +bsp; ### = +; https://pytorch.org/docs/stable/tensor= +s.html#torch.Tensor.contiguous
+ + &n=
+bsp; ### =
+;
&n= +bsp; word_ids =3D self.words2charindices(sents) # @return word_ids (list[list[list[int]]]):&nbs= +p;sentence(s) in indices
+ +&n= +bsp; # shape (batch_size, ma= +x_sentence_length, max_word_length)
+ +&n= +bsp; sents_padded =3D pad_sents_char(word_ids, self.char_pad)
+ +&n= +bsp; # tensor size (max_sent= +ence_length, batch_size, max_word_length)
+ + &n=
+bsp; sent_var =3D torch.tensor(sents_padded).permute(1,0
&n= +bsp; return sent_var
+ +&n= +bsp; ### END YOUR CODE
+ +class Highway(nn.Module):
+ +"""
+ +Class that computes X_highway = +from X_conv_out
+ +"""
+ +# Remember to delete&nb= +sp;the above 'pass' after your implementation
+ +### YOUR CODE HERE = +;for part 1f
+ +def __init__(self, word_embed_size):
+ +"""
+ +Init Highway Layer.
+ +@param word_embed_size (int)= +: Embedding size (dimensionality) of word
+ ++ +
"""
+ +super(Highway, self).__init__() # Initialize self._modules&n= +bsp;as OrderedDict
+ +self.word_embed_size =3D word_embed_size
+ ++ +
# default values
+ +self.w_proj =3D None
+ +self.w_gate =3D None
+ ++ +
# initialize variables
+ +# torch.nn.Linear(in_features,&nb= +sp;out_features, bias=3DTrue)
+ +self.w_proj =3D nn.Linear(word_embed_size, word_emb= +ed_size, bias=3DTrue) #&nbs= +p;W_project
+ +self.w_gate =3D nn.Linear(word_embed_size, word_emb= +ed_size, bias=3DTrue) #&nbs= +p;W_gate
+ ++ +
def forward(self, x):
+ +"""Maps x_conv_out= + to x_highway
+ +# nn.Linear
+ +@param x (tensor): x_co= +nv_out tensor of shape (b, word_embed_size), = +
+ +&n= +bsp; = +;where b =3D batch size
+ +@returns x_highway (tensor):= + tenosor of shape (b, word_embed_size)
+ +"""
+ +x_= +proj =3D F.relu(self.w_proj(= +x)) # (b, word_embed_size)
+ +x_= +gate =3D torch.sigmoid(self.w_gate(= +x)) # (b, word_embed_size)
+ +# element wise multipli= +cation: * or mul()
+ +x_= +highway =3D x_gate * x_proj + (1 -&nbs= +p;x_gate) * x # (b, word_embed_size)
+ +return x_highway
+ +### END YOUR CODE
+ +%cd /content/drive/My Drive/cs224n/assignment5
+ +from highway import Highway
+ +import torch
+ +import torch.nn as nn
+ +# We know that x_conv_out =3D MaxPo= +ol（ReLU (x_cov)) and
+ +# that x_cov_out has a shape of (b,&nbs= +p;word_embed_size), where b is the batch size= +
+ +# and word_embed_size is the size of fi= +nal word embedding for word x.
+ +# Having been through a ReLU function, = +the values within each x_conv_out
+ +# must be greater or equal to 0, a= +nd having been through the MaxPool function,<= +/p> + +
# these values are presumbly not too sm= +all.
+ +# Using the above logic, we construct d= +ummy test_x_conv_out with shape (3, 4) for&nb= +sp;sanity check,
+ +# and we expect the output x_highway to= + have the same shape as input test_x_con= +v_out.
+ +#
+ +print('=3D=3D=3D= +=3DTEST HIGHWAY=3D=3D=3D=3D')
+ +print('---- = +Step-by-step Test----')
+ +word_embed_size =3D <= +span +style=3D'color:#899C7F' lang=3Dzh-CN>4
+ +print('Use w= +ord_embed_size ', word_embed_size)
+ +test_x_conv_out =3D torch.te=
+nsor([[0,
print('x_conv_out= + shape', test_x_conv_out.shape, ' expected [3, 4]')
+ +print(test_x_conv_out)
+ +w_proj =3D nn.Linear<= +span +style=3D'color:#979797' lang=3Dzh-CN>(word_embed_size, word_embed_size= +, bias=3DTrue) #&nbs= +p;W_project
+ +x_proj =3D F.relu(w_proj(test_x_conv_out)) = +# (b, word_embed_size)
+ +print('x_conv_out= + =3D> x_proj, shape ', x_proj.shape, ' expected [3, 4]')
+ +w_gate =3D nn.Linear<= +span +style=3D'color:#979797' lang=3Dzh-CN>(word_embed_size, word_embed_size= +, bias=3DTrue) #&nbs= +p;W_gate
+ +x_gate =3D torch.sigmoid(w_gate(test_x_conv_out)) = +# (b, word_embed_size)
+ +print('x_conv_out= + =3D> x_gate, shape ', x_gate.shape, ' expected [3, 4]')
+ +# element wise multiplication: * or mul()
+ +x_highway =3D x_gate *&= +nbsp;x_proj + (1 -&nbs= +p;x_gate) * test_x_conv_out # (b, word_embed_size)
+ +print('x_highway.=
+shape ', x_highway.shape,
print(x_highway)
+ +print('----Class&= +nbsp;Test----')
+ +print('Use w= +ord_embed_size 8')
+ +# Use word_embed_size =3D 8, and generate&nb= +sp;test tensor accordingly.
+ +hw =3D Highway(8)
+ +dummy_x_conv_out =3D torch.r= +and(10, 8)
+ +print('dummy_x_co= +nv_out shape ', dummy_x_conv_out.shape, ' expected [10, 8= +]')
+ +x_highway =3D hw(dummy_x_c= +onv_out)
+ +print('x_highway&=
+nbsp;shape ', x_highway.shape,
class CNN(nn.Module):
+ +"""
+ +Class of Convolution Neural Ne= +twork
+ +that applys kernel over x_resh= +aped to compute x_conv_out
+ +"""
+ +#pass
+ +# Remember to delete&nb= +sp;the above 'pass' after your implementation
+ +### YOUR CODE HERE = +;for part 1g
+ +def __init__(self, word_embed_size, char_embe= +d_size=3D50, k=3D5= +, padding=3D1): + +
&n= +bsp; """
+ +Init CNN Layers.= +
+ +@param word_embed_siz= +e (int): size of word embedding
+ +@param k (int):&= +nbsp;kernel size for convolution
+ +@padding (int): = +size of padding applied to x_reshaped bilater= +ally
+ ++ +
"""
+ +&n= +bsp; super(CNN, s= +elf).__init__() # Initialize self._modules&n= +bsp;as OrderedDict
+ +&n= +bsp; self.word_embed_size =3D word_embed_size
+ +&n= +bsp; self.char_embed_size =3D char_embed_size
+ +&n= +bsp; self.k =3D k
+ +&n= +bsp; self.padding =3D padding
+ +&n= +bsp; # Default Values
+ +&n= +bsp; self.apply_conv =3D None
+ +&n= +bsp; self.apply_maxpool =3D None
+ ++ +
&n= +bsp; # Initialize Variables
+ +&n= +bsp; """
+ +torch.nn.Conv1d(in_channel= +s, #
+ +&n= +bsp; out_c= +hannels, # f =3D number of output channe= +ls =3D word_embed_size
+ +&n= +bsp; kerne= +l_size, # k=3D5
+ +&n= +bsp; strid= +e=3D1,
+ +&n= +bsp; paddi= +ng=3D0, # padding=3D1
+ +&n= +bsp; dilat= +ion=3D1,
+ +&n= +bsp; group= +s=3D1,
+ +&n= +bsp; bias= +=3DTrue,
+ +&n= +bsp; paddi= +ng_mode=3D'zeros'
+ +&n= +bsp; )
+ +"""
+ +&n= +bsp; self.apply_conv =3D nn.Conv1d(in_channels =3D = +self.char_embed_size,
+ +&n= +bsp; = +; out_chan= +nels =3D self.word_embed_size,
+ +&n= +bsp; = +; kernel_s= +ize=3Dself.k,
+ +&n= +bsp; = +; padding= +=3Dself.padding
+ +&n= +bsp; = +; )
+ +def forward(self, x):
+ +&n= +bsp; """
+ +@param x (tensor= +): x_reshaped in A5 handout,
+ +&n= +bsp; tensor of&nb= +sp;shape (b, char_embed_size, m_word) ,
+ +&n= +bsp; where b = +;=3D batch size,
+ +&n= +bsp; char_embed_size&n= +bsp;=3D size of the character embedding, and<= +/p> + +
&n= +bsp; m_word =3D&n= +bsp;length of longest word in the batch.
+ +@return x_conv_out&nb= +sp;(tensor): tensor of shape (b, word_embed_size)<= +/p> + +
"""
+ +&n= +bsp; # x_conv shape (b, = +;word_embed_size, m_word-k+1)
+ +&n= +bsp; x_conv =3D self.apply_conv(x) # (b, word_embed_size, m_word+2*padding-k+1)<= +/span>
+ +&n= +bsp; m_word =3D x.shape[2]
+ +&n= +bsp; """
+ +torch.nn.MaxPool1d(kernel_= +size,
+ +&n= +bsp; = +; stride=3DNone,
+ +&n= +bsp; = +; padding=3D0,
+ +&n= +bsp; = +; dilation=3D1,
+ +&n= +bsp; = +; return_indices=3DFalse,
+ +&n= +bsp; = +; ceil_mode=3DFalse
+ +&n= +bsp; = +; )
+ +"""
+ +&n= +bsp; apply_maxpool =3D nn.MaxPool1d(kernel_size =3D m_word = +;+ 2*self= +.padding - self.k + 1)
+ +x_conv_out&nbs= +p;=3D apply_maxpool(F.relu(x_conv)).squeeze()
+ ++ +
&n= +bsp; return x_conv_out
+ ++ +
### END YOUR CODE
+ +%cd /content/drive/My Drive/cs224n/assignment5
+ +from cnn import CNN
+ +import torch
+ +import torch.nn as nn
+ +import torch.nn.functional as F
+ +"""
+ +We know that the convolution layers takes&nb= +sp;input x_reshaped,
+ +which has a shape of (b, f, m_word),&nb= +sp;where
+ +b =3D batch size
+ +f =3D word_embed_size
+ +m_word =3D the length of = +longest word in the batch.
+ +Given k =3D 5 and padding =3D 1, w= +e create a dummy test tensor
+ +with b =3D 4, char_embed_size =3D 8 and= + m_word =3D 10, in shape of (4,8,10),
+ +and feed this test tensor through CNN w= +ith word_embed_size =3D 16
+ +We expect the output from CNN to have&n= +bsp;a shape of
+ +(b, word_embed_size) =3D (4, 16).
+ +For intermediary steps, the following shapes = +;are expected:
+ +x_conv (b, word_embed_size, m_word + 2*padding&nb= +sp;- k +1) =3D (4, 16, 8)
+ +"""
+ +print('----------= +---------CNN TEST---------------------')
+ +print('----Test&n= +bsp;Inputs----')
+ +test_word_embed_size =3D 16
+ +print('word_embed= +_size ', test_word_embed_size)
+ +test_char_embed_size =3D 8
+ +print('char_embed= +_size ', char_embed_size)
+ +k =3D 5
+ +print('k ', k)
+ +b =3D 4
+ +print('b ', b)
+ +m_word =3D 10
+ +print('m_word&nbs= +p;', m_word)
+ +padding =3D 1
+ +print('padding&nb= +sp;', padding)
+ +print('--------ST= +EP BY STEP TEST----------')
+ +# x_reshaped in shape of (b, char_embed_size= +, m_word)
+ +test_x_reshaped =3D torch.rand(b, test_char_embed_size, m_word)
+ +print('test_x_res= +haped shape ', test_x_reshaped.shape, ' expected [4, 8, = +10]')
+ +apply_conv =3D nn.Conv1d(i= +n_channels=3Dtest_char_embed_size,
+ +&n= +bsp; out_channels=3D&n= +bsp;test_word_embed_size,
+ +&n= +bsp; kernel_size=3D&nb= +sp;k,
+ +&n= +bsp; padding=3D padding
+ +&n= +bsp; )
+ +x_conv =3D apply_conv(test= +_x_reshaped)
+ +print('x_conv&nbs= +p;shape ', x_conv.shape, ' expected [4, 16, = +;8]')
+ +mp =3D nn.MaxPool1d
#relu_x_conv =3D F.relu(x_conv)
+ +#print(relu_x_conv)
+ +x_conv_out =3D mp(F.relu(x= +_conv)).squeeze()
+ +print('x_conv_out= + shape ', x_conv_out.shape, <= +span +style=3D'color:#CB8F76' lang=3Dzh-CN>' expected [4, 16]')
+ +#print(x_conv_out)
+ ++ +
print('--------CL= +ASS TEST----------')
+ +cnn =3D CNN(test_word_embe= +d_size, test_char_embed_size, k, padding)
+ +cnn_x_cov_out =3D cnn.forward(test_x_reshaped)
+ +# Expected shape (b, word_embed_size, 1), af= +ter squeeze (b, word_embed_size)
+ +print('\ncnn_x_co= +v_out shape ', cnn_x_cov_out.shape, ' expected [4, 16]')
+ +print(cnn_x_cov_out)
+ +def __init__(= +)
+ ++ +
&n= +bsp; ### INITIALIZE VARIABLES
+ +&n= +bsp; """
+ +torch.nn.Embedding(num_emb= +eddings,
+ +&n= +bsp; = +; embedding_dim,
+ +&n= +bsp; = +; padding_idx=3DNone,
+ +&n= +bsp; = +; max_norm=3DNone,
+ +&n= +bsp; = +; norm_type=3D2.0,
+ +&n= +bsp; = +; scale_grad_by_freq=3DFalse,
+ +&n= +bsp; = +; sparse=3DFalse,
+ +&n= +bsp; = +; _weight=3DNone)
+ ++ +
Input: (*) ,&nbs= +p;LongTensor of arbitrary shape containing the&nbs= +p;indices to extract
+ +Output: (*, H),&= +nbsp;where * is the input shape and H=3D= +embedding_dim
+ +"""
+ +&n= +bsp; self.char_embedding =3D nn.Embedding(num_embeddings = +;=3D self.num_char_embeddings,
+ +&n= +bsp; = +; &n= +bsp; embed= +ding_dim =3D self.char_embed_size,
+ +&n= +bsp; = +; &n= +bsp; paddi= +ng_idx =3D self.char_pad
+ +&n= +bsp; = +; &n= +bsp; )
+ +&n= +bsp; self.cnn =3D CNN(self.word_embed_size, self.char_embed_size, self.k, = +;self.padding)
+ +&n= +bsp; self.highway =3D Highway(self.word_em= +bed_size)
+ +&n= +bsp; self.dropout =3D nn.Dropout(p=3Dself.dropout= +_rate)
+ ++ +
&n= +bsp; ### END YOUR CODE
+ +def forward()= +
+ ++ +
&n= +bsp; ### YOUR CODE HERE = +;for part 1h
+ ++ +
&n= +bsp; # [max_sentence_length, b,&n= +bsp;m_word, char_embed_size]
+ +&n= +bsp; X_char_embedding =3D self.char_embedding(input)
+ ++ +
&n= +bsp; # X_char_embedding =3D>&n= +bsp;X_reshaped (b*max_sentence_length, char_embed_size, m_wo= +rd)
+ +max_len_sents<= +/span>, b_sents, m_word, char_embed_size =3D&nbs= +p;X_char_embedding.shape
+ +X_reshaped&nbs= +p;=3D X_char_embedding.reshape(max_len_sents*b_sents, m_word, char_embed_size)
+ +&n= +bsp; X_reshaped =3D X_reshaped.permute(0, 2, 1).contiguous()
+ ++ +
&n= +bsp; # X_reshaped =3D> cn= +n =3D> X_conv_out
+ +&n= +bsp; # X_conv_out: [b, word_= +embed_size]
+ +&n= +bsp; X_conv_out =3D self.cnn(X_reshaped)
+ ++ +
&n= +bsp; # X_conv_out =3D>&n= +bsp;highway =3D> X_highway
+ +&n= +bsp; # X_highway: [b, word_e= +mbed_size]
+ +&n= +bsp; X_highway =3D self.highway(X_conv_out)
+ ++ +
&n= +bsp; # X_highway =3D> dro= +pout =3D> X_word_embed
+ +&n= +bsp; # X_word_embed: [b, wor= +d_embed_size]
+ +&n= +bsp; X_word_embed =3D self.dropout(X_highway)
+ ++ +
&n= +bsp; # Reshape X_word_embed = +to sents level, [sentence_length, batch_size, word= +_embed_size]
+ +&n= +bsp; # b =3D max_len_sents*b_sents
+ +&n= +bsp; X_word_embed =3D X_word_embed.reshape(max_len_sents, b_sents, -1).contiguous()
+ ++ +
&n= +bsp; return X_word_embed
+ ++ +
&n= +bsp; ### END YOUR CODE
+ +Adjust embed_s= +ize +=3D=3D> word_embed_size
+ +&= +nbsp;
+ +def forward()<= +/p> + +
+ +
&n= +bsp; ### YOUR CODE HERE = +;for part 1i
+ +&n= +bsp; ### TODO:
+ +&n= +bsp; ### Modif= +y the code lines above as needed to = +;fetch the character-level tensor
+ +&n= +bsp; ### to&nb= +sp;feed into encode() and decode(). You shoul= +d:
+ +&n= +bsp; ### -&nbs= +p;Keep `target_padded` from A4 code above for= + predictions
+ +&n= +bsp; ### -&nbs= +p;Add `source_padded_chars` for character level pa= +dded encodings for source
+ +&n= +bsp; ### -&nbs= +p;Add `target_padded_chars` for character level pa= +dded encodings for target
+ +&n= +bsp; ### -&nbs= +p;Modify calls to encode() and decode() to&nb= +sp;use the character level encodings
+ ++ +
#source_padded =3D self.vocab.src.to_input_tensor(source, d= +evice=3Dself.device) # [src_len, b] from&nb= +sp;A4
+ +&n= +bsp; target_padded =3D self.vocab.tgt.to_input_tensor(target, device=3Dself.device) # [tgt_len, b]
+ ++ +
&n= +bsp; # tensor of [max_senten= +ce_length, b, max_word_length]
+ +source_padded_chars = +=3D self.vocab.src.to_input_tensor_char(source, device=3Dself.device) # [src_m_sent_len, b, src_m_word]
+ +target_padded_chars = +=3D self.vocab.tgt.to_input_tensor_char(target, device=3Dself.device) # [tgt_m_sent_len, b, tgt_m_word]
+ ++ +
&n= +bsp; ### Run&n= +bsp;the network forward:
+ +&n= +bsp; ### 1.&nb= +sp;Apply the encoder to `source_padded` by ca= +lling `self.encode()`
+ +&n= +bsp; ### 2.&nb= +sp;Generate sentence masks for `source_padded` by&= +nbsp;calling `self.generate_sent_masks()`
+ +&n= +bsp; ### 3.&nb= +sp;Apply the decoder to compute combined-output&nb= +sp;by calling `self.decode()`
+ +&n= +bsp; ### 4.&nb= +sp;Compute log probability distribution over the&n= +bsp;target vocabulary using the
+ +&n= +bsp; ### = +; combined_outputs returned by the `self.dec= +ode()` function.
+ ++ +
&n= +bsp; enc_hiddens, dec_init_state =3D self.encode(source_padded_chars, source_lengths)
+ +&n= +bsp; enc_masks =3D self.generate_sent_masks(enc_hiddens, source_lengths)<= +/p> + +
&n= +bsp; combined_outputs =3D self.decode(enc_hiddens, enc_masks, dec_init_state, = +;target_padded_chars)
+ ++ +
= +;### END YOUR COD= +E
+ ++ +
![](3D"Ans1Coding_files/image009.png")
&= +nbsp;
+ +![](3D"Ans1Coding_files/image010.png")
![3D"Untitled](3D"Ans1Written_files/image001.png")
![](3D"Ans1Written_files/image002.png")
![3D"Ink](3D"Ans1Written_files/image003.png")
![](3D"Ans1Written_files/image006.png")
![](3D"Ans2Coding_files/image001.png")
![](3D"Ans3_files/image001.png")
![3D"Untitled](3D"Ans3_files/image004.png")
![3D"Untitled](3D"Ans3_files/image005.png")
![3D"Machine](3D"Ans3_files/image006.png")
![3D"Machine](3D"Ans3_files/image007.png")
![3D"Machine](3D"Ans3_files/image008.png")
![3D"Machine](3D"Ans3_files/image009.png")
![3D"Machine](3D"Ans3_files/image010.png")
![3D"Machine](3D"Ans3_files/image011.png")