traverse the code, fixed bugs

Now, the model initialization part is working

Author: ZhengTang1120

main/src/main/python/embeddings/wordEmbeddingMap.py

@@ -1,29 +1,29 @@
 import numpy as np
+import math
 
 class WordEmbeddingMap:
     def __init__(self, config):
-        self.emb_dict = self.load(config)
-        self.dim = self.emb_dict.shape[-1]
-
-    def load(self):
-        emb_matrix = None
-        emb_dict = dict()
-        for line in open(config.get_string("glove.matrixResourceName")):
-            if not len(line.split()) == 2:
-                if "\t" in line:
-                    delimiter = "\t"
-                else:
-                    delimiter = " "
-                line_split = line.rstrip().split(delimiter)
-                # extract word and vector
-                word = line_split[0]
-                x = np.array([float(i) for i in line_split[1:]])
-                vector = (x /np.linalg.norm(x))
-                embedding_size = vector.shape[0]
-                emb_dict[word] = vector
-        base = math.sqrt(6/embedding_size)
-        emb_dict["<UNK>"] = np.random.uniform(-base,base,(embedding_size))
-        return emb_dict
+        self.emb_dict, self.dim = load(config)
 
     def isOutOfVocabulary(self, word):
-        return word not in self.emb_dict
+        return word not in self.emb_dict
+
+def load(config):
+    emb_matrix = None
+    emb_dict = dict()
+    for line in open(config.get_string("glove.matrixResourceName")):
+        if not len(line.split()) == 2:
+            if "\t" in line:
+                delimiter = "\t"
+            else:
+                delimiter = " "
+            line_split = line.rstrip().split(delimiter)
+            # extract word and vector
+            word = line_split[0]
+            x = np.array([float(i) for i in line_split[1:]])
+            vector = (x /np.linalg.norm(x))
+            embedding_size = vector.shape[0]
+            emb_dict[word] = vector
+    base = math.sqrt(6/embedding_size)
+    emb_dict["<UNK>"] = np.random.uniform(-base,base,(embedding_size))
+    return emb_dict, embedding_size

main/src/main/python/pytorch/constEmbeddingsGlove.py

@@ -1,16 +1,18 @@
 from dataclasses import dataclass
 import torch.nn as nn
 from embeddings.wordEmbeddingMap import *
+from pyhocon import ConfigFactory
 
 @dataclass
 class ConstEmbeddingParameters:
     emb: nn.Embedding
     w2i: dict
 
-def ConstEmbeddingsGlove:
+class _ConstEmbeddingsGlove:
     def __init__(self):
         self.SINGLETON_WORD_EMBEDDING_MAP = None
-        self.load('../resources/org/clulab/glove.conf')
+        config = ConfigFactory.parse_file('../resources/org/clulab/glove.conf')
+        self.load(config)
         self.dim = self.SINGLETON_WORD_EMBEDDING_MAP.dim
 
     def load(self, config):
@@ -25,3 +27,5 @@ def mkConstLookupParams(self, words):
         emd = nn.Embedding.from_pretrained(weight)
         emd.weight.requires_grad=False
         return ConstEmbeddingParameters(emb ,w2i)
+
+ConstEmbeddingsGlove = _ConstEmbeddingsGlove()

main/src/main/python/pytorch/embeddingLayer.py

@@ -1,8 +1,9 @@
-from initialLayer import InitialLayer
+from pytorch.initialLayer import InitialLayer
 import random
-from utils import *
+from pytorch.utils import *
 import torch.nn as nn
 import torch
+from pytorch.constEmbeddingsGlove import ConstEmbeddingsGlove
 
 DEFAULT_DROPOUT_PROB: float = DEFAULT_DROPOUT_PROBABILITY
 DEFAULT_LEARNED_WORD_EMBEDDING_SIZE: int = 128
@@ -16,7 +17,7 @@
 DEFAULT_USE_IS_PREDICATE: int = -1
 
 class EmbeddingLayer(InitialLayer):
-    def __init__(w2i, # word to index
+    def __init__(self, w2i, # word to index
                  w2f, # word to frequency
                  c2i, # character to index
                  tag2i, # POS tag to index
@@ -68,7 +69,7 @@ def __init__(w2i, # word to index
         positionDim = 1 if distanceLookupParameters and useIsPredicate else 0
         predicateDim = positionEmbeddingSize if positionLookupParameters else 0
 
-        self.outDim =    ConstEmbeddingsGlove().dim + learnedWordEmbeddingSize + charRnnStateSize * 2 + posTagDim + neTagDim + distanceDim + positionDim + predicateDim
+        self.outDim =    ConstEmbeddingsGlove.dim + learnedWordEmbeddingSize + charRnnStateSize * 2 + posTagDim + neTagDim + distanceDim + positionDim + predicateDim
         random.seed(RANDOM_SEED)
 
     def forward(self, sentence, constEmbeddings, doDropout):
@@ -81,9 +82,9 @@ def forward(self, sentence, constEmbeddings, doDropout):
         # const word embeddings such as GloVe
         constEmbeddingsExpressions = self.mkConstEmbeddings(words, constEmbeddings)
         assert(constEmbeddingsExpressions.size(0) == len(words))
-        if(tags) assert(len(tags) == len(words))
-        if(nes) assert(len(nes) == len(words))
-        if(headPositions) assert(len(headPositions) == len(words))
+        if(tags): assert(len(tags) == len(words))
+        if(nes): assert(len(nes) == len(words))
+        if(headPositions): assert(len(headPositions) == len(words))
 
         # build the word embeddings one by one
         embeddings = self.mkEmbeddings(words, constEmbeddingsExpressions, tags, nes, headPositions)
@@ -250,7 +251,7 @@ def load(cls, x2i):
     @classmethod
     def initialize(cls, config, paramPrefix, wordCounter):
 
-        if(not config.__contains__(paramPrefix)):
+        if(not config.contains(paramPrefix)):
             return None
 
         learnedWordEmbeddingSize = config.get_int(paramPrefix + ".learnedWordEmbeddingSize",DEFAULT_LEARNED_WORD_EMBEDDING_SIZE)
@@ -260,9 +261,9 @@ def initialize(cls, config, paramPrefix, wordCounter):
         neTagEmbeddingSize       = config.get_int(paramPrefix + ".neTagEmbeddingSize",DEFAULT_NE_TAG_EMBEDDING_SIZE)
         distanceEmbeddingSize    = config.get_int(paramPrefix + ".distanceEmbeddingSize",DEFAULT_DISTANCE_EMBEDDING_SIZE)
         distanceWindowSize       = config.get_int(paramPrefix + ".distanceWindowSize",DEFAULT_DISTANCE_WINDOW_SIZE)
-        useIsPredicate           = config.getArgBoolean(paramPrefix + ".useIsPredicate",DEFAULT_USE_IS_PREDICATE == 1)
+        useIsPredicate           = config.get_bool(paramPrefix + ".useIsPredicate",DEFAULT_USE_IS_PREDICATE == 1)
         positionEmbeddingSize    = config.get_int(paramPrefix + ".positionEmbeddingSize",DEFAULT_POSITION_EMBEDDING_SIZE)
-        dropoutProb              = config.get_float(paramPrefix + ".dropoutProb",EmbeddingLayer.DEFAULT_DROPOUT_PROB)
+        dropoutProb              = config.get_float(paramPrefix + ".dropoutProb",DEFAULT_DROPOUT_PROB)
 
         wordList = [UNK_WORD] + sorted(wordCounter.keys())
         w2i = {w:i for i, w in enumerate(wordList)}
@@ -293,7 +294,7 @@ def initialize(cls, config, paramPrefix, wordCounter):
         distanceLookupParameters = nn.Embedding(distanceWindowSize * 2 + 3, distanceEmbeddingSize) if distanceEmbeddingSize > 0 else None
         positionLookupParameters = nn.Embedding(101, positionEmbeddingSize) if positionEmbeddingSize > 0 else None
 
-        return cls(w2i, w2f, c2i, tag2i, ne2i,
+        return cls(w2i, wordCounter, c2i, tag2i, ne2i,
                   learnedWordEmbeddingSize,
                   charEmbeddingSize,
                   charRnnStateSize,
@@ -331,6 +332,8 @@ def initialize(cls, config, paramPrefix, wordCounter):
 
 
 
+
+
 
 
 

main/src/main/python/pytorch/forwardLayer.py

@@ -3,14 +3,13 @@
 from torch.autograd import Variable
 import torch.nn.functional as F
 
-from finalLayer import FinalLayer
-from greedyForwardLayer import GreedyForwardLayer
-from viterbiForwardLayer import ViterbiForwardLayer
+from pytorch.finalLayer import FinalLayer
 
-from utils import *
+from pytorch.utils import *
 
 class ForwardLayer(FinalLayer):
     def __init__(self, inputSize, isDual, t2i, i2t, actualInputSize, nonlinearity, dropoutProb, spans = None):
+        super().__init__()
         self.inputSize = inputSize
         self.isDual = isDual
         self.t2i = t2i
@@ -73,6 +72,8 @@ def forward(inputExpressions, doDropout, headPositionsOpt = None):
 
     @staticmethod
     def load(x2i):
+        from pytorch.greedyForwardLayer import GreedyForwardLayer
+        from pytorch.viterbiForwardLayer import ViterbiForwardLayer
         inferenceType = x2i["inferenceType"]
         if inferenceType == TYPE_VITERBI:
             pass
@@ -85,7 +86,9 @@ def load(x2i):
 
     @staticmethod
     def initialize(config, paramPrefix, labelCounter, isDual, inputSize):
-        if(not config.__contains__(paramPrefix)):
+        from pytorch.greedyForwardLayer import GreedyForwardLayer
+        from pytorch.viterbiForwardLayer import ViterbiForwardLayer
+        if(not config.contains(paramPrefix)):
             return None
 
         inferenceType = config.get_string(paramPrefix + ".inference", "greedy")

main/src/main/python/pytorch/greedyForwardLayer.py

@@ -1,5 +1,5 @@
-from forwardLayer import *
-from utils import *
+from pytorch.forwardLayer import *
+from pytorch.utils import *
 import numpy as np
 
 class GreedyForwardLayer(ForwardLayer):
@@ -23,7 +23,7 @@ def saveX2i(self):
         return x2i
 
     def __str__(self):
-        return f"GreedyForwardLayer({inDim}, {outDim})"
+        return f"GreedyForwardLayer({self.inDim}, {self.outDim})"
 
     def inference(self, emissionScores):
         labelIds = np.argmax(lattice.data.numpy(), axis=1).tolist()

main/src/main/python/pytorch/layers.py

@@ -1,7 +1,9 @@
 import torch.nn as nn
-from utils import *
-from embeddingLayer import EmbeddingLayer
-from constEmbeddingsGlove import ConstEmbeddingsGlove
+from pytorch.utils import *
+from pytorch.embeddingLayer import EmbeddingLayer
+from pytorch.rnnLayer import RnnLayer
+from pytorch.forwardLayer import ForwardLayer
+from pytorch.constEmbeddingsGlove import ConstEmbeddingsGlove
 
 class Layers(object):
     def __init__(self, initialLayer, intermediateLayers, finalLayer):
@@ -14,8 +16,7 @@ def __init__(self, initialLayer, intermediateLayers, finalLayer):
         else:
             self.outDim = None
 
-        if initialLayer and intermediateLayers and finalLayer:
-            self.nonEmpty = True
+        self.nonEmpty = initialLayer is not None and intermediateLayers is not None and finalLayer is not None
         self.isEmpty = not self.nonEmpty
 
         self.initialLayer = initialLayer
@@ -25,43 +26,53 @@ def __init__(self, initialLayer, intermediateLayers, finalLayer):
     def __str__(self):
         s = ""
         started = False
-        if(initialLayer.nonEmpty):
-            s += "initial = " + initialLayer
+        if(self.initialLayer is not None):
+            s += "initial = " + str(self.initialLayer)
             started = True
-        for i in intermediateLayers.indices:
-            if(started) s += " "
-            s += s"intermediate ({i+1}) = " + intermediateLayers[i]
+        for i in range(len(self.intermediateLayers)):
+            if(started): s += " "
+            s += f"intermediate ({i+1}) = " + str(self.intermediateLayers[i])
             started = True
-        if(finalLayer.nonEmpty):
-          if(started) s += " "
-          s += "final = " + finalLayer
+        if(self.finalLayer is not None):
+          if(started): s += " "
+          s += "final = " + str(self.finalLayer)
         return s
 
+    def get_parameters(self):
+        parameters = list()
+        if self.initialLayer is not None:
+            parameters += [p for p in self.initialLayer.parameters() if p.requires_grad]
+        for il in self.intermediateLayers:
+            parameters += [p for p in il.parameters() if p.requires_grad]
+        if self.finalLayer is not None:
+            parameters += [p for p in self.finalLayer.parameters() if p.requires_grad]
+        return parameters
+
     def forward(self, sentence, constEmbeddings, doDropout):
         if self.initialLayer.isEmpty:
             raise RuntimeError(f"ERROR: you can't call forward() on a Layers object that does not have an initial layer: {self}!")
         states = self.initialLayer(sentence, constEmbeddings, doDropout)
         for intermediateLayer in self.intermediateLayers:
             states = intermediateLayer(states, doDropout)
-        if self.finalLayer.nonEmpty:
+        if self.finalLayer is not None:
             states = self.finalLayer(states, sentence.headPositions, doDropout)
 
         return states
 
     def forwardFrom(self, inStates, headPositions, doDropout):
-        if self.initialLayer.nonEmpty:
+        if self.initialLayer is not None:
             raise RuntimeError(f"ERROR: you can't call forwardFrom() on a Layers object that has an initial layer: {self}")
         states = inStates
         for intermediateLayer in self.intermediateLayers:
             states = intermediateLayer(states, doDropout)
-        if self.finalLayer.nonEmpty:
+        if self.finalLayer is not None:
             states = self.finalLayer(states, sentence.headPositions, doDropout)
 
         return states
 
     def saveX2i(self):
         x2i = dict()
-        if self.initialLayer.nonEmpty:
+        if self.initialLayer is not None:
             x2i['hasInitial'] = 1
             x2i['initialLayer'] = self.initialLayer.saveX2i()
         else:
@@ -70,7 +81,7 @@ def saveX2i(self):
         x2i['intermediateLayers'] = list()
         for il in self.intermediateLayers:
             x2i['intermediateLayers'].append(il.saveX2i())
-        if self.finalLayer.nonEmpty:
+        if self.finalLayer is not None:
             x2i['hasFinal'] = 1
             x2i['finalLayer'] = self.finalLayer.saveX2i()
         else:
@@ -227,7 +238,7 @@ def parse(layers, sentence, constEmbeddings):
     @staticmethod
     def loss(layers, taskId, sentence, goldLabels):
         # Zheng: I am not sure this is the suitable way to load embeddings or not, need help...
-        constEmbeddings = ConstEmbeddingsGlove().mkConstLookupParams(sentence.words)
+        constEmbeddings = ConstEmbeddingsGlove.mkConstLookupParams(sentence.words)
         states = Layers.forwardForTask(layers, taskId, sentence, constEmbeddings, doDropout=True) # use dropout during training!
         return layers[taskId+1].finalLayer.loss(states, goldLabels)