diff --git a/TFFactory/DataLoader.py b/TFFactory/DataLoader.py index 04a5634..86fb714 100644 --- a/TFFactory/DataLoader.py +++ b/TFFactory/DataLoader.py @@ -1,11 +1,12 @@ import numpy as np + class DataLoader(object): def __init__(self): self.FileCache = {} return - def sampleFile(self, filePath, nRows, caching = True): + def sampleFile(self, filePath, nRows, caching=True): data = [] if caching: if filePath in self.FileCache: @@ -27,5 +28,3 @@ def sampleFile(self, filePath, nRows, caching = True): data[r] = row self.FileCache[filePath] = data return data - - diff --git a/TFFactory/Factory.py b/TFFactory/Factory.py index 4146e56..e920e43 100644 --- a/TFFactory/Factory.py +++ b/TFFactory/Factory.py @@ -1,81 +1,25 @@ import tensorflow import numpy as np import json +import TFFactory.GraphBuilder as GraphBuilder +import TFFactory.SupportedFunctions as SupportedFunctions from .TFFactoryException import TFFactoryException -from .DataLoader import DataLoader -import TFFactory.GraphBuilder as GB - - -class Node: - EvalContext = None - def __init__(self, id, backingVariable = None, evalFunc = None, positionalArgs = [], dictParams = {}, needtoFeed = {}): - if backingVariable is None and evalFunc is None: - raise AssertionError('Node cannot be created without a backing function or backing variable') - assert id is not None, 'Nodes need a unique ID' - - self.BackVariable = backingVariable - self.EvalFunc = evalFunc - self.PositionalArgs = positionalArgs - self.DictParams = dictParams - self.NeedtoFeed = needtoFeed - self.LastContext = {} - self.JSONRep = {} - self.ID = id - return - - def eval(self, session = None, feed_dict = {}, newContext = True): - if self.ID in feed_dict: - return feed_dict[self.ID] - elif self.ID in self.NeedtoFeed: - raise TFFactoryException('Node {} was not fed during execution.'.format(self.ID)) - - if newContext: - Node.EvalContext = np.random.random() - if Node.EvalContext not in self.LastContext: - self.LastContext = {} - if Node.EvalContext is not None: - if Node.EvalContext in self.LastContext: - return self.LastContext[Node.EvalContext] - - val = None - if self.EvalFunc is not None: - args = [] - for a in self.PositionalArgs: - if isinstance(a, Node): - args.append(a.eval(session = session, feed_dict = feed_dict, newContext = False)) - else: - args.append(a) - _feed_dict = {} - for name, node in self.NeedtoFeed.items(): - _feed_dict[name] = node.eval(session = session, feed_dict = feed_dict, newContext = False) - val = self.EvalFunc(*args, **self.DictParams, **_feed_dict) - elif self.BackVariable is not None: - if len(self.NeedtoFeed) == 0: - val = self.BackVariable.eval(session = session) - else: - _feed_dict = {} - # Swap out placeholders with node eval function results. - for key, node in self.NeedtoFeed.items(): - _feed_dict[node.BackVariable] = node.eval(session = session, feed_dict = feed_dict, newContext = False) - val = self.BackVariable.eval(feed_dict = _feed_dict, session = session) - if Node.EvalContext is not None: - self.LastContext[Node.EvalContext] = val - if val is None: raise AssertionError('Good luck with this one.') - return val - - def __str__(self): - return json.dumps(self.JSONRep) +from .Pointer import Pointer +from .PythonNode import PythonNode +from .TFNode import TFNode +from .Utilities import findAndApply +from functools import partial def CreateTFGraph(graph): nodes = {} - - for key in graph: + for key, _ in graph.items(): __buildBranch(graph, key, nodes) return nodes -def __buildBranch(graph, key, allNodes, needtofeed = None): + +def __buildBranch(graph, key, allNodes): """ @graph -- The full json graph for traversal/reference @key -- the key to access the desired eval end point in @graph @@ -83,103 +27,120 @@ def __buildBranch(graph, key, allNodes, needtofeed = None): @needtofeed -- the keys of the nodes that need to be supplied to the feed_dict for evaluation @allNodes[key] will hold a reference to the node. """ - if needtofeed is None: - needtofeed = {} - if key in allNodes: # Just be done, but update the feeding dependencies for the caller. - if allNodes[key].EvalFunc is not None: # If it a non-TF op, it has a non-TF feed dict, and needs to be fed. Also it needs to have a backing variable. - needtofeed.update({key : allNodes[key]}) - else: # If it is a TF op, then compile its child dependencies. - needtofeed.update(**allNodes[key].NeedtoFeed) - return - - key = str(key) - graphNode = graph[key] # Stop it with the data type key errors... - type = graphNode['type'] - inputs = graphNode['inputs'] - node = None + if key in allNodes: + return allNodes[key] - tfOp = __resolveTFRef(type) - # Build a function reference to eval when it comes time. - if type in GB.PYTHON_FUNCTIONS: - func = GB.PYTHON_FUNCTIONS[type] - dictParams = {} - args = [] - childNeeds = {} - # Parse the provided kwargs - for name, value in inputs.get('kwargs', {}).items(): - v, isRef = GB.Deserialize(value) - if isRef: - __buildBranch(graph, v, allNodes) - childNeeds[name] = allNodes[v] - else: - dictParams[name] = v - # Parse the provided args - for p in inputs.get('args', []): - v, isRef = GB.Deserialize(p) - if isRef: - __buildBranch(graph, v, allNodes) - childNeeds[name] = allNodes[v] - args.append(allNodes[v]) - else: - args.append(p['value']) - - placeholder = None - if 'Shape' in inputs: - placeholder = tensorflow.placeholder(tensorflow.float32, - shape = inputs['Shape']['value'], - name = 'Placeholder_{}'.format(key)) - # Here the childNeeds is a dict of [param name] : node. - # This is because we don't actually call it recursively here, and still need to - # actually call the function. - node = Node(key, - placeholder, - evalFunc = func, - positionalParams = args, - dictParams = dictParams, - needtoFeed = childNeeds) - if placeholder is not None: - needtofeed[key] = node - allNodes[key] = node - # Apply TF functions to get a reference to a tensor - elif tfOp is not None: - params = {} - args = [] - childNeeds = {} - # Parse kwargs - for name, p in inputs.get('kwargs', {}).items(): - v, isRef = GB.Deserialize(p) - if isRef: - __buildBranch(graph, v, allNodes, childNeeds) - params[name] = allNodes[v].BackVariable - else: - params[name] = v - # Parse args - for p in inputs.get('args', []): - v, isRef = GB.Deserialize(p) - if isRef: - __buildBranch(graph, v, allNodes, childNeeds) - args.append(allNodes[v].BackVariable) - else: - args.append(v) - # We don't need the name of a parameter here, because we have already called the function - # If there is something we need to feed, it has a tensor attached, and we will feed that in - # at runtime. We just need to know which nodes need evaluation/replacing. - node = Node(key, tfOp(*args, **params), needtoFeed = childNeeds) - needtofeed.update(**childNeeds) - if tfOp == tensorflow.placeholder: - needtofeed.update({key : node}) - # It can't eval itself without being fed. So hack that in, I guess? - # Will cause an infinite loop if things break. Wont if they dont! - node.NeedtoFeed.update({key : node}) + _type = graph[key].get('_type') + + node = None + if _type == 'pythonNode': + node = __buildPythonNode(key, graph, allNodes) + elif _type == 'tensorflowNode': + node = __buildTFNode(key, graph, allNodes) if node is None: - raise AssertionError('Unsupported node type: {}'.format(type)) - node.JSONRep = {key : graphNode} + raise AssertionError('Unsupported node type: {}'.format(_type)) allNodes[key] = node - return + return node + + +def __buildPythonNode(key, graph, allNodes): + graphNode = graph[key] + funcName = graphNode['funcName'] + inputs = graphNode['inputs'] + func = __resolveRef(funcName) + needToFeed = {} + + allArgs = { + 'args': inputs.get('args', []), + 'kwargs': inputs.get('kwargs', {}) + } + dependencies = set() + allArgs = findAndApply(allArgs, Pointer.IsInstance, + partial(__markPythonDependency, + dependencies=dependencies, + grpah=graph, + allNodes=allNodes)) + + args = allArgs['args'] + kwargs = allArgs['kwargs'] + for d in dependencies: + needToFeed[d.ID] = d + + placeholder = None + if '_shape' in inputs: + placeholder = tensorflow.placeholder(tensorflow.float32, + shape=inputs['_shape'], + name='Placeholder_{}'.format(key)) + + node = PythonNode(key, + tensor=placeholder, + evalFunc=func, + args=args, + kwargs=kwargs, + needToFeed=needToFeed) -def __resolveTFRef(ref): - if 'tensorflow' not in ref: + allNodes[key] = node + return node + + +def __markPythonDependency(pointer, dependencies, graph, allNodes): + node = __buildBranch(graph, pointer.Ref, allNodes) + dependencies.add(node) + return pointer + + +def __buildTFNode(key, graph, allNodes): + graphNode = graph[key] + funcName = graphNode['funcName'] + inputs = graphNode['inputs'] + tfOp = __resolveRef(funcName) + needToFeed = {} + + allArgs = { + 'args': inputs.get('args', []), + 'kwargs': inputs.get('kwargs', {}) + } + dependencies = set() + allArgs = findAndApply(allArgs, Pointer.IsInstance, + partial(__markTFDependency, + dependencies=dependencies, + graph=graph, + allNodes=allNodes)) + + args = allArgs['args'] + kwargs = allArgs['kwargs'] + for d in dependencies: + if isinstance(d, PythonNode): + # Add any python node dependencies directly to the list. + needToFeed[d.ID] = d + elif isinstance(d, TFNode): + # For all of the TF node dependencies + # Add all of their dependencies to the list. + needToFeed.update(**d.NeedToFeed) + try: + node = TFNode(key, tfOp(*args, **kwargs), needToFeed=needToFeed) + except Exception as e: + raise TFFactoryException( + 'Error occured while compiling node: {}'.format(str(graphNode))) from e + if tfOp == tensorflow.placeholder: + # It can't eval itself without being fed. So hack that in, I guess? + # Will cause an infinite loop if things break. Wont if they dont! + node.NeedToFeed.update({key: node}) + return node + + +def __markTFDependency(pointer, dependencies, graph, allNodes): + node = __buildBranch(graph, pointer.Ref, allNodes) + if node not in dependencies: + dependencies.add(node) + if isinstance(node, TFNode) or isinstance(node, PythonNode): + return node.Tensor + return None + + +def __resolveRef(ref): + if 'tensorflow' not in ref and 'SupportedFunctions' not in ref: return None try: obj = eval(ref) @@ -190,3 +151,26 @@ def __resolveTFRef(ref): return obj + +def __findPointers(obj, pointers): + if isinstance(obj, list): + for i, o in enumerate(obj): + next = {} + pointer = __findPointers(o, next) + if pointer is not None: + pointers[i] = pointer + elif len(next.keys()) > 0: + pointers[i] = next + elif isinstance(obj, dict): + if obj.get('_type') == 'pointer': + value = obj.get('value') + return Pointer(value) + else: + for k, v in obj.items(): + next = {} + pointer = __findPointers(v, next) + if pointer is not None: + pointers[k] = pointer + elif len(next.keys()) > 0: + pointers[k] = next + return None diff --git a/TFFactory/GraphBuilder.py b/TFFactory/GraphBuilder.py index fc24eca..a2ca5a5 100644 --- a/TFFactory/GraphBuilder.py +++ b/TFFactory/GraphBuilder.py @@ -3,19 +3,68 @@ import json from collections import defaultdict, Hashable from types import ModuleType -from TFFactory.SupportedFunctions import * +import TFFactory.SupportedFunctions as functions +from TFFactory.Pointer import Pointer +from .Utilities import findAndApply ID_COUNTER = defaultdict(int) CURRENT_GRAPH = {} -PYTHON_FUNCTIONS = { - 'fileSource' : lambda FilePath, NRows : readFile(FilePath, ',', NRows), - 'parser' : splitFile -} -MOCKED_FUNCTIONS = [ +def NewGraph(): + global ID_COUNTER, CURRENT_GRAPH + ID_COUNTER = defaultdict(int) + CURRENT_GRAPH = {} + return + + +def _assignFunctions(this, functions, type): + for f in functions: + attrs = f.split('.') + curObj = this + for v in attrs[1:-1]: + if not hasattr(curObj, v): + setattr(curObj, v, ModuleType(v)) + curObj = getattr(curObj, v) + setattr(curObj, attrs[-1], _mockFunction(f, type)) + + +def _mockFunction(funcName, type): + def MockedFunction(*args, **kwargs): + global ID_COUNTER, CURRENT_GRAPH + name = kwargs.get('name', 'unnamed') + shape = kwargs.pop('_shape', None) + count = ID_COUNTER[name] + ID_COUNTER[name] += 1 + if count > 0: + name = '{}:{}'.format(name, count) + n = JSONNode(name, funcName, list(args), kwargs, shape, type) + CURRENT_GRAPH.update({n.ID: n}) + return n + return MockedFunction + + +this = sys.modules[__name__] +PYTHON_FUNCTIONS = [ + 'SupportedFunctions.fileSource', + 'SupportedFunctions.parser', + 'SupportedFunctions.testAdd' +] +_assignFunctions(this, PYTHON_FUNCTIONS, 'pythonNode') + +COMPOSITE_FUNCTIONS = [ + 'SupportedFunctions.AdamOptimizer', + 'SupportedFunctions.MomentumOptimizer', + 'SupportedFunctions.GradientDescentOptimizer', + 'SupportedFunctions.SampleDirichlet', + 'SupportedFunctions.GetItem' +] +_assignFunctions(this, COMPOSITE_FUNCTIONS, 'tensorflowNode') + +MOCKED_FUNCTIONS = [ 'tensorflow.placeholder', 'tensorflow.Variable', + 'tensorflow.abs', 'tensorflow.add', 'tensorflow.subtract', 'tensorflow.multiply', @@ -23,7 +72,10 @@ 'tensorflow.log', 'tensorflow.tanh', 'tensorflow.tensordot', + 'tensorflow.square', + 'tensorflow.sqrt', 'tensorflow.reduce_sum', + 'tensorflow.reduce_mean', 'tensorflow.shape', 'tensorflow.transpose', 'tensorflow.expand_dims', @@ -34,143 +86,145 @@ 'tensorflow.layers.batch_normalization', 'tensorflow.layers.dense', 'tensorflow.distributions.Dirichlet', + 'tensorflow.summary.tensor_summary', + 'tensorflow.summary.scalar', + 'tensorflow.summary.histogram', + 'tensorflow.summary.image', + 'tensorflow.summary.text', + 'tensorflow.summary.audio', + 'tensorflow.summary.merge', + 'tensorflow.summary.merge_all' ] - -SERIALIZE_MAP = { - tensorflow.float16 : 'tensorflow.float16', - tensorflow.float32 : 'tensorflow.float32', - tensorflow.float64 : 'tensorflow.float64', - tensorflow.int8 : 'tensorflow.int8', - tensorflow.int16 : 'tensorflow.int16', - tensorflow.int32 : 'tensorflow.int32', - tensorflow.int64 : 'tensorflow.int64' -} - -DESERIALIZE_MAP = { - 'tensorflow.float16': tensorflow.float16, - 'tensorflow.float32': tensorflow.float32, - 'tensorflow.float64': tensorflow.float64, - 'tensorflow.int8': tensorflow.int8, - 'tensorflow.int16': tensorflow.int16, - 'tensorflow.int32': tensorflow.int32, - 'tensorflow.int64': tensorflow.int64 -} - - -def Serialize(value): - type = 'unknown' - if isinstance(value, JSONNode): - v = value.ID - type = 'ref' - elif isinstance(value, Hashable): - v = SERIALIZE_MAP.get(value, value) - else: - v = value - - d = { - 'value' : v, - 'type' : type +_assignFunctions(this, MOCKED_FUNCTIONS, 'tensorflowNode') + + +class Encoder(json.JSONEncoder): + SERIALIZE_MAP = { + tensorflow.float16: 'tensorflow.float16', + tensorflow.float32: 'tensorflow.float32', + tensorflow.float64: 'tensorflow.float64', + tensorflow.int8: 'tensorflow.int8', + tensorflow.int16: 'tensorflow.int16', + tensorflow.int32: 'tensorflow.int32', + tensorflow.int64: 'tensorflow.int64' } - - return d - -def Deserialize(value): - """ - Given the value object with format : - value = { - 'value' : <>, - 'type' : <> - } - returns (the deserialized value, whether or not it is a node reference) - """ - v = value['value'] - if value['type'] == 'ref': - return (v, True) - if isinstance(v, Hashable): - v = DESERIALIZE_MAP.get(v, v) + def default(self, obj): + if isinstance(obj, JSONNode): + return { + '_type': obj.Type, + 'funcName': obj.FuncName, + 'inputs': obj.Inputs + } + elif isinstance(obj, Pointer): + return { + 'value': obj.Ref, + '_type': 'pointer' + } + elif isinstance(obj, slice): + return { + 'value': [obj.start, obj.stop, obj.step], + '_type': 'slice' + } + elif isinstance(obj, Hashable) and obj in Encoder.SERIALIZE_MAP: + return { + 'value': Encoder.SERIALIZE_MAP[obj], + '_type': 'tensorflow' + } - return (v, False) + return json.JSONEncoder.default(self, obj) -def NewGraph(): - global ID_COUNTER, CURRENT_GRAPH - ID_COUNTER = defaultdict(int) - CURRENT_GRAPH = {} - return -def MockFunction(funcName): - def MockedFunction(*args, **kwargs): - global ID_COUNTER, CURRENT_GRAPH - name = kwargs.get('name','Variable') - count = ID_COUNTER[name] - ID_COUNTER[name] += 1 - if count > 0: - name = '{}_{}'.format(name, count) - n = JSONNode(name, funcName, args, kwargs) - CURRENT_GRAPH.update(n.asDict()) - return n - return MockedFunction +class Decoder(json.JSONDecoder): + DESERIALIZE_MAP = { + 'tensorflow.float16': tensorflow.float16, + 'tensorflow.float32': tensorflow.float32, + 'tensorflow.float64': tensorflow.float64, + 'tensorflow.int8': tensorflow.int8, + 'tensorflow.int16': tensorflow.int16, + 'tensorflow.int32': tensorflow.int32, + 'tensorflow.int64': tensorflow.int64 + } -this = sys.modules[__name__] -for f in MOCKED_FUNCTIONS: - attrs = f.split('.') - curObj = this - for v in attrs[1:-1]: - if not hasattr(curObj, v): - setattr(curObj, v, ModuleType(v)) - curObj = getattr(curObj, v) - setattr(curObj, attrs[-1], MockFunction(f)) + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__( + self, object_hook=self.object_hook, *args, **kwargs) + return + + def object_hook(self, obj): + if '_type' in obj: + t = obj['_type'] + if t == 'tensorflow': + return Decoder.DESERIALIZE_MAP[obj['value']] + elif t == 'pointer': + return Pointer(obj['value']) + elif t == 'slice': + return slice(*obj['value']) + return obj class JSONNode: - def __init__(self, id, type, args, kwargs): + def __init__(self, id, funcName, args, kwargs, shape, type): self.ID = str(id) - self.type = str(type) + self.FuncName = funcName + self.Type = str(type) self.Inputs = { - 'args' : [], - 'kwargs' : {} + 'args': args, + 'kwargs': kwargs, + '_shape': shape } - for name, value in kwargs.items(): - self.Inputs['kwargs'][name] = Serialize(value) - self.Inputs['args'] = [] - for value in args: - self.Inputs['args'].append(Serialize(value)) + self.Inputs = findAndApply( + self.Inputs, self._shouldBePointer, self._replaceWithPointer) + return - - def asDict(self): - d = { - self.ID : { - 'type' : self.type, - 'inputs' : self.Inputs - } - } - return d - + + @staticmethod + def _shouldBePointer(obj): + return isinstance(obj, JSONNode) + + @staticmethod + def _replaceWithPointer(obj): + return Pointer(obj.ID) + + def __getitem__(self, key): + return GetItem(self, key) + + def __neg__(self): + return multiply(-1.0, self) + + def __pos__(self): + return abs(self) + def __add__(self, other): return add(self, other) + def __radd__(self, other): return add(other, self) + __iadd__ = __add__ def __sub__(self, other): return subtract(self, other) + def __rsub__(self, other): return subtract(other, self) + __isub__ = __sub__ def __mul__(self, other): return multiply(self, other) + def __rmul__(self, other): return multiply(other, self) + __imul__ = __mul__ - def __div__(self, other): + def __truediv__(self, other): return divide(self, other) - def __rdiv__(self, other): - return divide(other, self) - def __str__(self): - return json.dumps(self.asDict()) + def __rtruediv__(self, other): + return divide(other, self) + __itruediv__ = __truediv__ def __eq__(self, other): return other.ID == self.ID def __hash__(self): - return hash(self.ID) \ No newline at end of file + return hash(self.ID) diff --git a/TFFactory/Node.py b/TFFactory/Node.py new file mode 100644 index 0000000..8c3e313 --- /dev/null +++ b/TFFactory/Node.py @@ -0,0 +1,37 @@ +import numpy as np + + +class Node: + EvalContext = None + + def __init__(self, id): + self.ID = id + self.LastContext = {} + return + + def Eval(self, session=None, feed_dict={}, newContext=True): + if self.ID in feed_dict: + return feed_dict[self.ID] + + if newContext: + Node.EvalContext = np.random.random() + if Node.EvalContext not in self.LastContext: + self.LastContext = {} + if Node.EvalContext is not None: + if Node.EvalContext in self.LastContext: + return self.LastContext[Node.EvalContext] + + val = self._eval(session=session, feed_dict=feed_dict, + newContext=newContext) + if Node.EvalContext is not None: + self.LastContext[Node.EvalContext] = val + return val + + def _eval(self, session=None, feed_dict={}, newContext=True): + raise NotImplementedError + + def __hash__(self): + return hash(self.ID) + + def __eq__(self, other): + return other.ID == self.ID diff --git a/TFFactory/Pointer.py b/TFFactory/Pointer.py new file mode 100644 index 0000000..ad53a27 --- /dev/null +++ b/TFFactory/Pointer.py @@ -0,0 +1,7 @@ +class Pointer(): + def __init__(self, ref): + self.Ref = ref + + @classmethod + def IsInstance(cls, obj): + return isinstance(obj, cls) \ No newline at end of file diff --git a/TFFactory/PythonNode.py b/TFFactory/PythonNode.py new file mode 100644 index 0000000..495ade0 --- /dev/null +++ b/TFFactory/PythonNode.py @@ -0,0 +1,61 @@ +from .Node import Node +from .Utilities import findAndApply +from functools import partial +from .GraphBuilder import Encoder, Decoder +from .Pointer import Pointer + + +class PythonNode(Node): + Encoder = Encoder() + Decoder = Decoder() + + def __init__(self, id, evalFunc, tensor=None, + args=None, kwargs=None, pointerMap=None, needToFeed=None): + + self.EvalFunc = evalFunc + self.Tensor = tensor + self.Args = args if args is not None else [] + self.Kwargs = kwargs if kwargs is not None else {} + self.AllArgs = { + 'args': self.Args, + 'kwargs': self.Kwargs + } + self.NeedToFeed = needToFeed if needToFeed is not None else {} + self.PointerMap = pointerMap if pointerMap is not None else {} + super().__init__(id) + + def _eval(self, session=None, feed_dict={}, newContext=True): + pointers = PythonNode.Decoder.decode( + PythonNode.Encoder.encode(self.PointerMap)) + + pointers = findAndApply(pointers, Pointer.IsInstance, + partial(self._feed, + nodeMap=self.NeedToFeed, + session=session, + feed_dict=feed_dict, + newContext=newContext)) + self._mergeObj(self.AllArgs, pointers) + val = self.EvalFunc(*self.AllArgs['args'], **self.AllArgs['kwargs']) + return val + + @staticmethod + def _feed(nodeMap, session, feed_dict, newContext, pointer): + return nodeMap[pointer.Ref].Eval(session, feed_dict, newContext) + + @staticmethod + def _mergeObj(o1, o2): + if o2 is None: + return o1 + if o1 is None: + return o2 + if isinstance(o2, dict): + for k, v in o2.items(): + o1[k] = PythonNode._mergeObj(o1[k], v) + return o1 + elif isinstance(o2, list): + for i, v in enumerate(o2): + o1[i] = PythonNode._mergeObj(o1[i], v) + return o1 + else: + return o2 + return None diff --git a/TFFactory/SupportedFunctions.py b/TFFactory/SupportedFunctions.py index a5b28c2..013d743 100644 --- a/TFFactory/SupportedFunctions.py +++ b/TFFactory/SupportedFunctions.py @@ -1,11 +1,52 @@ from .DataLoader import DataLoader +import numpy as np DL = DataLoader() + def readFile(fp, delim, nRows): - return DL.sampleFile(fp, nRows, delim, caching = True) + global DL + return DL.sampleFile(fp, nRows, caching=True) + def splitFile(Source, SegmentDelimeter, DataDelimeter, SegmentIndex, Shape): data = [] for row in Source: - data.append(list(map(float, row.split(SegmentDelimeter)[SegmentIndex].split(DataDelimeter)))) + data.append(list(map(float, row.split(SegmentDelimeter) + [SegmentIndex].split(DataDelimeter)))) return np.array(data).reshape((*Shape)) + + +def testAdd(a, b): + return [a + b] + + +# TF ones +import tensorflow + + +def AdamOptimizer(loss, learningRate): + optimizer = tensorflow.train.AdamOptimizer(learningRate) + return optimizer.minimize(loss) + + +def MomentumOptimizer(loss, learningRate, momentum): + optimizer = tensorflow.train.MomentumOptimizer( + learningRate, momentum=momentum) + return optimizer.minimize(loss) + + +def GradientDescentOptimizer(loss, learningRate): + optimizer = tensorflow.train.GradientDescentOptimizer(learningRate) + return optimizer.minimize(loss) + + +def SampleDirichlet(concentration, sampleShape, validateArgs=False, allowNanStats=True, **kwargs): + dist = tensorflow.distributions.Dirichlet(concentration, + validate_args=validateArgs, + allow_nan_stats=allowNanStats) + + return dist.sample(sampleShape, **kwargs) + + +def GetItem(tensor, key): + return tensor[key] diff --git a/TFFactory/TFNode.py b/TFFactory/TFNode.py new file mode 100644 index 0000000..6dbd0c8 --- /dev/null +++ b/TFFactory/TFNode.py @@ -0,0 +1,26 @@ +from .Node import Node +from .TFFactoryException import TFFactoryException + + +class TFNode(Node): + def __init__(self, id, tensor, needToFeed=None): + + self.Tensor = tensor + self.NeedToFeed = needToFeed if needToFeed is not None else {} + super().__init__(id) + + def _eval(self, session=None, feed_dict={}, newContext=True): + if self.ID in self.NeedToFeed: + raise TFFactoryException( + 'Node {} was not fed during evaluation.'.format(self.ID)) + + if len(self.NeedToFeed) == 0: + val = session.run(self.Tensor) + else: + tf_feed_dict = {} + # Swap out placeholders with node eval function results. + for key, node in self.NeedToFeed.items(): + tf_feed_dict[node.Tensor] = node.Eval( + session, feed_dict, False) + val = session.run(self.Tensor, feed_dict=tf_feed_dict) + return val diff --git a/TFFactory/Utilities.py b/TFFactory/Utilities.py new file mode 100644 index 0000000..3edf2c6 --- /dev/null +++ b/TFFactory/Utilities.py @@ -0,0 +1,12 @@ +# These just don't belong elsewhere + +def findAndApply(obj, found, apply): + if found(obj): + return apply(obj) + if isinstance(obj, list): + for i, o in enumerate(obj): + obj[i] = findAndApply(o, found, apply) + elif isinstance(obj, dict): + for k, v in obj.items(): + obj[k] = findAndApply(v, found, apply) + return obj \ No newline at end of file diff --git a/TFFactory/file.txt b/TFFactory/file.txt deleted file mode 100644 index 6138637..0000000 --- a/TFFactory/file.txt +++ /dev/null @@ -1,2 +0,0 @@ -1,2,3,4,5;2,3,4,5,6 -3,4,5,6,7;7,6,3,2,1 \ No newline at end of file diff --git a/setup.py b/setup.py index 2a15234..66eb158 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="TFFactory", - version="0.0.1", + version="0.1.5002", author="Kevin Welsh", author_email="kevinwelsh132@gmail.com", description="A mock tensorflow package that constructs JSON objects instead of tensors.", diff --git a/test.py b/test.py index bd6554f..386d426 100644 --- a/test.py +++ b/test.py @@ -4,19 +4,24 @@ import json if __name__ == '__main__': - placeHolder = tff.placeholder(tf.int32, shape = [3], name = 'input') - n = tff.Variable([-1, -2, -3], name = 'n1') - b = tff.Variable(initial_value = [4, 5, 6], name = 'b') - n = n + b + placeHolder - graph = json.dumps(tff.CURRENT_GRAPH) + pythonNode = tff.testAdd(1, -100, _shape=[1]) + + placeHolder = tff.placeholder(tf.float32, shape=[3], name='input') + n = tff.Variable([-1, -2, -3], name='n1', dtype=tf.float32) + sliced = tff.SampleDirichlet([0.2, 0.8], [1, 5], name='dirichlet')[0, :] + b = tff.Variable(initial_value=[4, 5, 6], name='b', dtype=tf.float32) + n = n + b + placeHolder + pythonNode + n = tff.AdamOptimizer(n, 1.0) + + graph = json.dumps(tff.CURRENT_GRAPH, cls=tff.Encoder, indent=2) print(graph) - graph = json.loads(graph) + graph = json.loads(graph, cls=tff.Decoder) + print('Compiling graph!') compiledGraph = factory.CreateTFGraph(graph) with tf.Session() as sess: - tf.global_variables_initializer().run(session = sess) - for k,v in compiledGraph.items(): - print('{} = {}'.format(k, v.eval(feed_dict = {'input' : [1,2,3]}))) - - + tf.global_variables_initializer().run(session=sess) + for k, v in compiledGraph.items(): + print('{} = {}'.format( + k, v.Eval(session=sess, feed_dict={'input': [1, 2, 3]})))