Merge pull request #453 from tensorflow/shrestha/fix_pip

Shrestha/fix pip

README.md

@@ -88,7 +88,7 @@ Once TensorFlow's dependencies are installed, clone the `ngraph-bridge` repo:
 
         git clone https://github.com/tensorflow/ngraph-bridge.git
         cd ngraph-bridge
-        git checkout v0.22.0-rc3
+        git checkout v0.22.0-rc4
 
 Run the following Python script to build TensorFlow, nGraph, and the bridge. Use Python 3.5:
 

ngraph_bridge/enable_variable_ops/ngraph_capture_variables.cc

@@ -114,6 +114,13 @@ Status CaptureVariables(Graph* graph, std::set<string> skip_these_nodes) {
   // If Prefetch is requested
   if (std::getenv(NGraphPrefetchSharedResouce::NGRAPH_TF_USE_PREFETCH) !=
       nullptr) {
+    if (make_iterator_nodes.size() == 0) {
+      // No MakeIterator Op found in the Graph
+      make_iterator_nodes.clear();
+      nodes_to_capture.clear();
+      return Status::OK();
+    }
+
     if (make_iterator_nodes.size() > 1) {
       return errors::Internal(
           "Found more than 1 MakeIterator nodes. This case is not supported.");

ngraph_bridge/ngraph_encapsulate_clusters.h

@@ -26,6 +26,10 @@
 #include <iostream>
 #include "tensorflow/core/graph/graph.h"
 
+#include "ngraph/ngraph.hpp"
+
+#include "ngraph_bridge/ngraph_partial_shapes.h"
+
 namespace tensorflow {
 
 namespace ngraph_bridge {
@@ -35,10 +39,110 @@ typedef std::map<std::string, std::vector<int>> ShapeHintMap;
 // the integer represent AOT level requested.
 typedef std::pair<bool, std::set<ShapeHintMap>> AOTInfo;
 
+// TODO: an optimization would be to separate the analysis and rewriting passes
+// cleanly, so that analysis pass is run in mark_for_clustering, and its
+// information is reused here instead of recalculating
+// To do that an Encapsulator object with AnalysisPass run can be created in
+// MarkForClustering, and that can be passed to EncapsulateClusters
+
+/// Takes a TF graph where ngraph_cluster attributes has been marked in a
+/// preceeding pass (assign_clusters), then replaces TF subgraphs and inserts
+/// encapsulate ops in their place. Optionally can perform ahead of time
+/// compilation.
 Status EncapsulateClusters(
     Graph* graph, int graph_id, FunctionDefLibrary* fdeflib,
-    std::unordered_map<std::string, std::string> device_config,
-    AOTInfo aot_info);
+    const std::unordered_map<std::string, std::string>& device_config,
+    const AOTInfo& aot_info);
+
+// TODO Encapsulator is dependent on ClusterManager. They could be made
+// independent.
+
+// A class to perform analysis (identify subgraphs)
+// and rewriting (create encapsulates and splice them in)
+// Order of calling: construction -> AnalysisPass -> RewritePass
+//                                       |
+//                                       v
+//                                 NewClusterIds
+// Any other order of calling will generate errors
+// Cannot be copied/moved or reset
+class Encapsulator {
+ public:
+  Encapsulator(Graph* g);
+  // Populate ClusterManager with the subgraphs for each potential encapsulate
+  Status AnalysisPass();
+  // Perform the actual graph surgery
+  Status RewritePass(
+      FunctionDefLibrary* fdeflib, int graph_id,
+      const std::unordered_map<std::string, std::string>& device_config);
+  // Returns the newly created cluster ids after AnalysisPass is done
+  // Needed because ClusterManager (CM) might have contained old stuff,
+  // so it might not be possible to query the CM itself to get this
+  Status GetNewClusterIDs(std::set<int>& result);
+
+  Encapsulator(const Encapsulator&) = delete;
+  Encapsulator(Encapsulator&&) = delete;
+  Encapsulator& operator=(const Encapsulator&) = delete;
+  Encapsulator& operator=(Encapsulator&&) = delete;
+
+ private:
+  Graph* graph;
+  // boolean to indicate if analysis has been done
+  // If not rewritepass should not be called
+  bool analysis_done;
+  // boolean to indicate that rewrite is done;
+  bool rewrite_done;
+  // A map from cluster indices to the expected device name for nodes
+  // in that cluster.
+  std::map<int, std::string> device_name_map;
+
+  // We *should* eventually have a way of monitoring the device and the backend
+  // together
+  std::map<int, std::string> backend_name_map;
+
+  // As we build the graph we will be tracking the.. TODO(amprocte): finish
+  // this comment.
+  std::map<std::tuple<int, int>, std::tuple<int, int>> output_remap_map;
+  std::map<std::tuple<int, int, int>, int> input_remap_map;
+  std::map<std::tuple<int, std::string, int>, string> input_rename_map;
+
+  // A map from cluster indices to a vector of input data types.
+  std::map<int, std::vector<std::tuple<int, int, DataType>>> cluster_input_map;
+  // A map from cluster indices to a vector of output data types.
+  std::map<int, std::vector<DataType>> cluster_output_dt_map;
+
+  // A map from cluster indices to corresponding NGraphEncapsulate nodes.
+  std::map<int, Node*> cluster_node_map;
+
+  std::set<int> cluster_indices_for_this_graph;
+
+  static void AddInput(NodeDef* dst, StringPiece src_name, int src_slot);
+};
+
+// Translates TF subgraph to ng function then compiles it
+Status PerformAOTOnEncapsulates(Graph* graph, const AOTInfo& aot_info);
+
+std::string HintAsString(ShapeHintMap single_hint);
+
+// Given a node, partialshape info from TF (present in the .pb itself) and a
+// shape hint, combine all that information
+PartialShape CombineNodeInfoAndHint(Node* node,
+                                    PartialShape partial_shape_from_node,
+                                    const ShapeHintMap& single_hint);
+
+// Given a TF graph, it scans it for inputs and finds what TF is saying about
+// their shapes (in the .pb itself)
+// Creates a map between input node names and PartialShape information we get
+// from the TF graph
+std::map<std::string, PartialShape> GetShapesFromTFInputnodes(
+    Graph* graph, const string& input_node_type);
+
+// Given an encapsulate node, and the input shapes,
+// performs TranslateGraph and returns an ng function and a signature
+Status PerformTranslation(Node* node,
+                          const std::map<std::string, std::vector<int>>&
+                              inputs_node_shapes_for_compilation,
+                          std::string& signature,
+                          std::shared_ptr<ngraph::Function>& ng_function);
 
 }  // namespace ngraph_bridge
 }  // namespace tensorflow

ngraph_bridge/version.cc

@@ -32,7 +32,7 @@
 // candidate such as v0.7.0-rc0
 // The code in master will always have the last released version number
 // with a suffix of '-master'
-#define NG_TF_VERSION_SUFFIX "-rc3"
+#define NG_TF_VERSION_SUFFIX "-rc4"
 
 #define VERSION_STR_HELPER(x) #x
 #define VERSION_STR(x) VERSION_STR_HELPER(x)

python/setup.in.py

@@ -59,7 +59,7 @@ def get_tag(self):
 
 setup( 
     name='ngraph_tensorflow_bridge',
-    version='0.22.0rc3',
+    version='0.22.0rc4',
     description='Intel nGraph compiler and runtime for TensorFlow',
     long_description=long_description,
     long_description_content_type="text/markdown",

test/ci/buildkite/ngtf-cpu_centos-grappler.yaml

@@ -56,6 +56,7 @@
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
         /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl
+      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_centos.yaml

@@ -55,6 +55,7 @@
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
         /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl
+      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_macos.yaml

@@ -39,7 +39,7 @@
       source /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install -U \
         /usr/local/var/buildkite-agent/tf_1.14.0_install/artifacts/tensorflow/tensorflow-1.14.0-cp37-cp37m-macosx_10_7_x86_64.whl
-      pip install psutil && \
+      pip install psutil pip==19.3.1 && \
         pip install -U /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_model-test.yaml

@@ -26,6 +26,7 @@
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
         /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl
+      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_ubuntu-bin-build.yaml

@@ -38,6 +38,7 @@
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil 
+      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \
         --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID --test_resnet

test/ci/buildkite/ngtf-cpu_ubuntu-external-fork.yaml

@@ -68,6 +68,7 @@
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
         /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl
+      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_ubuntu-variables.yaml

@@ -54,6 +54,7 @@
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
         /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl
+      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_ubuntu.yaml

@@ -68,6 +68,7 @@
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
         /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl
+      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_ubuntu_18_04.yaml

@@ -63,6 +63,7 @@
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
         /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl
+      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       
     label: ":gear: Install"

test/ci/buildkite/ngtf-gpu_ubuntu.yaml

@@ -42,6 +42,7 @@
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
         /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0-cp??-cp??m-linux_x86_64.whl
+      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
       
     label: ":gear: Install"