Merge pull request #1434 from finos/1185-optimise-walker-for-profile-…

…with-small-decision-set

refactor(#1185): optimize walker for small datasets

generator/src/main/java/com/scottlogic/deg/generator/guice/DecisionTreeWalkerProvider.java

@@ -18,10 +18,9 @@
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
-import com.scottlogic.deg.common.ValidationException;
 import com.scottlogic.deg.generator.config.detail.DataGenerationType;
 import com.scottlogic.deg.generator.generation.GenerationConfigSource;
-import com.scottlogic.deg.generator.walker.*;
+import com.scottlogic.deg.generator.walker.DecisionTreeWalker;
 import com.scottlogic.deg.generator.walker.rowspec.RandomRowSpecDecisionTreeWalker;
 import com.scottlogic.deg.generator.walker.rowspec.RowSpecDecisionTreeWalker;
 

generator/src/main/java/com/scottlogic/deg/generator/guice/GeneratorModule.java

@@ -68,5 +68,8 @@ protected void configure() {
 
         bind(JavaUtilRandomNumberGenerator.class)
             .toInstance(new JavaUtilRandomNumberGenerator(OffsetDateTime.now().getNano()));
+        bind(int.class)
+            .annotatedWith(Names.named("config:internalRandomRowSpecStorage"))
+            .toInstance(256);
     }
 }

generator/src/main/java/com/scottlogic/deg/generator/walker/decisionbased/RandomOptionPicker.java

@@ -1,6 +1,5 @@
 package com.scottlogic.deg.generator.walker.decisionbased;
 
-import com.google.inject.Inject;
 import com.scottlogic.deg.generator.decisiontree.ConstraintNode;
 import com.scottlogic.deg.generator.decisiontree.DecisionNode;
 

generator/src/main/java/com/scottlogic/deg/generator/walker/rowspec/PotentialRowSpecCount.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2019 Scott Logic Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.scottlogic.deg.generator.walker.rowspec;
+
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+import com.scottlogic.deg.generator.decisiontree.ConstraintNode;
+import com.scottlogic.deg.generator.decisiontree.DecisionNode;
+import com.scottlogic.deg.generator.decisiontree.DecisionTree;
+
+public class PotentialRowSpecCount {
+    private final int max;
+
+    @Inject
+    public PotentialRowSpecCount(@Named("config:internalRandomRowSpecStorage") int max) {
+        this.max = max;
+    }
+
+    /**
+     *  recursively traverses the tree counting the maximum potential number of
+     *  decisions that could result. Breaks early if count goes over given max value
+     * @param decisionTree tree to count
+     * @return whether the tree has less than the max number of decisions
+     */
+    boolean lessThanMax(DecisionTree decisionTree){
+        Integer total = countConstraintNode(decisionTree.rootNode);
+        return total != null;
+    }
+
+    private Integer countConstraintNode(ConstraintNode constraintNode){
+        int total = 1;
+        for (DecisionNode decision : constraintNode.getDecisions()) {
+            Integer count = countDecisionNode(decision);
+            if (count == null) return null;
+
+            total *= count;
+            if (total > max) return null;
+        }
+        return total;
+    }
+
+    private Integer countDecisionNode(DecisionNode decision) {
+        int total = 0;
+        for (ConstraintNode option : decision.getOptions()) {
+            Integer count = countConstraintNode(option);
+            if (count == null) return null;
+
+            total += count;
+            if (total > max) return null;
+        }
+        return total;
+    }
+}

generator/src/main/java/com/scottlogic/deg/generator/walker/rowspec/RandomRowSpecDecisionTreeWalker.java

@@ -1,34 +1,66 @@
+/*
+ * Copyright 2019 Scott Logic Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.scottlogic.deg.generator.walker.rowspec;
 
 import com.google.inject.Inject;
 import com.scottlogic.deg.generator.decisiontree.DecisionTree;
 import com.scottlogic.deg.generator.fieldspecs.RowSpec;
 import com.scottlogic.deg.generator.generation.databags.DataBag;
 import com.scottlogic.deg.generator.generation.databags.RowSpecDataBagGenerator;
+import com.scottlogic.deg.generator.utils.JavaUtilRandomNumberGenerator;
 import com.scottlogic.deg.generator.walker.DecisionTreeWalker;
 import com.scottlogic.deg.generator.walker.decisionbased.RowSpecTreeSolver;
 
+import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public class RandomRowSpecDecisionTreeWalker implements DecisionTreeWalker {
     private final RowSpecTreeSolver rowSpecTreeSolver;
     private final RowSpecDataBagGenerator rowSpecDataBagGenerator;
+    private PotentialRowSpecCount potentialRowSpecCount;
+    private final JavaUtilRandomNumberGenerator random;
 
     @Inject
-    public RandomRowSpecDecisionTreeWalker(RowSpecTreeSolver rowSpecTreeSolver, RowSpecDataBagGenerator rowSpecDataBagGenerator) {
+    public RandomRowSpecDecisionTreeWalker(RowSpecTreeSolver rowSpecTreeSolver,
+                                           RowSpecDataBagGenerator rowSpecDataBagGenerator,
+                                           PotentialRowSpecCount potentialRowSpecCount,
+                                           JavaUtilRandomNumberGenerator random) {
         this.rowSpecTreeSolver = rowSpecTreeSolver;
         this.rowSpecDataBagGenerator = rowSpecDataBagGenerator;
+        this.potentialRowSpecCount = potentialRowSpecCount;
+        this.random = random;
     }
 
     @Override
     public Stream<DataBag> walk(DecisionTree tree) {
         if (tree.rootNode.getDecisions().isEmpty()) {
             return generateWithoutRestarting(tree);
         }
+        boolean useCache = potentialRowSpecCount.lessThanMax(tree);
+        Stream<RowSpec> rowSpecStream = useCache  ? getFromCachedRowSpecs(tree): getRowSpecAndRestart(tree);
+
+        return rowSpecStream.map(this::createDataBag);
+    }
 
-        return getRowSpecAndRestart(tree)
-            .map(this::createDataBag);
+    private Stream<RowSpec> getFromCachedRowSpecs(DecisionTree tree) {
+        List<RowSpec> rowSpecCache = rowSpecTreeSolver.createRowSpecs(tree).collect(Collectors.toList());
+        return Stream.generate(() -> getRandomRowSpec(rowSpecCache));
     }
 
     private Stream<DataBag> generateWithoutRestarting(DecisionTree tree) {
@@ -50,6 +82,10 @@ private Optional<RowSpec> getFirstRowSpec(DecisionTree tree) {
         return rowSpecTreeSolver.createRowSpecs(tree).findFirst();
     }
 
+    private RowSpec getRandomRowSpec(List<RowSpec> rowSpecCache) {
+        return rowSpecCache.get(random.nextInt(rowSpecCache.size()));
+    }
+
     private DataBag createDataBag(RowSpec rowSpec) {
         return rowSpecDataBagGenerator.createDataBags(rowSpec).findFirst().get();
     }