diff --git a/java-vtl-model/src/main/java/no/ssb/vtl/model/AbstractComponent.java b/java-vtl-model/src/main/java/no/ssb/vtl/model/AbstractComponent.java deleted file mode 100644 index 3ab78270..00000000 --- a/java-vtl-model/src/main/java/no/ssb/vtl/model/AbstractComponent.java +++ /dev/null @@ -1,36 +0,0 @@ -package no.ssb.vtl.model; - -/** - * Abstract component implementation. - */ -@Deprecated -public abstract class AbstractComponent { //implements Component { - - /*@Override - public String toString() { - return MoreObjects.toStringHelper(role()) - .add("type", type().getSimpleName()) - //.add("name", name()) - //.addValue(Optional.fromNullable(get()).transform(Object::toString).or("NULL")) - .toString(); - } - - @Override - public int hashCode() { - return Objects.hash( - name(), - type(), - get() - ); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null) return false; // not class equality || getClass() != o.getClass()) return false; - Component that = (Component) o; - return Objects.equals(that.name(), name()) - && Objects.equals(that.type(), type()) - && Objects.equals(that.get(), get()); - }*/ -} diff --git a/java-vtl-model/src/main/java/no/ssb/vtl/model/Component.java b/java-vtl-model/src/main/java/no/ssb/vtl/model/Component.java index 9c430e10..ed77ff6e 100644 --- a/java-vtl-model/src/main/java/no/ssb/vtl/model/Component.java +++ b/java-vtl-model/src/main/java/no/ssb/vtl/model/Component.java @@ -22,8 +22,6 @@ import com.google.common.base.MoreObjects; -import java.util.Objects; - import static com.google.common.base.Preconditions.checkNotNull; /** @@ -68,18 +66,20 @@ public boolean isAttribute() { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Component component = (Component) o; - return Objects.equals(getType(), component.getType()) && - Objects.equals(getName(), component.getName()) && - getRole() == component.getRole(); + return this == o; +// if (this == o) return true; +// if (o == null || getClass() != o.getClass()) return false; +// Component component = (Component) o; +// return Objects.equals(getType(), component.getType()) && +// //Objects.equals(getName(), component.getName()) && +// getRole() == component.getRole(); } @Override public int hashCode() { - return Objects.hash(getType(), getName(), getRole()); + return System.identityHashCode(this); + ///return Objects.hash(getType(), getName(), getRole()); } diff --git a/java-vtl-model/src/main/java/no/ssb/vtl/model/DataStructure.java b/java-vtl-model/src/main/java/no/ssb/vtl/model/DataStructure.java index b9e9a3dd..94426bb4 100644 --- a/java-vtl-model/src/main/java/no/ssb/vtl/model/DataStructure.java +++ b/java-vtl-model/src/main/java/no/ssb/vtl/model/DataStructure.java @@ -20,15 +20,17 @@ * #L% */ -import com.google.common.collect.BiMap; +import com.google.common.annotations.Beta; import com.google.common.collect.ForwardingMap; -import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.Comparator; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.function.BiFunction; -import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; @@ -41,30 +43,49 @@ */ public class DataStructure extends ForwardingMap { - private final BiMap delegate; private final BiFunction, ?> converter; - protected DataStructure(BiFunction, ?> converter) { - delegate = HashBiMap.create(); + private final ImmutableMap delegate; + + private final IdentityHashMap inverseCache; + private final ImmutableMap roleCache; + private final ImmutableMap> typeCache; + + protected DataStructure(BiFunction, ?> converter, ImmutableMap map) { this.converter = checkNotNull(converter); + this.delegate = checkNotNull(map); + this.inverseCache = computeInverseCache(this.delegate); + this.roleCache = computeRoleCache(delegate); + this.typeCache = computeTypeCache(delegate); } - public static DataStructure copyOf( - BiFunction, ?> converter, - Map newComponents + private static ImmutableMap computeRoleCache(ImmutableMap delegate) { + return ImmutableMap.copyOf(Maps.transformValues(delegate, Component::getRole)); + } + + private static ImmutableMap> computeTypeCache(ImmutableMap delegate) { + return ImmutableMap.copyOf(Maps.transformValues(delegate, Component::getType)); + } + + public static DataStructure.Builder builder() { + return new DataStructure.Builder(); + } + + public static DataStructure.Builder copyOf( + Map dataStructure ) { - DataStructure instance = new DataStructure(converter); - instance.putAll(newComponents); - return instance; + Builder builder = new Builder(); + builder.putAll(dataStructure); + return builder; } public static DataStructure of(BiFunction, ?> converter, Map> types, Map roles) { checkArgument(types.keySet().equals(roles.keySet())); - DataStructure instance = new DataStructure(converter); + Builder builder = builder(); for (String name : types.keySet()) { - instance.put(name, new Component(types.get(name), roles.get(name), name)); + builder.put(name, roles.get(name), types.get(name)); } - return instance; + return builder.build(); } /** @@ -72,9 +93,9 @@ public static DataStructure of(BiFunction, ?> converter, Map, ?> converter, String name1, Component.Role role1, Class type1) { - DataStructure instance = new DataStructure(converter); - instance.put(name1, new Component(type1, role1, name1)); - return instance; + Builder builder = builder(); + builder.put(name1, role1, type1); + return builder.build(); } /** @@ -83,10 +104,10 @@ public static DataStructure of(BiFunction, ?> converter, public static DataStructure of(BiFunction, ?> converter, String name1, Component.Role role1, Class type1, String name2, Component.Role role2, Class type2) { - DataStructure instance = new DataStructure(converter); - instance.put(name1, new Component(type1, role1, name1)); - instance.put(name2, new Component(type2, role2, name2)); - return instance; + return builder() + .put(name1, role1, type1).put(name2, role2, type2) + .build(); + } /** @@ -96,11 +117,9 @@ public static DataStructure of(BiFunction, ?> converter, String name1, Component.Role role1, Class type1, String name2, Component.Role role2, Class type2, String name3, Component.Role role3, Class type3) { - DataStructure instance = new DataStructure(converter); - instance.put(name1, new Component(type1, role1, name1)); - instance.put(name2, new Component(type2, role2, name2)); - instance.put(name3, new Component(type3, role3, name3)); - return instance; + return builder() + .put(name1, role1, type1).put(name2, role2, type2).put(name3, role3, type3) + .build(); } /** @@ -111,12 +130,10 @@ public static DataStructure of(BiFunction, ?> converter, String name2, Component.Role role2, Class type2, String name3, Component.Role role3, Class type3, String name4, Component.Role role4, Class type4) { - DataStructure instance = new DataStructure(converter); - instance.put(name1, new Component(type1, role1, name1)); - instance.put(name2, new Component(type2, role2, name2)); - instance.put(name3, new Component(type3, role3, name3)); - instance.put(name4, new Component(type4, role4, name4)); - return instance; + return builder() + .put(name1, role1, type1).put(name2, role2, type2).put(name3, role3, type3).put(name4, role4, type4) + .build(); + } /** @@ -128,13 +145,10 @@ public static DataStructure of(BiFunction, ?> converter, String name3, Component.Role role3, Class type3, String name4, Component.Role role4, Class type4, String name5, Component.Role role5, Class type5) { - DataStructure instance = new DataStructure(converter); - instance.put(name1, new Component(type1, role1, name1)); - instance.put(name2, new Component(type2, role2, name2)); - instance.put(name3, new Component(type3, role3, name3)); - instance.put(name4, new Component(type4, role4, name4)); - instance.put(name5, new Component(type5, role5, name5)); - return instance; + return builder() + .put(name1, role1, type1).put(name2, role2, type2).put(name3, role3, type3).put(name4, role4, type4) + .put(name5, role5, type5) + .build(); } /** @@ -147,45 +161,32 @@ public static DataStructure of(BiFunction, ?> converter, String name4, Component.Role role4, Class type4, String name5, Component.Role role5, Class type5, String name6, Component.Role role6, Class type6) { - DataStructure instance = new DataStructure(converter); - instance.put(name1, new Component(type1, role1, name1)); - instance.put(name2, new Component(type2, role2, name2)); - instance.put(name3, new Component(type3, role3, name3)); - instance.put(name4, new Component(type4, role4, name4)); - instance.put(name5, new Component(type5, role5, name5)); - instance.put(name6, new Component(type6, role6, name6)); - return instance; + return builder() + .put(name1, role1, type1).put(name2, role2, type2).put(name3, role3, type3).put(name4, role4, type4) + .put(name5, role5, type5).put(name6, role6, type6) + .build(); } - public String getName(Component component) { - return delegate.inverse().get(component); + private static IdentityHashMap computeInverseCache(ImmutableMap delegate) { + IdentityHashMap map = Maps.newIdentityHashMap(); + for (Entry entry : delegate.entrySet()) { + map.put(entry.getValue(), entry.getKey()); + } + return map; } - public Component addComponent(String name, Component.Role role, Class type) { - Component component = new Component(type, role, name); - put(name, component); - return component; + public String getName(Component component) { + return this.inverseCache.get(component); } public Map getRoles() { - // TODO: Cache. - return this.entrySet().stream() - .collect(Collectors.toMap( - Entry::getKey, - entry -> entry.getValue().getRole() - )); + return this.roleCache; } public Map> getTypes() { - // TODO: Cache. - return this.entrySet().stream() - .collect(Collectors.toMap( - Entry::getKey, - entry -> entry.getValue().getType() - )); + return this.typeCache; } - public BiFunction, ?> converter() { return this.converter; } @@ -238,4 +239,92 @@ public Dataset.Tuple wrap(Map map) { protected Map delegate() { return delegate; } + + public static class Builder { + + private final ImmutableMap.Builder builder = ImmutableMap.builder(); + private final BiFunction, ?> converter; + + private Builder(BiFunction, ?> converter) { + this.converter = checkNotNull(converter); + } + + private Builder() { + this.converter = (o, aClass) -> o; + } + + /** + * Associates {@code key} with {@code value} in the built map. Duplicate + * keys are not allowed, and will cause {@link #build} to fail. + * + * @param key + * @param value + */ + public Builder put(String key, Component value) { + builder.put(key, value); + return this; + } + + public Builder put(String key, Component.Role role, Class type) { + return put(key, new Component(type, role, key)); + } + + /** + * Adds the given {@code entry} to the map, making it immutable if + * necessary. Duplicate keys are not allowed, and will cause {@link #build} + * to fail. + * + * @param entry + */ + public Builder put(Entry entry) { + builder.put(entry); + return this; + } + + /** + * Associates all of the given map's keys and values in the built map. + * Duplicate keys are not allowed, and will cause {@link #build} to fail. + * + * @param map + * @throws NullPointerException if any key or value in {@code map} is null + */ + public Builder putAll(Map map) { + builder.putAll(map); + return this; + } + + /** + * Adds all of the given entries to the built map. Duplicate keys are not + * allowed, and will cause {@link #build} to fail. + * + * @param entries + * @throws NullPointerException if any key, value, or entry is null + */ + @Beta + public Builder putAll(Iterable> entries) { + builder.putAll(entries); + return this; + } + + /** + * Configures this {@code Builder} to order entries by value according to the specified + * comparator. + *

+ *

The sort order is stable, that is, if two entries have values that compare + * as equivalent, the entry that was inserted first will be first in the built map's + * iteration order. + * + * @param valueComparator + * @throws IllegalStateException if this method was already called + */ + @Beta + public Builder orderEntriesByValue(Comparator valueComparator) { + builder.orderEntriesByValue(valueComparator); + return this; + } + + public DataStructure build() { + return new DataStructure(this.converter, this.builder.build()); + } + } } diff --git a/java-vtl-parser/pom.xml b/java-vtl-parser/pom.xml index 6035d8d7..651b2752 100644 --- a/java-vtl-parser/pom.xml +++ b/java-vtl-parser/pom.xml @@ -18,14 +18,12 @@ org.antlr antlr4-runtime - 4.6 org.antlr antlr4 - 4.5.3 test @@ -36,7 +34,7 @@ org.antlr antlr4-maven-plugin - 4.5 + 4.6 diff --git a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 index 201e75b2..714f69ed 100644 --- a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 +++ b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 @@ -21,9 +21,11 @@ grammar VTL; start : statement+ EOF; /* Assignment */ -statement : variableRef ':=' datasetExpression; +statement : variableID ':=' datasetExpression + | variableID ':=' block + ; -exprMember : datasetExpression ('#' componentID)? ; +block : '{' statement+ '}' ; /* Expressions */ datasetExpression : datasetExpression clauseExpression #withClause @@ -33,7 +35,6 @@ datasetExpression : datasetExpression clauseExpression #withClause | exprAtom #withAtom ; -componentID : IDENTIFIER; getExpression : 'get' '(' datasetId ')'; putExpression : 'put(todo)'; @@ -43,11 +44,15 @@ datasetId : STRING_CONSTANT ; /* Atom */ exprAtom : variableRef; -variableRef : constant - | varID - ; +datasetRef: variableRef ; + +componentRef : ( datasetRef '.')? variableRef ; +variableRef : identifier; + +identifier : '\'' STRING_CONSTANT '\'' | IDENTIFIER ; + +variableID : IDENTIFIER ; -varID : IDENTIFIER; constant : INTEGER_CONSTANT | FLOAT_CONSTANT | BOOLEAN_CONSTANT | STRING_CONSTANT | NULL_CONSTANT; @@ -67,7 +72,7 @@ clause : 'rename' renameParam (',' renameParam)* #renameClause // component as string role = MEASURE, // component as string role = ATTRIBUTE // ] -renameParam : from=varID 'as' to=varID ( 'role' '=' role )? ; +renameParam : from=componentRef 'as' to=identifier ( 'role' '=' role )? ; role : ( 'IDENTIFIER' | 'MEASURE' | 'ATTRIBUTE' ) ; @@ -96,6 +101,7 @@ booleanExpression booleanEquallity : booleanEquallity ( ( EQ | NE | LE | GE ) booleanEquallity ) | datasetExpression + | constant // typed constant? ; @@ -124,52 +130,52 @@ joinDefinition : INNER? joinParam #joinDefinitionInner | OUTER joinParam #joinDefinitionOuter | CROSS joinParam #joinDefinitionCross ; -joinParam : varID (',' varID )* ( 'on' dimensionExpression (',' dimensionExpression )* )? ; +joinParam : datasetRef (',' datasetRef )* ( 'on' dimensionExpression (',' dimensionExpression )* )? ; dimensionExpression : IDENTIFIER; //unimplemented joinBody : joinClause (',' joinClause)* ; -joinClause : role? varID '=' joinCalcExpression # joinCalcClause +joinClause : role? variableID '=' joinCalcExpression # joinCalcClause | joinDropExpression # joinDropClause | joinKeepExpression # joinKeepClause | joinRenameExpression # joinRenameClause | joinFilterExpression # joinFilterClause + | joinFoldExpression # joinFoldClause + | joinUnfoldExpression # joinUnfoldClause ; - //| joinFilter - //| joinKeep - //| joinRename ; - //| joinDrop - //| joinUnfold - //| joinFold ; + +joinFoldExpression : 'fold' elements=componentRefs 'to' dimension=identifier ',' measure=identifier ; +componentRefs : componentRef (',' componentRef)* ; + +joinUnfoldExpression : 'unfold' dimension=componentRef ',' measure=componentRef 'to' elements=foldUnfoldElements ; +// TODO: The spec writes examples with parentheses, but it seems unecessary to me. +// TODO: The spec is unclear regarding types of the elements, we support strings only for now. +// TODO: Reuse component references +foldUnfoldElements : STRING_CONSTANT (',' STRING_CONSTANT)* ; // Left recursive joinCalcExpression : leftOperand=joinCalcExpression sign=( '*' | '/' ) rightOperand=joinCalcExpression #joinCalcProduct | leftOperand=joinCalcExpression sign=( '+' | '-' ) rightOperand=joinCalcExpression #joinCalcSummation | '(' joinCalcExpression ')' #joinCalcPrecedence - | joinCalcRef #joinCalcReference + | componentRef #joinCalcReference | constant #joinCalcAtom ; -joinCalcRef : (aliasName=varID '.')? componentName=varID ; - // Drop clause -joinDropExpression : 'drop' joinDropRef (',' joinDropRef)* ; -joinDropRef : (aliasName=varID '.')? componentName=varID ; +joinDropExpression : 'drop' componentRef (',' componentRef)* ; // Keep clause -joinKeepExpression : 'keep' joinKeepRef (',' joinKeepRef)* ; -joinKeepRef : (aliasName=varID '.')? componentName=varID ; +joinKeepExpression : 'keep' componentRef (',' componentRef)* ; // TODO: Use in keep, drop and calc. // TODO: Make this the membership operator. // TODO: Revise this when the final version of the specification precisely define if the rename needs ' or not. -joinComponentReference : (aliasName=varID '.')? componentName=varID ; // Rename clause joinRenameExpression : 'rename' joinRenameParameter (',' joinRenameParameter)* ; -joinRenameParameter : from=joinComponentReference 'to' to=varID ; +joinRenameParameter : from=componentRef 'to' to=identifier ; // Filter clause joinFilterExpression : 'filter' booleanExpression ; @@ -180,7 +186,6 @@ INNER : 'inner' ; OUTER : 'outer' ; CROSS : 'cross' ; -IDENTIFIER:LETTER(LETTER|'_'|DIGIT)* ; INTEGER_CONSTANT : DIGIT+; BOOLEAN_CONSTANT : 'true' | 'false' ; @@ -192,6 +197,13 @@ FLOAT_CONSTANT : (DIGIT)+ '.' (DIGIT)* FLOATEXP? NULL_CONSTANT : 'null'; +IDENTIFIER : REG_IDENTIFIER | ESCAPED_IDENTIFIER ; +//regular identifiers start with a (lowercase or uppercase) English alphabet letter, followed by zero or more letters, decimal digits, or underscores +REG_IDENTIFIER: LETTER(LETTER|'_'|DIGIT)* ; //TODO: Case insensitive?? +//VTL 1.1 allows us to escape the limitations imposed on regular identifiers by enclosing them in single quotes (apostrophes). +fragment ESCAPED_IDENTIFIER: QUOTE (~'\'' | '\'\'')+ QUOTE; +fragment QUOTE : '\''; + PLUS : '+'; MINUS : '-'; diff --git a/java-vtl-parser/src/test/java/no/ssb/vtl/parser/FoldAndUnfoldTest.java b/java-vtl-parser/src/test/java/no/ssb/vtl/parser/FoldAndUnfoldTest.java new file mode 100644 index 00000000..fa23b4c5 --- /dev/null +++ b/java-vtl-parser/src/test/java/no/ssb/vtl/parser/FoldAndUnfoldTest.java @@ -0,0 +1,98 @@ +package no.ssb.vtl.parser; + +import com.google.common.io.Resources; +import org.antlr.v4.tool.Grammar; +import org.assertj.core.api.AutoCloseableSoftAssertions; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.ExternalResource; + +import java.net.URL; +import java.nio.charset.Charset; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.io.Resources.getResource; +import static no.ssb.vtl.parser.ParserTestHelper.filterWhiteSpaces; +import static no.ssb.vtl.parser.ParserTestHelper.parse; + +public class FoldAndUnfoldTest { + private static Grammar grammar; + @ClassRule + public static ExternalResource grammarResource = new ExternalResource() { + @Override + protected void before() throws Throwable { + URL grammarURL = getResource(this.getClass(), "VTL.g4"); + String grammarString = Resources.toString(grammarURL, Charset.defaultCharset()); + grammar = new Grammar(checkNotNull(grammarString)); + } + }; + + @Test + public void testJoinFold() throws Exception { + String rule = "joinFoldExpression"; + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + String parseTree; + + parseTree = parse("fold \"varID1\", \"varID2\" to dataset.component, component", rule, grammar); + softly.assertThat( + filterWhiteSpaces(parseTree) + ).isEqualTo(filterWhiteSpaces("(joinFoldExpression:1fold(foldUnfoldElements:1\"varID1\",\"varID2\")to(joinFoldUnfoldRef:1(varID:1dataset).(componentID:1component)),(joinFoldUnfoldRef:2(componentID:1component)))")); + + parseTree = parse("fold \"varID1\" to component, dataset.component", rule, grammar); + softly.assertThat( + filterWhiteSpaces(parseTree) + ).isEqualTo(filterWhiteSpaces("(joinFoldExpression:1fold(foldUnfoldElements:1\"varID1\")to(joinFoldUnfoldRef:2(componentID:1component)),(joinFoldUnfoldRef:1(varID:1dataset).(componentID:1component)))")); + + parseTree = parse("fold \"varID1\" to component, component", rule, grammar); + softly.assertThat( + filterWhiteSpaces(parseTree) + ).isEqualTo(filterWhiteSpaces("(joinFoldExpression:1fold(foldUnfoldElements:1\"varID1\")to(joinFoldUnfoldRef:2(componentID:1component)),(joinFoldUnfoldRef:2(componentID:1component)))")); + + parseTree = parse("fold \"varID1\" to dataset.component, dataset.component", rule, grammar); + softly.assertThat( + filterWhiteSpaces(parseTree) + ).isEqualTo(filterWhiteSpaces("(joinFoldExpression:1fold(foldUnfoldElements:1\"varID1\")to(joinFoldUnfoldRef:1(varID:1dataset).(componentID:1component)),(joinFoldUnfoldRef:1(varID:1dataset).(componentID:1component)))")); + + softly.assertThatThrownBy(() -> parse("fold to dataset.component, component", rule, grammar)); + softly.assertThatThrownBy(() -> parse("fold \"varID1\" to component", rule, grammar)); + softly.assertThatThrownBy(() -> parse("fold \"varID1\" to ,component", rule, grammar)); + } + + + } + + @Test + public void testJoinUnfold() throws Exception { + String rule = "joinUnfoldExpression"; + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + String parseTree; + + parseTree = parse("unfold dataset.component, component to \"varID1\", \"varID2\"", rule, grammar); + softly.assertThat( + filterWhiteSpaces(parseTree) + ).isEqualTo(filterWhiteSpaces("(joinUnfoldExpression:1unfold(joinFoldUnfoldRef:1(varID:1dataset).(componentID:1component)),(joinFoldUnfoldRef:2(componentID:1component))to(foldUnfoldElements:1\"varID1\",\"varID2\"))")); + + parseTree = parse("unfold component, dataset.component to \"varID1\"", rule, grammar); + softly.assertThat( + filterWhiteSpaces(parseTree) + ).isEqualTo(filterWhiteSpaces("(joinUnfoldExpression:1unfold(joinFoldUnfoldRef:2(componentID:1component)),(joinFoldUnfoldRef:1(varID:1dataset).(componentID:1component))to(foldUnfoldElements:1\"varID1\"))")); + + parseTree = parse("unfold dataset.component, dataset.component to \"varID1\"", rule, grammar); + softly.assertThat( + filterWhiteSpaces(parseTree) + ).isEqualTo(filterWhiteSpaces("(joinUnfoldExpression:1unfold(joinFoldUnfoldRef:1(varID:1dataset).(componentID:1component)),(joinFoldUnfoldRef:1(varID:1dataset).(componentID:1component))to(foldUnfoldElements:1\"varID1\"))")); + + parseTree = parse("unfold component, component to \"varID1\"", rule, grammar); + softly.assertThat( + filterWhiteSpaces(parseTree) + ).isEqualTo(filterWhiteSpaces("(joinUnfoldExpression:1unfold(joinFoldUnfoldRef:2(componentID:1component)),(joinFoldUnfoldRef:2(componentID:1component))to(foldUnfoldElements:1\"varID1\"))")); + + softly.assertThatThrownBy(() -> parse("unfold dataset.component, component to ", rule, grammar)); + softly.assertThatThrownBy(() -> parse("unfold dataset.component, component, component to \"varID1\"", rule, grammar)); + softly.assertThatThrownBy(() -> parse("unfold component to \"varID1\"", rule, grammar)); + softly.assertThatThrownBy(() -> parse("unfold ,component to \"varID1\"", rule, grammar)); + } + } +} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptContext.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptContext.java new file mode 100644 index 00000000..8b0062b4 --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptContext.java @@ -0,0 +1,189 @@ +package no.ssb.vtl.script; + +import javax.script.Bindings; +import javax.script.SimpleBindings; +import javax.script.SimpleScriptContext; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class VTLScriptContext extends SimpleScriptContext{ + + private Map scopes; + + @SuppressWarnings("WeakerAccess") + public VTLScriptContext() { + super(); + scopes = new HashMap<>(2); + scopes.put(ENGINE_SCOPE, engineScope); + } + + public void addScope(int scope) { + scopes.put(scope, new SimpleBindings()); + } + + + + /** + * Associates a Bindings instance with a particular scope in this + * ScriptContext. Calls to the getAttribute and + * setAttribute methods must map to the get and + * put methods of the Bindings for the specified scope. + * If the scope does not already exists in this ScriptContext it will be added + * @param bindings The Bindings to associate with the given scope + * @param scope The scope + * @throws NullPointerException if the specified Bindings is null. + */ + @Override + public void setBindings(Bindings bindings, int scope) { + if (bindings == null) { + throw new NullPointerException("Bindings for a scope cannot be null"); + } + scopes.put(scope, bindings); + } + + /** + * Gets the Bindings associated with the given scope in this + * ScriptContext. + * @param scope The scope + * @return The associated Bindings. Returns null if it has not + * been set. + * @throws IllegalArgumentException If no Bindings is defined for the + * specified scope value in ScriptContext of this type. + */ + @Override + public Bindings getBindings(int scope) { + checkScope(scope); + return scopes.get(scope); + } + + /** + * Sets the value of an attribute in a given scope. + * @param name The name of the attribute to set + * @param value The value of the attribute + * @param scope The scope in which to set the attribute + * @throws IllegalArgumentException if the name is empty or if the scope is invalid. + * @throws NullPointerException if the name is null. + */ + @Override + public void setAttribute(String name, Object value, int scope) { + checkName(name); + checkScope(scope); + scopes.get(scope).put(name, value); + } + + /** + * Sets the value of an attribute in the default engine scope. + * @param name The name of the attribute to set + * @param value The value of the attribute + * @throws IllegalArgumentException if the name is empty. + * @throws NullPointerException if the name is null. + */ + public void setAttribute(String name, Object value) { + setAttribute(name, value, ENGINE_SCOPE); + } + + /** + * Gets the value of an attribute in a given scope. + * @param name The name of the attribute to retrieve. + * @param scope The scope in which to retrieve the attribute. + * @return The value of the attribute. Returns null is the name + * does not exist in the given scope. + * @throws IllegalArgumentException if the name is empty or if the value of scope is invalid. + * @throws NullPointerException if the name is null. + */ + @Override + public Object getAttribute(String name, int scope) { + checkName(name); + checkScope(scope); + return scopes.get(scope).get(name); + } + + /** + * Remove an attribute in a given scope. + * @param name The name of the attribute to remove + * @param scope The scope in which to remove the attribute + * @return The removed value. + * @throws IllegalArgumentException if the name is empty or if the scope is invalid. + * @throws NullPointerException if the name is null. + */ + @Override + public Object removeAttribute(String name, int scope) { + checkScope(scope); + return scopes.get(scope).remove(name); + } + + /** + * Retrieves the value of the attribute with the given name in + * the scope occurring earliest in the search order. The order + * is determined by the numeric value of the scope parameter (lowest + * scope values first.) + * @param name The name of the the attribute to retrieve. + * @return The value of the attribute in the lowest scope for + * which an attribute with the given name is defined. Returns + * null if no attribute with the name exists in any scope. + * @throws NullPointerException if the name is null. + * @throws IllegalArgumentException if the name is empty. + */ + @Override + public Object getAttribute(String name) { + checkName(name); + for (int scope : getScopes()) { + Bindings bindings = scopes.get(scope); + if (bindings.containsKey(name)) { + return bindings.get(name); + } + } + return null; + } + + /** + * Get the lowest scope in which an attribute is defined. + * @param name Name of the attribute + * . + * @return The lowest scope. Returns -1 if no attribute with the given + * name is defined in any scope. + * @throws NullPointerException if name is null. + * @throws IllegalArgumentException if name is empty. + */ + @Override + public int getAttributesScope(String name) { + checkName(name); + for (int scope : getScopes()) { + Bindings bindings = scopes.get(scope); + if (bindings.containsKey(name)) { + return scope; + } + } + return -1; + } + + /** + * Returns immutable List of all the valid values for + * scope in the ScriptContext. + * @return list of scope values + */ + @Override + public List getScopes() { + List scopeList = new ArrayList<>(scopes.keySet()); + Collections.sort(scopeList); + return Collections.unmodifiableList(scopeList); + } + + private void checkName(String name) { + Objects.requireNonNull(name); + if (name.isEmpty()) { + throw new IllegalArgumentException("name cannot be empty"); + } + } + + private void checkScope(int scope) { + if (!scopes.containsKey(scope)) { + throw new IllegalArgumentException("Illegal scope value"); + } + } + +} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptEngine.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptEngine.java index 498be21f..7319d180 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptEngine.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptEngine.java @@ -51,7 +51,7 @@ public class VTLScriptEngine extends AbstractScriptEngine { */ public VTLScriptEngine(Connector... connectors) { this.connectors = ImmutableList.copyOf(connectors); - + context = new VTLScriptContext(); } /** @@ -63,6 +63,7 @@ public VTLScriptEngine(Connector... connectors) { public VTLScriptEngine(Bindings n, Connector... connectors) { super(n); this.connectors = ImmutableList.copyOf(connectors); + context = new VTLScriptContext(); } @Override diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/DropOperator.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/DropOperator.java index 81278339..20a3625c 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/DropOperator.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/DropOperator.java @@ -1,13 +1,10 @@ package no.ssb.vtl.script.operations; import com.google.common.base.MoreObjects; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; import no.ssb.vtl.model.Component; import no.ssb.vtl.model.DataStructure; import no.ssb.vtl.model.Dataset; -import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.stream.Stream; @@ -19,17 +16,18 @@ * TODO: Implement "operator" and "function" interfaces. */ public class DropOperator implements Dataset { + // The dataset we are applying the KeepOperator on. private final Dataset dataset; - private final Set components; + private final Set components; private DataStructure cache; - public DropOperator(Dataset dataset, Set names) { + public DropOperator(Dataset dataset, Set components) { this.dataset = checkNotNull(dataset, "the dataset was null"); - this.components = checkNotNull(names, "the component list was null"); + this.components = checkNotNull(components, "the component list was null"); - checkArgument(!names.isEmpty(), "the list of component to drop was null"); + checkArgument(!components.isEmpty(), "the list of component to drop was null"); } @Override @@ -39,26 +37,16 @@ public DataStructure getDataStructure() { /** * Compute the new data structure. - * - * @return */ private DataStructure computeDataStructure() { - DataStructure structure = dataset.getDataStructure(); - Map roles = Maps.newHashMap(); - Map> types = Maps.newHashMap(); - for (String componentName : structure.keySet()) { - if (!components.contains(componentName) || structure.get(componentName).isIdentifier()) { - Class type = structure.getTypes().get(componentName); - Component.Role role = structure.getRoles().get(componentName); - roles.put(componentName, role); - types.put(componentName, type); + DataStructure.Builder newDataStructure = DataStructure.builder(); + for (Map.Entry componentEntry : dataset.getDataStructure().entrySet()) { + Component component = componentEntry.getValue(); + if (!components.contains(component) || component.isIdentifier()) { + newDataStructure.put(componentEntry); } } - return DataStructure.of( - structure.converter(), - types, - roles - ); + return newDataStructure.build(); } @Override @@ -66,7 +54,7 @@ public Stream get() { DataStructure structure = getDataStructure(); return dataset.get().map( dataPoints -> { - dataPoints.removeIf(dataPoint -> !structure.containsKey(dataPoint.getName())); + dataPoints.removeIf(dataPoint -> !structure.containsValue(dataPoint.getComponent())); return dataPoints; } ); @@ -75,13 +63,8 @@ public Stream get() { @Override public String toString() { MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); - Integer limit = 5; - Iterables.limit(Iterables.concat( - components, - Collections.singletonList("and " + (components.size() - limit) + " more") - ), Math.min(limit, components.size())).forEach( - helper::addValue - ); - return helper.toString(); + helper.addValue(components); + helper.add("structure", cache); + return helper.omitNullValues().toString(); } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/FilterOperator.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/FilterOperator.java index ccbc85d3..3ff2c1ec 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/FilterOperator.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/FilterOperator.java @@ -40,6 +40,7 @@ public String toString() { MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); Map> predicateResultMap = dataset.stream().collect(Collectors.partitioningBy(predicate)); helper.addValue(predicateResultMap); - return helper.toString(); + helper.add("structure", cache); + return helper.omitNullValues().toString(); } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/FoldClause.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/FoldClause.java new file mode 100644 index 00000000..661d1caa --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/FoldClause.java @@ -0,0 +1,126 @@ +package no.ssb.vtl.script.operations; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.*; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.DataPoint; +import no.ssb.vtl.model.DataStructure; +import no.ssb.vtl.model.Dataset; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static no.ssb.vtl.model.Component.Role; + +/** + * Fold clause. + */ +public class FoldClause implements Dataset { + + // Source dataset. + private final Dataset dataset; + + private final String dimension; + private final String measure; + private final Set elements; + + private DataStructure cache = null; + + public FoldClause(Dataset dataset, String dimensionReference, String measureReference, Set elements) { + // TODO: Introduce type here. Elements should be of the type of the Component. + + this.dataset = checkNotNull(dataset, "dataset cannot be null"); + checkArgument(!(this.dimension = checkNotNull(dimensionReference, "dimensionReference cannot be null")).isEmpty(), + "dimensionReference was empty"); + checkArgument(!(this.measure = checkNotNull(measureReference, "measureReference cannot be null")).isEmpty(), + "measureReference was empty"); + checkArgument(!(this.elements = checkNotNull(elements, "elements cannot be null")).isEmpty(), + "elements was empty"); + } + + private DataStructure computeDataStructure() { + DataStructure dataStructure = dataset.getDataStructure(); + + // TODO: Constraint error. + checkArgument( + dataStructure.values().containsAll(elements), + "the element(s) [%s] were not found in [%s]", + Sets.difference(elements, dataStructure.keySet()), dataStructure.keySet() + ); + + // Checks that elements are of the same type + ListMultimap, Component> classes = ArrayListMultimap.create(); + for (Component element : elements) { + classes.put(element.getType(), element); + } + checkArgument( + classes.asMap().size() == 1, + "the element(s) [%s] must be of the same type, found [%s] in dataset [%s]", + elements, classes, dataStructure + ); + + Map newRoles = Maps.newLinkedHashMap(); + Map> newTypes = Maps.newLinkedHashMap(); + for (Map.Entry componentEntry : dataStructure.entrySet()) { + if (!elements.contains(componentEntry.getValue())) { + newRoles.put(componentEntry.getKey(), componentEntry.getValue().getRole()); + newTypes.put(componentEntry.getKey(), componentEntry.getValue().getType()); + } + } + + newRoles.put(dimension, Role.IDENTIFIER); + newTypes.put(dimension, String.class); + + newRoles.put(measure, Role.MEASURE); + newTypes.put(measure, classes.keySet().iterator().next()); + + return DataStructure.of(dataStructure.converter(), newTypes, newRoles); + } + + @Override + public DataStructure getDataStructure() { + return cache = (cache == null ? computeDataStructure() : cache); + } + + @Override + public Stream get() { + DataStructure dataStructure = getDataStructure(); + return dataset.get().flatMap(tuple -> { + List tuples = Lists.newArrayList(); + Map commonValues = Maps.newLinkedHashMap(); + Map foldedValues = Maps.newLinkedHashMap(); + + for (DataPoint point : tuple) { + if (elements.contains(point.getComponent())) { + foldedValues.put(point.getName(), point.get()); + } else { + commonValues.put(point.getName(), point.get()); + } + } + + for (Component element : elements) { + if (foldedValues.containsKey(element.getName()) && foldedValues.get(element.getName()) != null) { + Map rowMap = Maps.newLinkedHashMap(commonValues); + rowMap.put(dimension, element.getName()); + rowMap.put(measure, foldedValues.get(element.getName())); + tuples.add(dataStructure.wrap(rowMap)); + } + } + return tuples.stream(); + }); + } + + @Override + public String toString() { + MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); + helper.addValue(elements); + helper.add("identifier", dimension); + helper.add("measure", measure); + helper.add("structure", cache); + return helper.omitNullValues().toString(); + } +} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/KeepOperator.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/KeepOperator.java index 82fdf48e..976d130e 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/KeepOperator.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/KeepOperator.java @@ -1,13 +1,10 @@ package no.ssb.vtl.script.operations; import com.google.common.base.MoreObjects; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; import no.ssb.vtl.model.Component; import no.ssb.vtl.model.DataStructure; import no.ssb.vtl.model.Dataset; -import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.stream.Stream; @@ -19,11 +16,11 @@ public class KeepOperator implements Dataset { // The dataset we are applying the KeepOperator on. private final Dataset dataset; - private final Set components; + private final Set components; private DataStructure cache; - public KeepOperator(Dataset dataset, Set names) { + public KeepOperator(Dataset dataset, Set names) { this.dataset = checkNotNull(dataset, "the dataset was null"); this.components = checkNotNull(names, "the component list was null"); @@ -37,27 +34,16 @@ public DataStructure getDataStructure() { /** * Compute the new data structure. - * @return */ private DataStructure computeDataStructure() { - DataStructure structure = dataset.getDataStructure(); - Map roles = Maps.newHashMap(); - Map> types = Maps.newHashMap(); - for (String componentName : structure.keySet()) { - // Must keep ID TODO: Should we fail here? Or should the failure come from the WorkingDataset? - // TODO: As it is now, the ids of the working dataset are immutable anyways. - if (components.contains(componentName) || structure.get(componentName).isIdentifier()) { - Class type = structure.getTypes().get(componentName); - Component.Role role = structure.getRoles().get(componentName); - roles.put(componentName, role); - types.put(componentName, type); + DataStructure.Builder newDataStructure = DataStructure.builder(); + for (Map.Entry componentEntry : dataset.getDataStructure().entrySet()) { + Component component = componentEntry.getValue(); + if (components.contains(component) || component.isIdentifier()) { + newDataStructure.put(componentEntry); } } - return DataStructure.of( - structure.converter(), - types, - roles - ); + return newDataStructure.build(); } @Override @@ -65,7 +51,7 @@ public Stream get() { DataStructure structure = getDataStructure(); return dataset.get().map( dataPoints -> { - dataPoints.removeIf(dataPoint -> !structure.containsKey(dataPoint.getName())); + dataPoints.removeIf(dataPoint -> !structure.containsValue(dataPoint.getComponent())); return dataPoints; } ); @@ -74,13 +60,8 @@ public Stream get() { @Override public String toString() { MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); - Integer limit = 5; - Iterables.limit(Iterables.concat( - components, - Collections.singletonList("and " + (components.size() - limit) + " more") - ), Math.min(limit, components.size())).forEach( - helper::addValue - ); - return helper.toString(); + helper.addValue(components); + helper.add("structure", cache); + return helper.omitNullValues().toString(); } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/RenameOperation.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/RenameOperation.java index 777e751a..1f37f428 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/RenameOperation.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/RenameOperation.java @@ -21,7 +21,8 @@ */ import com.google.common.base.MoreObjects; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import no.ssb.vtl.model.Component; import no.ssb.vtl.model.DataPoint; import no.ssb.vtl.model.DataStructure; @@ -29,98 +30,132 @@ import java.util.Collections; import java.util.Map; -import java.util.Set; -import java.util.function.BiFunction; import java.util.stream.Stream; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; /** * Rename operation. - * + *

* TODO: Implement {@link Dataset} */ public class RenameOperation implements Dataset { private final Dataset dataset; - private final ImmutableMap names; - private final ImmutableMap roles; - Map mapping = HashBiMap.create(); + private final Map mapping = Maps.newHashMap(); + private final Map newNames; + private final Map newRoles; + private DataStructure cache; - public RenameOperation(Dataset dataset, Map names, Map roles) { - this.dataset = checkNotNull(dataset, "dataset was null"); + public RenameOperation(Dataset dataset, Map newNames) { + this.dataset = checkNotNull(dataset); + this.newNames = newNames; + this.newRoles = Collections.emptyMap(); + } - // Checks that names key and values are unique. - HashBiMap.create(names); + public RenameOperation(Dataset dataset, Map newNames, Map newRoles) { + this.dataset = checkNotNull(dataset); + this.newNames = newNames; + this.newRoles = newRoles; + } - checkArgument(names.keySet().containsAll(roles.keySet()), "keys %s not present in %s", - Sets.difference(roles.keySet(), names.keySet()), names.keySet()); + /** + * Compute a Map + */ + private static ImmutableMap computeNames(Map newNames) { + checkNotNull(newNames); + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Map.Entry entry : newNames.entrySet()) { + builder.put(entry.getKey().getName(), entry.getValue()); + } + return builder.build(); + } - this.names = ImmutableMap.copyOf(names); - this.roles = ImmutableMap.copyOf(roles); + /** + * Compute a Map + */ + private static ImmutableMap computeRoles(Map newRoles) { + checkNotNull(newRoles); + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Map.Entry entry : newRoles.entrySet()) { + builder.put(entry.getKey().getName(), entry.getValue()); + } + return builder.build(); + } + + /** + * Compute the role from the components. + */ + private ImmutableMap computeSameRole(Map newNames) { + checkNotNull(newNames); + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Map.Entry entry : newNames.entrySet()) { + builder.put(entry.getValue(), entry.getKey().getRole()); + } + return builder.build(); } @Override public DataStructure getDataStructure() { + return cache = (cache == null ? computeDataStructure() : cache); + } - if (cache == null) { - Set oldNames = dataset.getDataStructure().keySet(); - checkArgument(oldNames.containsAll(names.keySet()), - "the data set %s did not contain components with names %s", dataset, - Sets.difference(names.keySet(), oldNames) - ); - - // Copy. - final Map oldRoles, newRoles; - oldRoles = dataset.getDataStructure().getRoles(); - newRoles = Maps.newHashMap(); - - final Map> oldTypes, newTypes; - oldTypes = dataset.getDataStructure().getTypes(); - newTypes = Maps.newHashMap(); - - - - for (String oldName : oldNames) { - String newName = names.getOrDefault(oldName, oldName); - newTypes.put(newName, oldTypes.get(oldName)); - newRoles.put(newName, roles.getOrDefault(oldName, oldRoles.get(oldName))); - mapping.put(oldName, newName); + private DataStructure computeDataStructure() { + Map map = Maps.newHashMap(); + DataStructure.Builder newDataStructure = DataStructure.builder(); + for (Map.Entry componentEntry : dataset.getDataStructure().entrySet()) { + Component component = componentEntry.getValue(); + if (newNames.containsKey(component)) { + String oldName = component.getName(); + String newName = newNames.get(component); + map.put(component, newName); + newDataStructure.put( + newName, + newRoles.getOrDefault(component, component.getRole()), + component.getType() + ); + } else { + newDataStructure.put(componentEntry); } + } + DataStructure builtDataStructure = newDataStructure.build(); - BiFunction, ?> converter = dataset.getDataStructure().converter(); - cache = DataStructure.of(converter, newTypes, newRoles); + // This is twisted, but there is no way to get the + // component before the builder is built. + for (Map.Entry entry : map.entrySet()) { + mapping.put(entry.getKey(), builtDataStructure.get(entry.getValue())); } - return cache; + return builtDataStructure; } @Override public Stream get() { - return dataset.get().map(components -> { - - components.replaceAll(component -> new DataPoint(getDataStructure().get(mapping.get(component.getName()))) { - @Override - public Object get() { - return component.get(); + return dataset.get().map(points -> { + points.replaceAll(point -> { + Component component = point.getComponent(); + if (mapping.containsKey(component)) { + return new DataPoint(mapping.get(component)) { + @Override + public Object get() { + return point.get(); + } + }; + } else { + return point; } }); - return components; + return points; }); } @Override public String toString() { MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); - Integer limit = 5; - Iterables.limit(Iterables.concat( - mapping.entrySet(), - Collections.singletonList("and " + (mapping.entrySet().size() - limit) + " more") - ), Math.min(limit, mapping.entrySet().size())).forEach( - helper::addValue - ); - return helper.toString(); + helper.addValue(newNames); + helper.addValue(newRoles); + helper.add("structure", cache); + return helper.omitNullValues().toString(); } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/UnfoldClause.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/UnfoldClause.java new file mode 100644 index 00000000..26a43d37 --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/UnfoldClause.java @@ -0,0 +1,165 @@ +package no.ssb.vtl.script.operations; + +import com.codepoetics.protonpack.StreamUtils; +import com.google.common.base.MoreObjects; +import com.google.common.collect.Maps; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.Component.Role; +import no.ssb.vtl.model.DataPoint; +import no.ssb.vtl.model.DataStructure; +import no.ssb.vtl.model.Dataset; + +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * Unfold clause. + */ +public class UnfoldClause implements Dataset { + + // Source dataset. + private final Dataset dataset; + + private final Component dimension; + private final Component measure; + private final Set elements; + private DataStructure cache; + + public UnfoldClause(Dataset dataset, Component dimensionReference, Component measureReference, Set elements) { + this.dataset = checkNotNull(dataset, "dataset cannot be null"); + + this.dimension = checkNotNull(dimensionReference, "dimensionReference cannot be null"); + this.measure = checkNotNull(measureReference, "measureReference cannot be null"); + checkArgument(!(this.elements = checkNotNull(elements, "elements cannot be null")).isEmpty(), + "elements was empty"); + // TODO: Introduce type here. Elements should be of the type of the Component. + } + + @Override + public DataStructure getDataStructure() { + return cache = (cache == null ? computeDataStructure() : cache); + } + + private DataStructure computeDataStructure() { + DataStructure dataStructure = dataset.getDataStructure(); + + // TODO: Does those check still make sense with the Reference visitor? + checkArgument( + dataStructure.containsValue(dimension), + "the dimension [%s] was not found in %s", dimension, dataset + ); + checkArgument( + dataStructure.containsValue(measure), + "the measure [%s] was not found in %s", measure, dataset + ); + + checkArgument( + dimension.isIdentifier(), + "[%s] in dataset [%s] was not a dimension", dimension, dataset + ); + checkArgument( + measure.isMeasure(), + "[%s] in dataset [%s] was not a measure", measure, dataset + ); + + /* + The spec is a bit unclear here; it does not say how to handle the measure values + that are not part of the fold operation. Given the dataset: + [A:ID, B:ID, C:ME, D:ME, E:AT] + + the result of "unfold B, C to foo, bar" could be either: + [A:ID, foo:ME, bar:ME, D:ME] with repeated values or nulls + or + [A:ID, foo:ME, bar:ME] + + until further clarification, the later is implemented. + */ + + DataStructure.Builder newDataStructure = DataStructure.builder(); + for (Map.Entry componentEntry : dataStructure.entrySet()) {; + Component component = componentEntry.getValue(); + if (component != dimension && component != measure) { + if (component.isIdentifier()) { + newDataStructure.put(componentEntry); + } + } + } + + Class type = measure.getType(); + for (String element : elements) { + newDataStructure.put(element, Role.MEASURE, type); + } + return newDataStructure.build(); + } + + @Override + public Stream get() { + // TODO: Handle sorting. Need to request sorting by dimension happens after all others. + // TODO: Maybe put a filter before. + // TODO: Expose more powerful methods in Datapoint. + DataStructure dataStructure = getDataStructure(); + return StreamUtils.aggregate(dataset.get(), (left, right) -> { + // Checks if the previous ids (except the one with unfold on) where different. + Iterator leftIt = left.iterator(); + Iterator rightIt = right.iterator(); + while (leftIt.hasNext() && rightIt.hasNext()) { + DataPoint leftValue = leftIt.next(); + DataPoint rightValue = rightIt.next(); + if (!leftValue.getRole().equals(Role.IDENTIFIER)) { + continue; + } + if (dimension == leftValue.getComponent()) { + continue; + } + if (!leftValue.equals(rightValue)) { + return false; + } + } + return true; + }).map(tuples -> { + // TODO: Naive implementation for now. + Map map = Maps.newLinkedHashMap(); + for (Tuple tuple : tuples) { + Object unfoldedValue = null; + String columnName = null; + for (DataPoint dataPoint : tuple) { + if (dimension == dataPoint.getComponent()) { + // TODO: Check type of the elements. Can we use element that are not String?? + if (elements.contains(dataPoint.get())) { + columnName = (String) dataPoint.get(); + continue; + } + } + if (measure == dataPoint.getComponent()) { + unfoldedValue = dataPoint.get(); + continue; + } + if (dataPoint.getRole().equals(Role.IDENTIFIER)) { + map.put(dataPoint.getName(), dataPoint.get()); + } + } + map.put(columnName, unfoldedValue); + } + // TODO: >_<' ... + for (String element : elements) { + map.putIfAbsent(element, null); + } + return dataStructure.wrap(map); + }); + } + + @Override + public String toString() { + MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); + helper.add("identifier", dimension); + helper.add("measure", measure); + helper.addValue(elements); + helper.add("structure", cache); + return helper.omitNullValues().toString(); + } +} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/AbstractJoinOperation.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/AbstractJoinOperation.java index 07735228..1dc133ef 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/AbstractJoinOperation.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/AbstractJoinOperation.java @@ -19,22 +19,15 @@ * #L% */ -import com.google.common.collect.HashMultiset; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Multiset; +import com.google.common.collect.*; import no.ssb.vtl.model.Component; import no.ssb.vtl.model.DataPoint; -import no.ssb.vtl.model.DataStructure; import no.ssb.vtl.model.Dataset; -import no.ssb.vtl.script.operations.RenameOperation; -import java.util.List; -import java.util.Map; -import java.util.RandomAccess; -import java.util.Set; +import javax.script.Bindings; +import javax.script.SimpleBindings; +import java.util.*; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.google.common.base.Preconditions.checkArgument; import static no.ssb.vtl.model.Component.Role; @@ -42,23 +35,15 @@ /** * Abstract join operation. */ -public abstract class AbstractJoinOperation implements Dataset { +public abstract class AbstractJoinOperation { // The datasets the join operates on. - private final Map datasets = Maps.newHashMap(); - - public Set getCommonIdentifierNames() { - return commonIdentifierNames; - } + private final Map datasets; // The identifiers that will be used to join the datasets. private final Set commonIdentifierNames; - // Holds the operations of the join. - private final List clauses = Lists.newArrayList(); - //private final Iterator clauseIterator = ; - - private WorkingDataset workingDataset; + private final Bindings joinScope; public AbstractJoinOperation(Map namedDatasets) { @@ -67,28 +52,50 @@ public AbstractJoinOperation(Map namedDatasets) { "join operation impossible on empty dataset list" ); - // Find the common identifier. - Multiset components = HashMultiset.create(); + Multiset components = getComponents(namedDatasets); + + Set> commonEntries = Sets.newHashSet(); for (Dataset dataset : namedDatasets.values()) { - DataStructure structure = dataset.getDataStructure(); - components.addAll(structure.values()); + for (Map.Entry entry : dataset.getDataStructure().entrySet()) { + if (entry.getValue().isIdentifier()) { + commonEntries.add(entry); + } + } } + checkArgument(!commonEntries.isEmpty(), "could not find common identifiers in the datasets %s", namedDatasets); - commonIdentifierNames = components.entrySet().stream() - .filter(entry -> entry.getCount() == namedDatasets.size()) - .map(Multiset.Entry::getElement) - .filter(component -> component.getRole() == Role.IDENTIFIER) + // TODO: Remove + commonIdentifierNames = commonEntries.stream() + .map(Map.Entry::getValue) .map(Component::getName) .collect(Collectors.toSet()); - checkArgument(!commonIdentifierNames.isEmpty(), "could not find common identifiers in the datasets %s", namedDatasets); + //joinScope = createJoinScope(namedDatasets, commonIdentifiers); // Rename all the components except the common identifiers. + //this.datasets = createDataset(namedDatasets); + this.datasets = namedDatasets; + this.joinScope = new JoinScopeBindings(this.datasets); + + } + + private Multiset getComponents(Map namedDatasets) { + List componentsList = namedDatasets.values().stream() + .flatMap(dataset -> dataset.getDataStructure().values().stream()) + .collect(Collectors.toList()); + return HashMultiset.create(componentsList); + } + + /** + * Creates a map of datasets with identity equivalent identifier components. + */ + private Map createDataset(Map namedDatasets) { + Map datasets = Maps.newHashMap(); for (String datasetName : namedDatasets.keySet()) { Dataset dataset = namedDatasets.get(datasetName); Map newNames = Maps.newHashMap(); - Map newRoles = Maps.newHashMap(); + Map newRoles = Maps.newHashMap(); for (Component component : dataset.getDataStructure().values()) { String newName; @@ -100,40 +107,41 @@ public AbstractJoinOperation(Map namedDatasets) { newNames.put(component.getName(), newName); newRoles.put(component.getName(), component.getRole()); } - this.datasets.put(datasetName, new RenameOperation(dataset, newNames, newRoles)); + //datasets.put(datasetName, new RenameOperation(dataset, newNames, newRoles)); } - + return datasets; } - Map getDatasets() { - return datasets; + private Bindings createJoinScope(Map namedDatasets, Set commonIdentifiers) { + Bindings bindings = new SimpleBindings(); + namedDatasets.forEach(bindings::put); + for (Map.Entry dataset : namedDatasets.entrySet()) { + + Collection datasetComponents = dataset.getValue().getDataStructure().values(); + for (Component component : datasetComponents) { + bindings.put(component.getName(), component); + } + } + commonIdentifiers.forEach(component -> bindings.put(component.getName(), component)); + return bindings; } - Set getIds() { + public Set getCommonIdentifierNames() { return commonIdentifierNames; } - public List getClauses() { - return clauses; + public Bindings getJoinScope() { + return joinScope; } - abstract WorkingDataset workDataset(); + public abstract WorkingDataset workDataset(); - private WorkingDataset applyClauses() { - WorkingDataset dataset = workDataset(); - for (JoinClause clause : clauses) { - dataset = clause.apply(dataset); - } - return dataset; - } - @Override - public Stream get() { - return (workingDataset = (workingDataset == null ? applyClauses() : workingDataset)).get(); + Map getDatasets() { + return datasets; } - @Override - public DataStructure getDataStructure() { - return (workingDataset = (workingDataset == null ? applyClauses() : workingDataset)).getDataStructure(); + Set getIds() { + return commonIdentifierNames; } /** @@ -153,4 +161,5 @@ protected List delegate() { } } + } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/CrossJoinOperation.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/CrossJoinOperation.java index cb3c1c91..8dbc41f7 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/CrossJoinOperation.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/CrossJoinOperation.java @@ -20,7 +20,6 @@ */ import no.ssb.vtl.model.Dataset; -import no.ssb.vtl.script.operations.join.AbstractJoinOperation; import java.util.Map; @@ -30,7 +29,7 @@ public CrossJoinOperation(Map namedDatasets) { } @Override - WorkingDataset workDataset() { + public WorkingDataset workDataset() { return null; } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/InnerJoinOperation.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/InnerJoinOperation.java index d32810ae..a5559c6e 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/InnerJoinOperation.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/InnerJoinOperation.java @@ -19,12 +19,12 @@ * #L% */ -import com.google.common.collect.Maps; +import com.google.common.base.MoreObjects; +import com.google.common.collect.Sets; import no.ssb.vtl.model.Component; import no.ssb.vtl.model.DataPoint; import no.ssb.vtl.model.DataStructure; import no.ssb.vtl.model.Dataset; -import no.ssb.vtl.script.operations.join.AbstractJoinOperation; import no.ssb.vtl.script.support.JoinSpliterator; import java.util.*; @@ -47,22 +47,37 @@ public InnerJoinOperation(Map namedDatasets) { } @Override - WorkingDataset workDataset() { + public WorkingDataset workDataset() { return new WorkingDataset() { + + @Override + public String toString() { + MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); + return helper.toString(); + } + @Override public DataStructure getDataStructure() { - Map newComponents = Maps.newHashMap(); Map datasets = getDatasets(); + Set ids = Sets.newHashSet(); + + DataStructure.Builder newDataStructure = DataStructure.builder(); for (String datasetName : datasets.keySet()) { DataStructure structure = datasets.get(datasetName).getDataStructure(); - for (String componentName : structure.keySet()) { - newComponents.put(componentName, structure.get(componentName)); + for (Map.Entry componentEntry : structure.entrySet()) { + if (!componentEntry.getValue().isIdentifier()) { + newDataStructure.put(datasetName.concat("_".concat(componentEntry.getKey())), componentEntry.getValue()); + } else { + if (ids.add(componentEntry.getKey())) { + newDataStructure.put(componentEntry); + } + } } } - return DataStructure.copyOf((o, aClass) -> o, newComponents); + return newDataStructure.build(); } @Override @@ -105,17 +120,15 @@ public Object get() { while (iterator.hasNext()) { Function> keyExtractor = tuple -> { // Filter by common ids. - List ids = tuple.stream().filter(dataPoint -> + return tuple.stream().filter(dataPoint -> getCommonIdentifierNames().contains(dataPoint.getName()) ).collect(Collectors.toList()); - return ids; }; Function> joinKeyExtractor = tuple -> { // Filter by common ids. - List ids = tuple.stream().filter(dataPoint -> + return tuple.stream().filter(dataPoint -> getCommonIdentifierNames().contains(dataPoint.getName()) ).collect(Collectors.toList()); - return ids; }; result = StreamSupport.stream( new JoinSpliterator<>( @@ -133,41 +146,34 @@ public Object get() { }; } - private BiFunction getMerger() { - return new BiFunction() { - - @Override - public JoinTuple apply(JoinTuple joinTuple, Tuple components) { - joinTuple.addAll(components.values()); - return joinTuple; - } + private BiFunction getMerger() { + return (joinTuple, components) -> { + joinTuple.addAll(components.values()); + return joinTuple; }; } private Comparator> getKeyComparator(final Set dimensions) { - return new Comparator>() { - @Override - public int compare(List l, List r) { - // TODO: Tuple should expose method to handle this. - // TODO: Evaluate migrating to DataPoint. - // TODO: When using on, the left over identifiers should be transformed to measures. - Map lIds = l.stream() - .collect(Collectors.toMap( - DataPoint::getName, - t -> (Comparable) t.get() - )); - Map rIds = r.stream() - .collect(Collectors.toMap( - DataPoint::getName, - Supplier::get - )); - for (String key : dimensions) { - int res = lIds.get(key).compareTo(rIds.get(key)); - if (res != 0) - return res; - } - return 0; + return (l, r) -> { + // TODO: Tuple should expose method to handle this. + // TODO: Evaluate migrating to DataPoint. + // TODO: When using on, the left over identifiers should be transformed to measures. + Map lIds = l.stream() + .collect(Collectors.toMap( + DataPoint::getName, + t -> (Comparable) t.get() + )); + Map rIds = r.stream() + .collect(Collectors.toMap( + DataPoint::getName, + Supplier::get + )); + for (String key : dimensions) { + int res = lIds.get(key).compareTo(rIds.get(key)); + if (res != 0) + return res; } + return 0; }; } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/JoinScopeBindings.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/JoinScopeBindings.java new file mode 100644 index 00000000..bb036e9c --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/JoinScopeBindings.java @@ -0,0 +1,147 @@ +package no.ssb.vtl.script.operations.join; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.DataStructure; +import no.ssb.vtl.model.Dataset; + +import javax.script.Bindings; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * A special kind of scope that exposes a flattened view of the component + * of the datasets that is used to create join scopes. + */ +final class JoinScopeBindings implements Bindings { + + private final ImmutableSet datasetNames; + private final Map scope = Maps.newHashMap(); + + JoinScopeBindings(Map datasets) { + checkNotNull(datasets); + this.datasetNames = ImmutableSet.copyOf(datasets.keySet()); + //this.scope.putAll(datasets); + + // Needed so that all id are the same reference. + // TODO: :´( ... + Map cleanedDatasets = Maps.newLinkedHashMap(); + Map identifiers = Maps.newLinkedHashMap(); + for (Entry datasetEntry : datasets.entrySet()) { + Dataset dataset = datasetEntry.getValue(); + DataStructure structure = dataset.getDataStructure(); + Map measuresAndAttributes = Maps.newLinkedHashMap(); + for (Entry componentEntry : structure.entrySet()) { + String name = componentEntry.getKey(); + Component component = componentEntry.getValue(); + if (component.isIdentifier()) { + identifiers.putIfAbsent(name, component); + } else { + measuresAndAttributes.put(name, component); + } + } + LinkedHashMap newStructure = Maps.newLinkedHashMap(); + newStructure.putAll(identifiers); + newStructure.putAll(measuresAndAttributes); + + DataStructure finalStructure = DataStructure.copyOf( + newStructure + ).build(); + cleanedDatasets.put(datasetEntry.getKey(), new Dataset() { + @Override + public DataStructure getDataStructure() { + return finalStructure; + } + + @Override + public Stream get() { + throw new UnsupportedOperationException("TODO"); + } + }); + } + + this.scope.putAll(cleanedDatasets); + for (Entry datasetEntry : cleanedDatasets.entrySet()) { + Dataset dataset = datasetEntry.getValue(); + for (Entry componentEntry : dataset.getDataStructure().entrySet()) { + this.scope.putIfAbsent(componentEntry.getKey(), componentEntry.getValue()); + } + } + } + + @Override + public void clear() { + scope.clear(); + } + + @Override + public Set keySet() { + return scope.keySet(); + } + + @Override + public Collection values() { + return scope.values(); + } + + @Override + public Set> entrySet() { + return scope.entrySet(); + } + + @Override + public int size() { + return scope.size(); + } + + @Override + public boolean isEmpty() { + return scope.isEmpty(); + } + + + @Override + public Object put(String name, Object value) { + checkArgument(!datasetNames.contains(name), "could not add [%s] to the scope", name); + return scope.put(name, value); + } + + @Override + public void putAll(Map toMerge) { + Set clashingNames = Sets.intersection(toMerge.keySet(), datasetNames); + checkArgument( + clashingNames.isEmpty(), + "could not add [%s] to the scope", clashingNames + ); + scope.putAll(toMerge); + } + + @Override + public boolean containsKey(Object key) { + return scope.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + return scope.containsValue(value); + } + + @Override + public Object get(Object key) { + return scope.get(key); + } + + @Override + public Object remove(Object key) { + checkArgument(!datasetNames.contains(key), "could not remove [%s] from the scope", key); + return scope.remove(key); + } +} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/OuterJoinOperation.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/OuterJoinOperation.java index 80f57e9a..8c92cadb 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/OuterJoinOperation.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/OuterJoinOperation.java @@ -24,9 +24,6 @@ import java.util.Map; -/** - * Created by hadrien on 16/11/2016. - */ public class OuterJoinOperation extends AbstractJoinOperation { public OuterJoinOperation(Map namedDatasets) { @@ -34,7 +31,7 @@ public OuterJoinOperation(Map namedDatasets) { } @Override - WorkingDataset workDataset() { + public WorkingDataset workDataset() { return null; } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/WorkingDataset.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/WorkingDataset.java index 61e7bc48..193637d9 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/WorkingDataset.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/operations/join/WorkingDataset.java @@ -3,7 +3,7 @@ import no.ssb.vtl.model.Dataset; /** - * The working dataset is used by the join operation + * The working dataset is a special dataset used by the join clauses. */ public interface WorkingDataset extends Dataset { } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/AssignmentVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/AssignmentVisitor.java index b3c590d4..8688849a 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/AssignmentVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/AssignmentVisitor.java @@ -33,7 +33,7 @@ import static com.google.common.base.Preconditions.checkNotNull; /** - * Assignement visitor. + * Assignment visitor. */ public class AssignmentVisitor extends VTLBaseVisitor { @@ -56,7 +56,7 @@ protected Dataset aggregateResult(Dataset aggregate, Dataset nextResult) { @Override public Dataset visitStatement(VTLParser.StatementContext ctx) { - String name = ctx.variableRef().getText(); + String name = ctx.variableID().getText(); Dataset dataset = visit(ctx.datasetExpression()); context.setAttribute(name, dataset, ScriptContext.ENGINE_SCOPE); return (Dataset) context.getAttribute(name, ScriptContext.ENGINE_SCOPE); diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ClauseVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ClauseVisitor.java index 9b101498..39da3ca5 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ClauseVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ClauseVisitor.java @@ -21,11 +21,11 @@ */ import com.google.common.collect.ImmutableMap; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.Dataset; import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; import no.ssb.vtl.script.operations.RenameOperation; -import no.ssb.vtl.model.Dataset; -import no.ssb.vtl.model.Component; import java.util.List; import java.util.Optional; @@ -51,36 +51,40 @@ protected Function aggregateResult(Function @Override public Function visitRenameClause(VTLParser.RenameClauseContext ctx) { - List parameters = ctx.renameParam(); + return dataset -> { + ReferenceVisitor visitor = new ReferenceVisitor(dataset.getDataStructure()); + + List parameters = ctx.renameParam(); - ImmutableMap.Builder names = ImmutableMap.builder(); - ImmutableMap.Builder roles = ImmutableMap.builder(); + ImmutableMap.Builder names = ImmutableMap.builder(); + ImmutableMap.Builder roles = ImmutableMap.builder(); - for (VTLParser.RenameParamContext parameter : parameters) { - String from = parameter.from.getText(); - String to = parameter.to.getText(); - names.put(from, to); + for (VTLParser.RenameParamContext parameter : parameters) { + Component from = (Component) visitor.visit(parameter.from); + String to = parameter.to.getText(); + names.put(from, to); - Optional role = ofNullable(parameter.role()).map(VTLParser.RoleContext::getText); - if (role.isPresent()) { - Component.Role roleEnum; - switch (role.get()) { - case "IDENTIFIER": - roleEnum = Component.Role.IDENTIFIER; - break; - case "MEASURE": - roleEnum = Component.Role.MEASURE; - break; - case "ATTRIBUTE": - roleEnum = Component.Role.ATTRIBUTE; - break; - default: - throw new RuntimeException("unknown component type " + role.get()); + Optional role = ofNullable(parameter.role()).map(VTLParser.RoleContext::getText); + if (role.isPresent()) { + Component.Role roleEnum; + switch (role.get()) { + case "IDENTIFIER": + roleEnum = Component.Role.IDENTIFIER; + break; + case "MEASURE": + roleEnum = Component.Role.MEASURE; + break; + case "ATTRIBUTE": + roleEnum = Component.Role.ATTRIBUTE; + break; + default: + throw new RuntimeException("unknown component type " + role.get()); + } + roles.put(from, roleEnum); } - roles.put(from, roleEnum); } - } - return dataset -> new RenameOperation(dataset, names.build(), roles.build()); + return new RenameOperation(dataset, names.build(), roles.build()); + }; } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ReferenceVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ReferenceVisitor.java new file mode 100644 index 00000000..a846dee4 --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ReferenceVisitor.java @@ -0,0 +1,110 @@ +package no.ssb.vtl.script.visitors; + +import com.google.common.collect.Queues; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.Dataset; +import no.ssb.vtl.parser.VTLBaseVisitor; +import no.ssb.vtl.parser.VTLParser; +import org.antlr.v4.runtime.misc.ParseCancellationException; + +import java.util.Deque; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.lang.String.format; + +/** + * The reference visitor tries to find references to object in a Bindings. + */ +public class ReferenceVisitor extends VTLBaseVisitor { + + private Deque> stack = Queues.newArrayDeque(); + + public ReferenceVisitor(Map scope) { + this.stack.push(checkNotNull(scope, "scope cannot be empty")); + } + + protected ReferenceVisitor() { + } + + private static String removeQuoteIfNeeded(String key) { + if (!key.isEmpty() && key.length() > 3) { + if (key.charAt(0) == '\'' && key.charAt(key.length() - 1) == '\'') { + return key.substring(1, key.length() - 1); + } + } + return key; + } + + private static T checkFound(String expression, T instance) { + if (instance != null) { + return instance; + } + throw new ParseCancellationException( + format( + "variable [%s] not found", + expression + ) + ); + } + + private static T checkType(String expression, Object instance, Class clazz) { + if (clazz.isAssignableFrom(instance.getClass())) { + //noinspection unchecked + return (T) instance; + } + throw new ParseCancellationException( + format( + "wrong type for [%s], expected %s, got %s", + expression, + clazz, + instance.getClass() + ) + ); + } + + @Override + public Object visitVariableRef(VTLParser.VariableRefContext ctx) { + // TODO: Would be nice to handle quote removal in ANTLR + String key = removeQuoteIfNeeded(ctx.identifier().getText()); + return checkFound(ctx.getText(), stack.peek().get(key)); + } + + /** + * {@inheritDoc} + *

+ *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ * + * @param ctx + */ + @Override + public Object visitComponentRef(VTLParser.ComponentRefContext ctx) { + // Ensure data type component. + Component component; + if (ctx.datasetRef() != null) { + Dataset ds = (Dataset) visit(ctx.datasetRef()); + this.stack.push(ds.getDataStructure()); + component = checkType(ctx.getText(), visit(ctx.variableRef()), Component.class); + this.stack.pop(); + } else { + component = checkType(ctx.getText(), visit(ctx.variableRef()), Component.class); + } + return component; + } + + /** + * {@inheritDoc} + *

+ *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ * + * @param ctx + */ + @Override + public Object visitDatasetRef(VTLParser.DatasetRefContext ctx) { + // Ensure data type dataset. + Dataset dataset = checkType(ctx.getText(), visit(ctx.variableRef()), Dataset.class); + return dataset; + } +} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/RelationalVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/RelationalVisitor.java index 008d7845..373138aa 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/RelationalVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/RelationalVisitor.java @@ -5,25 +5,25 @@ import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; import no.ssb.vtl.script.operations.UnionOperation; -import no.ssb.vtl.script.visitors.join.JoinDefinitionVisitor; +import no.ssb.vtl.script.visitors.join.JoinExpressionVisitor; import javax.script.ScriptContext; import java.util.List; import java.util.function.Supplier; -import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.*; /** * A visitor that handles the relational operators. */ public class RelationalVisitor extends VTLBaseVisitor> { - final AssignmentVisitor assignmentVisitor; - final JoinDefinitionVisitor joinVisitor; + private final AssignmentVisitor assignmentVisitor; + private final JoinExpressionVisitor joinVisitor; public RelationalVisitor(AssignmentVisitor assignmentVisitor, ScriptContext context) { this.assignmentVisitor = checkNotNull(assignmentVisitor); - this.joinVisitor = new JoinDefinitionVisitor(context); + this.joinVisitor = new JoinExpressionVisitor(context); } @@ -43,8 +43,4 @@ public Supplier visitJoinExpression(VTLParser.JoinExpressionContext ctx return () -> visit; } - @Override - public Supplier visitRelationalExpression(VTLParser.RelationalExpressionContext ctx) { - return super.visitRelationalExpression(ctx); - } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinBodyVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinBodyVisitor.java deleted file mode 100644 index 58e88abe..00000000 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinBodyVisitor.java +++ /dev/null @@ -1,192 +0,0 @@ -package no.ssb.vtl.script.visitors.join; - -import no.ssb.vtl.model.Component; -import no.ssb.vtl.model.DataStructure; -import no.ssb.vtl.model.Dataset; -import no.ssb.vtl.parser.VTLBaseVisitor; -import no.ssb.vtl.parser.VTLParser; -import no.ssb.vtl.script.operations.DropOperator; -import no.ssb.vtl.script.operations.FilterOperator; -import no.ssb.vtl.script.operations.KeepOperator; -import no.ssb.vtl.script.operations.RenameOperation; -import no.ssb.vtl.script.operations.join.AbstractJoinOperation; -import no.ssb.vtl.script.operations.join.JoinClause; -import no.ssb.vtl.script.operations.join.WorkingDataset; -import org.antlr.v4.runtime.RuleContext; - -import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Stream; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Sets up the join clauses in the given {@link AbstractJoinOperation}. - *

- * The last join clause is returned. - */ -public class JoinBodyVisitor extends VTLBaseVisitor { - - private final AbstractJoinOperation joinOperation; - - public JoinBodyVisitor(AbstractJoinOperation joinOperation) { - this.joinOperation = checkNotNull(joinOperation); - } - - @Override - public JoinClause visitJoinKeepClause(VTLParser.JoinKeepClauseContext ctx) { - - List clauses = this.joinOperation.getClauses(); - - JoinClause keepClause = new JoinClause() { - - @Override - public WorkingDataset apply(WorkingDataset workingDataset) { - JoinKeepClauseVisitor visitor = new JoinKeepClauseVisitor(workingDataset); - KeepOperator keep = visitor.visit(ctx); - return new WorkingDataset() { - @Override - public DataStructure getDataStructure() { - return keep.getDataStructure(); - } - - @Override - public Stream get() { - return keep.get(); - } - }; - } - }; - - clauses.add(keepClause); - - return keepClause; - - } - - @Override - public JoinClause visitJoinRenameClause(VTLParser.JoinRenameClauseContext ctx) { - List clauses = this.joinOperation.getClauses(); - - JoinClause renameClause = new JoinClause() { - - @Override - public WorkingDataset apply(WorkingDataset workingDataset) { - JoinRenameClauseVisitor visitor = new JoinRenameClauseVisitor(workingDataset); - RenameOperation renameOperator = visitor.visit(ctx); - return new WorkingDataset() { - @Override - public DataStructure getDataStructure() { - return renameOperator.getDataStructure(); - } - - @Override - public Stream get() { - return renameOperator.get(); - } - }; - } - }; - - clauses.add(renameClause); - - return renameClause; - } - - @Override - public JoinClause visitJoinDropClause(VTLParser.JoinDropClauseContext ctx) { - - List clauses = this.joinOperation.getClauses(); - - JoinClause dropClause = new JoinClause() { - - @Override - public WorkingDataset apply(WorkingDataset workingDataset) { - JoinDropClauseVisitor visitor = new JoinDropClauseVisitor(workingDataset); - DropOperator drop = visitor.visit(ctx); - return new WorkingDataset() { - @Override - public DataStructure getDataStructure() { - return drop.getDataStructure(); - } - - @Override - public Stream get() { - return drop.get(); - } - }; - } - }; - - clauses.add(dropClause); - - return dropClause; - } - - @Override - public JoinClause visitJoinCalcClause(VTLParser.JoinCalcClauseContext ctx) { - String variableName = ctx.varID().getText(); - - // TODO: Spec does not specify what is the default role. - String variableRole = Optional.ofNullable(ctx.role()).map(RuleContext::getText).orElse("MEASURE"); - - List clauses = this.joinOperation.getClauses(); - - //DataStructure dataStructure = joinOperation.getDataStructure(); - - JoinCalcClauseVisitor joinCalcClauseVisitor = new JoinCalcClauseVisitor(); - Function clauseFunction = joinCalcClauseVisitor.visit(ctx); - JoinClause calcClause = new JoinClause() { - - @Override - public WorkingDataset apply(WorkingDataset workingDataset) { - return new WorkingDataset() { - @Override - public DataStructure getDataStructure() { - DataStructure structure = workingDataset.getDataStructure(); - structure.addComponent(variableName, Component.Role.MEASURE, Number.class); - return structure; - } - - @Override - public Stream get() { - return workingDataset.get() - .map(tuple -> { - tuple.add(getDataStructure().wrap(variableName, clauseFunction.apply(tuple))); - return tuple; - }); - } - }; - } - }; - - clauses.add(calcClause); - return calcClause; - } - - @Override - public JoinClause visitJoinFilterClause(VTLParser.JoinFilterClauseContext ctx) { - List clauses = this.joinOperation.getClauses(); - - JoinClause filterClause = workingDataset -> { - JoinFilterClauseVisitor visitor = new JoinFilterClauseVisitor(workingDataset); - FilterOperator filter = visitor.visit(ctx); - return new WorkingDataset() { - @Override - public DataStructure getDataStructure() { - return filter.getDataStructure(); - } - - @Override - public Stream get() { - return filter.get(); - } - }; - }; - - clauses.add(filterClause); - - return filterClause; - } -} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinCalcClauseVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinCalcClauseVisitor.java index 0aef927b..5f9597e8 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinCalcClauseVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinCalcClauseVisitor.java @@ -1,9 +1,11 @@ package no.ssb.vtl.script.visitors.join; +import no.ssb.vtl.model.Component; import no.ssb.vtl.model.DataPoint; import no.ssb.vtl.model.Dataset; import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; +import no.ssb.vtl.script.visitors.ReferenceVisitor; import java.util.function.Function; @@ -14,39 +16,32 @@ */ public class JoinCalcClauseVisitor extends VTLBaseVisitor> { + private final ReferenceVisitor referenceVisitor; + + public JoinCalcClauseVisitor(ReferenceVisitor referenceVisitor) { + this.referenceVisitor = referenceVisitor; + } + @Override public Function visitJoinCalcReference(VTLParser.JoinCalcReferenceContext ctx) { - String variableName = ctx.getText(); - return new Function() { - @Override - public Object apply(Dataset.Tuple tuple) { - for (DataPoint dataPoint : tuple) { - if (variableName.equals(dataPoint.getName())) { - return dataPoint.get(); - } + Component component = (Component) referenceVisitor.visit(ctx.componentRef()); + return tuple -> { + for (DataPoint dataPoint : tuple) { + if (component == dataPoint.getComponent()) { + return dataPoint.get(); } - throw new RuntimeException(format("variable %s not found", variableName)); } + throw new RuntimeException(format("component %s not found in %s", component, tuple)); }; } - @Override - public Function visitJoinCalcRef(VTLParser.JoinCalcRefContext ctx) { - return super.visitJoinCalcRef(ctx); - } - - @Override public Function visitJoinCalcAtom(VTLParser.JoinCalcAtomContext ctx) { VTLParser.ConstantContext constantValue = ctx.constant(); if (constantValue.FLOAT_CONSTANT() != null) - return tuple -> { - return Float.valueOf(constantValue.FLOAT_CONSTANT().getText()); - }; + return tuple -> Float.valueOf(constantValue.FLOAT_CONSTANT().getText()); if (constantValue.INTEGER_CONSTANT() != null) - return tuple -> { - return Integer.valueOf(constantValue.INTEGER_CONSTANT().getText()); - }; + return tuple -> Integer.valueOf(constantValue.INTEGER_CONSTANT().getText()); throw new RuntimeException( format("unsuported constant type %s", constantValue) diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinDefinitionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinDefinitionVisitor.java index 88a16a64..01a01189 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinDefinitionVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinDefinitionVisitor.java @@ -1,92 +1,59 @@ package no.ssb.vtl.script.visitors.join; -import com.google.common.collect.Maps; import no.ssb.vtl.model.Dataset; import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; -import no.ssb.vtl.script.operations.join.*; +import no.ssb.vtl.script.operations.join.AbstractJoinOperation; +import no.ssb.vtl.script.operations.join.CrossJoinOperation; +import no.ssb.vtl.script.operations.join.InnerJoinOperation; +import no.ssb.vtl.script.operations.join.OuterJoinOperation; +import no.ssb.vtl.script.visitors.ReferenceVisitor; +import org.antlr.v4.runtime.RuleContext; -import javax.script.Bindings; import javax.script.ScriptContext; -import java.util.List; import java.util.Map; +import java.util.stream.Collectors; -import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.*; /** * Visitor that handle the join definition */ public class JoinDefinitionVisitor extends VTLBaseVisitor { - private final ScriptContext context; - + private final ReferenceVisitor referenceVisitor; + public JoinDefinitionVisitor(ScriptContext context) { - this.context = checkNotNull(context); - } - - @Override - public AbstractJoinOperation visitJoinExpression(VTLParser.JoinExpressionContext ctx) { - AbstractJoinOperation joinOperation = visit(ctx.joinDefinition()); - JoinBodyVisitor joinBodyVisitor = new JoinBodyVisitor(joinOperation); - - // TODO: Not used? - JoinClause joinClause = joinBodyVisitor.visitJoinBody(ctx.joinBody()); - - return joinOperation; + checkNotNull(context); + referenceVisitor = new ReferenceVisitor(context.getBindings(ScriptContext.ENGINE_SCOPE)); } @Override public AbstractJoinOperation visitJoinDefinitionInner(VTLParser.JoinDefinitionInnerContext ctx) { - List datasets = ctx.joinParam().varID(); - - Map theDatasets = createJoinScope(datasets); - - InnerJoinOperation joinOperation = new InnerJoinOperation(theDatasets); - return joinOperation; - + Map theDatasets = getDatasetParameters(ctx.joinParam()); + return new InnerJoinOperation(theDatasets); } - + @Override public AbstractJoinOperation visitJoinDefinitionOuter(VTLParser.JoinDefinitionOuterContext ctx) { - List datasets = ctx.joinParam().varID(); - - Map datasetMap = createJoinScope(datasets); - - OuterJoinOperation joinOperation = new OuterJoinOperation(datasetMap); - return joinOperation; + Map datasetMap = getDatasetParameters(ctx.joinParam()); + return new OuterJoinOperation(datasetMap); } - + @Override public AbstractJoinOperation visitJoinDefinitionCross(VTLParser.JoinDefinitionCrossContext ctx) { - List datasets = ctx.joinParam().varID(); - - Map datasetMap = createJoinScope(datasets); - - CrossJoinOperation joinOperation = new CrossJoinOperation(datasetMap); - return joinOperation; + Map datasetMap = getDatasetParameters(ctx.joinParam()); + return new CrossJoinOperation(datasetMap); } - - - /** - * Finds the datasets in the context. - */ - private Map createJoinScope(List names) { - Map datasets = Maps.newHashMap(); - Bindings bindings = context.getBindings(ScriptContext.ENGINE_SCOPE); - for (VTLParser.VarIDContext dataset : names) { - String datasetName = dataset.getText(); - if (!bindings.containsKey(datasetName)) { - // TODO: Exception, invalid type. - throw new RuntimeException(datasetName + " does not exist"); - } - Object datasetVariable = bindings.get(datasetName); - if (!(datasetVariable instanceof Dataset)) { - // TODO: Exception, invalid type. - throw new RuntimeException(datasetName + " was not a dataset"); - } - datasets.put(datasetName, (Dataset) datasetVariable); - } - return datasets; + + Map getDatasetParameters(VTLParser.JoinParamContext ctx) { + return ctx.datasetRef().stream() + .collect(Collectors.toMap(RuleContext::getText, this::getDataset)); } - + + private Dataset getDataset(VTLParser.DatasetRefContext ref) { + Object referencedObject = referenceVisitor.visit(ref); + return (Dataset) referencedObject; //TODO: Is this always safe? Hadrien: Yes, DatasetRefContext and ComponentRefContext will return the correct type + } + } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinDropClauseVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinDropClauseVisitor.java index b01609cc..a68bf408 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinDropClauseVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinDropClauseVisitor.java @@ -1,14 +1,17 @@ package no.ssb.vtl.script.visitors.join; -import com.google.common.collect.Sets; +import no.ssb.vtl.model.Component; import no.ssb.vtl.model.Dataset; import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; import no.ssb.vtl.script.operations.DropOperator; +import no.ssb.vtl.script.operations.join.WorkingDataset; +import no.ssb.vtl.script.visitors.ReferenceVisitor; import java.util.Set; +import java.util.stream.Collectors; -import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.*; /** * Visit the drop clauses. @@ -16,17 +19,25 @@ public class JoinDropClauseVisitor extends VTLBaseVisitor { private final Dataset dataset; + private final ReferenceVisitor referenceVisitor; - public JoinDropClauseVisitor(Dataset dataset) { + @Deprecated + public JoinDropClauseVisitor(WorkingDataset dataset) { this.dataset = checkNotNull(dataset, "dataset was null"); + this.referenceVisitor = null; + } + + public JoinDropClauseVisitor(Dataset dataset, ReferenceVisitor referenceVisitor) { + this.dataset = checkNotNull(dataset); + this.referenceVisitor = checkNotNull(referenceVisitor); } @Override public DropOperator visitJoinDropExpression(VTLParser.JoinDropExpressionContext ctx) { - Set components = Sets.newHashSet(); - for (VTLParser.JoinDropRefContext joinDropRefContext : ctx.joinDropRef()) { - components.add(joinDropRefContext.getText()); - } + Set components = ctx.componentRef().stream() + .map(referenceVisitor::visit) + .map(o -> (Component) o) //TODO: Safe? + .collect(Collectors.toSet()); return new DropOperator(dataset, components); } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinExpressionVisitor.java new file mode 100644 index 00000000..3c82200e --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinExpressionVisitor.java @@ -0,0 +1,177 @@ +package no.ssb.vtl.script.visitors.join; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ForwardingMap; +import com.google.common.collect.Maps; +import com.google.common.collect.Queues; +import com.google.common.collect.Sets; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.DataStructure; +import no.ssb.vtl.model.Dataset; +import no.ssb.vtl.parser.VTLBaseVisitor; +import no.ssb.vtl.parser.VTLParser; +import no.ssb.vtl.script.operations.join.AbstractJoinOperation; +import no.ssb.vtl.script.visitors.ReferenceVisitor; + +import javax.script.Bindings; +import javax.script.ScriptContext; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Stream; + +import static com.google.common.base.MoreObjects.firstNonNull; + +public class JoinExpressionVisitor extends VTLBaseVisitor { + + private final JoinDefinitionVisitor joinDefVisitor; + private ReferenceVisitor referenceVisitor; + private Dataset workingDataset; + private Bindings joinScope; + + public JoinExpressionVisitor(ScriptContext context) { + joinDefVisitor = new JoinDefinitionVisitor(context); + } + + @Override + public Dataset visitJoinExpression(VTLParser.JoinExpressionContext ctx) { + AbstractJoinOperation joinOperation = joinDefVisitor.visit(ctx.joinDefinition()); + joinScope = joinOperation.getJoinScope(); + + workingDataset = joinOperation.workDataset(); + referenceVisitor = new ReferenceVisitor(joinScope); + + Dataset finalDataset = visit(ctx.joinBody()); + + // TODO: Put somewhere else. + IdentityHashMap identityMap = Maps.newIdentityHashMap(); + Deque> stack = Queues.newArrayDeque(); + stack.addAll(joinScope.entrySet()); + int lastSize = -1; + while (!stack.isEmpty()) { + Map.Entry entry = stack.pop(); + Object value = entry.getValue(); + if (value instanceof Dataset) { + if (lastSize == identityMap.size()) { + Dataset dataset = (Dataset) value; + stack.addAll(dataset.getDataStructure().entrySet()); + } else { + lastSize = identityMap.size(); + stack.addLast(entry); + } + } else if (value instanceof Component) { + identityMap.put((Component) entry.getValue(), entry.getKey()); + } + } + + + + return finalDataset; +// JoinBodyVisitor joinBodyVisitor = new JoinBodyVisitor(joinScope); +// Function joinClause = joinBodyVisitor.visitJoinBody(ctx.joinBody()); +// WorkingDataset workingDataset = joinOperation.workDataset(); +// +// return joinClause.apply(workingDataset); + } + + @Override + public Dataset visitJoinCalcClause(VTLParser.JoinCalcClauseContext ctx) { + /* + TODO: Handle explicit and implicit component computation. + Need to parse the role + If implicit, error if already defined. + */ + String variableName = ctx.variableID().getText(); + Component.Role role = Component.Role.MEASURE; + Class type = Number.class; + + + DataStructure.Builder structureCopy = DataStructure.copyOf(workingDataset.getDataStructure()); + structureCopy.put(variableName, role, type); + JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(referenceVisitor); + Function componentExpression = visitor.visit(ctx); + + // TODO: Extract to its own visitor implementing dataset. + Dataset dataset = workingDataset; + DataStructure dataStructure = structureCopy.build(); + return new Dataset() { + @Override + public DataStructure getDataStructure() { + return dataStructure; + } + + @Override + public Stream get() { + return dataset.get().map(tuple -> { + tuple.add(dataStructure.wrap(variableName, componentExpression.apply(tuple))); + return tuple; + }); + } + + @Override + public String toString() { + MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper("Calc"); + return helper.omitNullValues().toString(); + } + }; + } + + @Override + protected Dataset aggregateResult(Dataset aggregate, Dataset nextResult) { + // Compute the new scope. + Dataset currentDataset = firstNonNull(nextResult, aggregate); + + Set previous = Optional.ofNullable(aggregate) + .map(Dataset::getDataStructure) + .map(ForwardingMap::keySet) + .orElse(Collections.emptySet()); + Set current = currentDataset.getDataStructure().keySet(); + + Set referencesToRemove = Sets.difference(previous, current); + Set referencesToAdd = Sets.difference(current, previous); + + for (String key : referencesToRemove) { + joinScope.remove(key); + } + for (String key : referencesToAdd) { + joinScope.put(key, currentDataset.getDataStructure().get(key)); + } + + return workingDataset = currentDataset; + } + + @Override + public Dataset visitJoinFoldClause(VTLParser.JoinFoldClauseContext ctx) { + JoinFoldClauseVisitor visitor = new JoinFoldClauseVisitor(workingDataset, referenceVisitor); + return visitor.visit(ctx); + } + + @Override + public Dataset visitJoinUnfoldClause(VTLParser.JoinUnfoldClauseContext ctx) { + JoinUnfoldClauseVisitor visitor = new JoinUnfoldClauseVisitor(workingDataset, referenceVisitor); + return visitor.visit(ctx); + } + + @Override + public Dataset visitJoinKeepClause(VTLParser.JoinKeepClauseContext ctx) { + JoinKeepClauseVisitor visitor = new JoinKeepClauseVisitor(workingDataset, referenceVisitor); + return visitor.visit(ctx); + } + + @Override + public Dataset visitJoinDropClause(VTLParser.JoinDropClauseContext ctx) { + JoinDropClauseVisitor visitor = new JoinDropClauseVisitor(workingDataset, referenceVisitor); + return visitor.visit(ctx); + } + + @Override + public Dataset visitJoinFilterClause(VTLParser.JoinFilterClauseContext ctx) { + JoinFilterClauseVisitor visitor = new JoinFilterClauseVisitor(workingDataset, referenceVisitor); + return visitor.visit(ctx); + } + + @Override + public Dataset visitJoinRenameClause(VTLParser.JoinRenameClauseContext ctx) { + JoinRenameClauseVisitor visitor = new JoinRenameClauseVisitor(workingDataset, referenceVisitor); + return visitor.visit(ctx); + } +} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitor.java index 21318dc4..52f3c6da 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFilterClauseVisitor.java @@ -4,6 +4,7 @@ import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; import no.ssb.vtl.script.operations.FilterOperator; +import no.ssb.vtl.script.visitors.ReferenceVisitor; import java.util.Set; import java.util.function.Predicate; @@ -19,7 +20,11 @@ public class JoinFilterClauseVisitor extends VTLBaseVisitor { JoinFilterClauseVisitor(Dataset dataset) { this.dataset = checkNotNull(dataset, "dataset was null"); } - + + public JoinFilterClauseVisitor(Dataset dataset, ReferenceVisitor referenceVisitor) { + this.dataset = checkNotNull(dataset, "dataset was null"); + } + @Override public FilterOperator visitJoinFilterClause(VTLParser.JoinFilterClauseContext ctx) { Set components = Stream.of("id1").collect(Collectors.toSet()); diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFoldClauseVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFoldClauseVisitor.java new file mode 100644 index 00000000..d26c0225 --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinFoldClauseVisitor.java @@ -0,0 +1,38 @@ +package no.ssb.vtl.script.visitors.join; + +import me.yanaga.guava.stream.MoreCollectors; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.Dataset; +import no.ssb.vtl.parser.VTLBaseVisitor; +import no.ssb.vtl.parser.VTLParser; +import no.ssb.vtl.script.operations.FoldClause; +import no.ssb.vtl.script.visitors.ReferenceVisitor; + +import java.util.Set; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class JoinFoldClauseVisitor extends VTLBaseVisitor { + + private final Dataset dataset; + private final ReferenceVisitor referenceVisitor; + + public JoinFoldClauseVisitor(Dataset dataset, ReferenceVisitor referenceVisitor) { + this.dataset = checkNotNull(dataset); + this.referenceVisitor = checkNotNull(referenceVisitor); + } + + @Override + public FoldClause visitJoinFoldExpression(VTLParser.JoinFoldExpressionContext ctx) { + // TODO: Migrate to component type. + String dimension = ctx.dimension.getText(); + String measure = ctx.measure.getText(); + + Set elements = ctx.elements.componentRef().stream() + .map(referenceVisitor::visitComponentRef) + .map(o -> (Component) o) + .collect(MoreCollectors.toImmutableSet()); + + return new FoldClause(dataset, dimension, measure, elements); + } +} diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinKeepClauseVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinKeepClauseVisitor.java index e1594c7b..90a8d1e4 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinKeepClauseVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinKeepClauseVisitor.java @@ -1,12 +1,17 @@ package no.ssb.vtl.script.visitors.join; -import com.google.common.collect.ImmutableSet; +import no.ssb.vtl.model.Component; import no.ssb.vtl.model.Dataset; import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; import no.ssb.vtl.script.operations.KeepOperator; +import no.ssb.vtl.script.operations.join.WorkingDataset; +import no.ssb.vtl.script.visitors.ReferenceVisitor; -import static com.google.common.base.Preconditions.checkNotNull; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.*; /** * Visit the keep clauses. @@ -14,17 +19,25 @@ public class JoinKeepClauseVisitor extends VTLBaseVisitor { private final Dataset dataset; + private final ReferenceVisitor referenceVisitor; + + @Deprecated + public JoinKeepClauseVisitor(WorkingDataset dataset) { + this.dataset = checkNotNull(dataset); + referenceVisitor = null; + } - public JoinKeepClauseVisitor(Dataset dataset) { + public JoinKeepClauseVisitor(Dataset dataset, ReferenceVisitor referenceVisitor) { this.dataset = checkNotNull(dataset); + this.referenceVisitor = checkNotNull(referenceVisitor); } @Override public KeepOperator visitJoinKeepExpression(VTLParser.JoinKeepExpressionContext ctx) { - ImmutableSet.Builder components = ImmutableSet.builder(); - for (VTLParser.JoinKeepRefContext joidKeepRefContext : ctx.joinKeepRef()) { - components.add(joidKeepRefContext.getText()); - } - return new KeepOperator(dataset, components.build()); + Set components = ctx.componentRef().stream() + .map(referenceVisitor::visit) + .map(o -> (Component) o) //TODO: Safe? Yup! + .collect(Collectors.toSet()); + return new KeepOperator(dataset, components); } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinRenameClauseVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinRenameClauseVisitor.java index b62dcdaf..2ad5c8c7 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinRenameClauseVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinRenameClauseVisitor.java @@ -2,14 +2,12 @@ import com.google.common.collect.ImmutableMap; import no.ssb.vtl.model.Component; -import no.ssb.vtl.model.DataStructure; import no.ssb.vtl.model.Dataset; import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; import no.ssb.vtl.script.operations.RenameOperation; -import no.ssb.vtl.script.operations.join.WorkingDataset; +import no.ssb.vtl.script.visitors.ReferenceVisitor; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; /** @@ -18,27 +16,26 @@ public class JoinRenameClauseVisitor extends VTLBaseVisitor { private final Dataset dataset; + private final ReferenceVisitor referenceVisitor; - public JoinRenameClauseVisitor(WorkingDataset dataset) { + @Deprecated + public JoinRenameClauseVisitor(Dataset dataset) { + this.dataset = checkNotNull(dataset); this.referenceVisitor = null; + } + + public JoinRenameClauseVisitor(Dataset dataset, ReferenceVisitor referenceVisitor) { this.dataset = checkNotNull(dataset); + this.referenceVisitor = checkNotNull(referenceVisitor); } @Override public RenameOperation visitJoinRenameExpression(VTLParser.JoinRenameExpressionContext ctx) { - DataStructure dataStructure = dataset.getDataStructure(); - ImmutableMap.Builder newNames = ImmutableMap.builder(); - ImmutableMap.Builder newRoles = ImmutableMap.builder(); + ImmutableMap.Builder newNames = ImmutableMap.builder(); for (VTLParser.JoinRenameParameterContext renameParam : ctx.joinRenameParameter()) { - String from = renameParam.from.getText(); + Component component = (Component) referenceVisitor.visit(renameParam.componentRef()); String to = renameParam.to.getText(); - newNames.put(from, to); - checkArgument( - dataStructure.containsKey(from), - "could not find component with name %s", - from - ); - newRoles.put(from, dataStructure.get(from).getRole()); + newNames.put(component, to); } - return new RenameOperation(dataset, newNames.build(), newRoles.build()); + return new RenameOperation(dataset, newNames.build()); } } diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinUnfoldClauseVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinUnfoldClauseVisitor.java new file mode 100644 index 00000000..0d941d3c --- /dev/null +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinUnfoldClauseVisitor.java @@ -0,0 +1,39 @@ +package no.ssb.vtl.script.visitors.join; + + +import com.google.common.collect.Sets; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.Dataset; +import no.ssb.vtl.parser.VTLBaseVisitor; +import no.ssb.vtl.parser.VTLParser; +import no.ssb.vtl.script.operations.UnfoldClause; +import no.ssb.vtl.script.visitors.ReferenceVisitor; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.Set; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class JoinUnfoldClauseVisitor extends VTLBaseVisitor { + + private final Dataset dataset; + private final ReferenceVisitor referenceVisitor; + + public JoinUnfoldClauseVisitor(Dataset dataset, ReferenceVisitor referenceVisitor) { + this.dataset = checkNotNull(dataset); + this.referenceVisitor = referenceVisitor; + } + + @Override + public UnfoldClause visitJoinUnfoldExpression(VTLParser.JoinUnfoldExpressionContext ctx) { + Component dimension = (Component) referenceVisitor.visit(ctx.dimension); + Component measure = (Component) referenceVisitor.visit(ctx.measure); + + Set elements = Sets.newLinkedHashSet(); + for (TerminalNode element : ctx.elements.STRING_CONSTANT()) { + String constant = element.getText(); + elements.add(constant.substring(1, constant.length()-1)); + } + return new UnfoldClause(dataset, dimension, measure, elements); + } +} diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java index 6d3fe548..c4849aa7 100644 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java @@ -21,6 +21,7 @@ */ import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import no.ssb.vtl.connector.Connector; import no.ssb.vtl.model.DataPoint; import no.ssb.vtl.model.DataStructure; @@ -32,7 +33,9 @@ import javax.script.ScriptContext; import javax.script.ScriptEngine; import java.util.Arrays; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.stream.Stream; import static no.ssb.vtl.model.Component.Role; @@ -90,7 +93,7 @@ public void testJoin() throws Exception { Dataset ds1 = mock(Dataset.class); Dataset ds2 = mock(Dataset.class); - DataStructure ds = DataStructure.of( + DataStructure structure1 = DataStructure.of( (o, aClass) -> o, "id1", Role.IDENTIFIER, String.class, "id2", Role.IDENTIFIER, String.class, @@ -98,40 +101,48 @@ public void testJoin() throws Exception { "m2", Role.MEASURE, Double.class, "at1", Role.MEASURE, String.class ); - when(ds1.getDataStructure()).thenReturn(ds); - when(ds2.getDataStructure()).thenReturn(ds); + DataStructure structure2 = DataStructure.of( + (o, aClass) -> o, + "id1", Role.IDENTIFIER, String.class, + "id2", Role.IDENTIFIER, String.class, + "m1", Role.MEASURE, Integer.class, + "m2", Role.MEASURE, Double.class, + "at2", Role.MEASURE, String.class + ); + when(ds1.getDataStructure()).thenReturn(structure1); + when(ds2.getDataStructure()).thenReturn(structure2); when(ds1.get()).then(invocation -> Stream.of( - ds.wrap(ImmutableMap.of( + structure1.wrap(ImmutableMap.of( "id1", "1", "id2", "1", "m1", 10, "m2", 20, - "at1", "attr1" + "at1", "attr1-1" )), - ds.wrap(ImmutableMap.of( + structure1.wrap(ImmutableMap.of( "id1", "2", "id2", "2", "m1", 100, "m2", 200, - "at1", "attr2" + "at1", "attr1-2" )) )); when(ds2.get()).then(invocation -> Stream.of( - ds.wrap(ImmutableMap.of( + structure2.wrap(ImmutableMap.of( "id1", "1", "id2", "1", "m1", 30, "m2", 40, - "at1", "attr1" + "at2", "attr2-1" )), - ds.wrap(ImmutableMap.of( + structure2.wrap(ImmutableMap.of( "id1", "2", "id2", "2", "m1", 300, "m2", 400, - "at1", "attr2" + "at2", "attr2-2" )) )); @@ -139,12 +150,12 @@ public void testJoin() throws Exception { bindings.put("ds2", ds2); engine.eval("" + - "ds3 := [ds1, ds2]{" + - " filter id1=1," + - " ident = ds1.m1 + ds2.m2 - ds1.m2 - ds2.m1," + - " keep ident, ds1.m1, ds2.m1, ds2.m2," + // id1, id2, ident, ds1.m1, ds2.m1, ds2.m2 - " drop ds2.m1," + // id1, id2, ident, ds1.m1, ds2.m2 - " rename id1 to renamedId1, ds1.m1 to m1" + // renamedId1, id2, ident, m1, ds2.m2 + "ds3 := [ds1, ds2]{" + // id1, id2, ds1.m1, ds1.m2, d2.m1, d2.m2, at1, at2 + " filter id1 = 1," + // id1, id2, ds1.m1, ds1.m2, d2.m1, d2.m2, at1, at2 + " ident = ds1.m1 + ds2.m2 - ds1.m2 - ds2.m1," + // id1, id2, ds1.m1, ds1.m2, d2.m1, d2.m2, at1, at2, ident + " keep ident, ds1.m1, ds2.m1, ds2.m2," + // id1, id2, ds1.m1, ds2.m1, ds2.m2, ident + " drop ds2.m1," + // id1, id2, ds1.m1, ds2.m2, ident + " rename id1 to renamedId1, ds1.m1 to m1, ds2.m2 to m2" + // renamedId1, id2, m1, m2, ident "}" + ""); @@ -153,21 +164,146 @@ public void testJoin() throws Exception { assertThat(bindings.get("ds3")).isInstanceOf(Dataset.class); Dataset ds3 = (Dataset) bindings.get("ds3"); - assertThat(ds3.getDataStructure()).containsOnlyKeys( - "renamedId1", "id2", "ds2.m2", "m1", "ident" + assertThat(ds3.getDataStructure()) + .describedAs("data structure of d3") + .containsOnlyKeys( + "renamedId1", "id2", "m2", "m1", "ident" ); + ListAssert datapoints = assertThat(ds3.get()) .flatExtracting(input -> input); datapoints.extracting(DataPoint::getName).containsExactly( - "renamedId1", "id2", "ds2.m2", "m1", "ident" + "renamedId1", "id2", "m2", "m1", "ident" ); datapoints.extracting(DataPoint::get).containsExactly( "1", "1", 40, 10, 0 ); + } + + @Test + public void testJoinFold() throws Exception { + Dataset ds1 = mock(Dataset.class); + DataStructure ds = DataStructure.of( + (o, aClass) -> o, + "id1", Role.IDENTIFIER, String.class, + "m1", Role.MEASURE, Number.class, + "m2", Role.MEASURE, Number.class, + "m3", Role.MEASURE, Number.class + ); + when(ds1.getDataStructure()).thenReturn(ds); + when(ds1.get()).then(invocation -> Stream.of( + Arrays.asList("1", 101, 102, 103), + Arrays.asList("2", 201, 202, 203), + Arrays.asList("3", 301, 302, 303) + ).map(list -> { + Iterator it = list.iterator(); + List points = Lists.newArrayList(); + for (String name : ds.keySet()) { + Object value = it.hasNext() ? it.next() : null; + points.add(ds.wrap(name, value)); + } + return Dataset.Tuple.create(points); + })); + + bindings.put("ds1", ds1); + engine.eval("ds2 := [ds1] {" + + " total = ds1.m1 + ds1.m2 + ds1.m3," + + " fold ds1.m1, ds1.m2, ds1.m3, total to type, value" + + "}" + ); + + assertThat(bindings).containsKey("ds2"); + Dataset ds2 = (Dataset) bindings.get("ds2"); + + assertThat(ds2.getDataStructure().getRoles()).containsOnly( + entry("id1", Role.IDENTIFIER), + entry("type", Role.IDENTIFIER), + entry("value", Role.MEASURE) + ); + assertThat(ds2.get()).flatExtracting(input -> input) + .extracting(DataPoint::get) + .containsExactly( + "1", "m1", 101, + "1", "m2", 102, + "1", "m3", 103, + "1", "total", 101 + 102 + 103, + + "2", "m1", 201, + "2", "m2", 202, + "2", "m3", 203, + "2", "total", 201 + 202 + 203, + + "3", "m1", 301, + "3", "m2", 302, + "3", "m3", 303, + "3", "total", 301 + 302 + 303 + ); + } + + @Test + public void testJoinUnfold() throws Exception { + Dataset ds1 = mock(Dataset.class); + DataStructure ds = DataStructure.of( + (o, aClass) -> o, + "id1", Role.IDENTIFIER, String.class, + "id2", Role.IDENTIFIER, String.class, + "m1", Role.MEASURE, Integer.class, + "m2", Role.MEASURE, Double.class, + "at1", Role.MEASURE, String.class + ); + when(ds1.getDataStructure()).thenReturn(ds); + when(ds1.get()).then(invocation -> Stream.of( + (Map) ImmutableMap.of( + "id1", "1", + "id2", "one", + "m1", 101, + "at1", "attr1" + ), + ImmutableMap.of( + "id1", "1", + "id2", "two", + "m1", 102, + "at1", "attr2" + ), + ImmutableMap.of( + "id1", "2", + "id2", "one", + "m1", 201, + "at1", "attr2" + ), ImmutableMap.of( + "id1", "2", + "id2", "two", + "m1", 202, + "at1", "attr2" + ) + ).map(ds::wrap)); + + bindings.put("ds1", ds1); + engine.eval("ds2 := [ds1] {" + + " unfold id2, ds1.m1 to \"one\", \"two\"," + + " onePlusTwo = one + two" + + "}" + ); + + assertThat(bindings).containsKey("ds2"); + Dataset ds2 = (Dataset) bindings.get("ds2"); + + assertThat(ds2.getDataStructure().getRoles()).containsOnly( + entry("id1", Role.IDENTIFIER), + entry("one", Role.MEASURE), + entry("two", Role.MEASURE), + entry("onePlusTwo", Role.MEASURE) + ); + assertThat(ds2.get()).flatExtracting(input -> input) + .extracting(DataPoint::get) + .containsExactly( + "1", 101, 102, 101 + 102, + "2", 201, 202, 201 + 202 + ); } @Test diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/FoldClauseTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/FoldClauseTest.java new file mode 100644 index 00000000..74688241 --- /dev/null +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/FoldClauseTest.java @@ -0,0 +1,210 @@ +package no.ssb.vtl.script.operations; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.DataPoint; +import no.ssb.vtl.model.DataStructure; +import no.ssb.vtl.model.Dataset; +import org.assertj.core.api.AutoCloseableSoftAssertions; +import org.junit.Test; + +import java.time.Instant; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static com.google.common.base.Preconditions.checkArgument; +import static no.ssb.vtl.model.Component.Role.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FoldClauseTest { + + private static Dataset.Tuple tuple(DataStructure structure, Object... values) { + checkArgument(values.length == structure.size()); + Map map = Maps.newLinkedHashMap(); + Iterator iterator = Lists.newArrayList(values).iterator(); + for (String name : structure.keySet()) { + map.put(name, iterator.next()); + } + return structure.wrap(map); + } + + @Test + public void testArguments() throws Exception { + + String validDimensionReference = "aDimension"; + String validMeasureReference = "aDimension"; + + DataStructure structure = DataStructure.of((o, aClass) -> o, + "element1", MEASURE, String.class, + "element2", MEASURE, String.class + ); + + Set validElements = Sets.newHashSet(structure.values()); + Dataset dataset = mock(Dataset.class); + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + + softly.assertThatThrownBy(() -> new FoldClause(null, validDimensionReference, validMeasureReference, validElements)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("dataset") + .hasMessageContaining("null"); + + softly.assertThatThrownBy(() -> new FoldClause(dataset, null, validMeasureReference, validElements)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("dimensionReference") + .hasMessageContaining("null"); + + softly.assertThatThrownBy(() -> new FoldClause(dataset, validDimensionReference, null, validElements)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("measureReference") + .hasMessageContaining("null"); + + softly.assertThatThrownBy(() -> new FoldClause(dataset, validDimensionReference, validMeasureReference, null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("elements") + .hasMessageContaining("null"); + + softly.assertThatThrownBy(() -> new FoldClause(dataset, "", validMeasureReference, validElements)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("dimensionReference") + .hasMessageContaining("empty"); + + softly.assertThatThrownBy(() -> new FoldClause(dataset, validDimensionReference, "", validElements)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("measureReference") + .hasMessageContaining("empty"); + + softly.assertThatThrownBy(() -> new FoldClause(dataset, validDimensionReference, validMeasureReference, Collections.emptySet())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("elements") + .hasMessageContaining("empty"); + } + } + + @Test + public void testConstraint() throws Exception { + + Dataset dataset = mock(Dataset.class); + DataStructure structure = DataStructure.of((o, aClass) -> o, + "id1", IDENTIFIER, String.class, + "id2", IDENTIFIER, String.class, + "m1", MEASURE, String.class, + "m2", MEASURE, String.class, + "m3", MEASURE, String.class + ); + Dataset invalidDataset = mock(Dataset.class); + DataStructure wrongTypesDataset = DataStructure.of((o, aClass) -> o, + "id1", IDENTIFIER, String.class, + "id2", IDENTIFIER, String.class, + "m1", MEASURE, Number.class, + "m2", MEASURE, String.class, + "m3", MEASURE, Instant.class + ); + + Set validElements = Sets.newHashSet( + structure.get("m1"), + structure.get("m2"), + structure.get("m3") + ); + + Set invalidElements = Sets.newHashSet( + structure.get("m1"), + structure.get("m2"), + structure.get("m3"), + wrongTypesDataset.get("m1") + ); + + Set wrongTypesElements = Sets.newHashSet( + wrongTypesDataset.values() + ); + + + when(dataset.getDataStructure()).thenReturn(structure); + when(invalidDataset.getDataStructure()).thenReturn(wrongTypesDataset); + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + softly.assertThatThrownBy(() -> { + FoldClause clause = new FoldClause(dataset, "newDimension", "newMeasure", invalidElements); + clause.getDataStructure(); + }) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("m1") + .hasMessageContaining("not found"); + + softly.assertThatThrownBy(() -> { + FoldClause clause = new FoldClause(invalidDataset, "newDimension", "newMeasure", wrongTypesElements); + clause.getDataStructure(); + }) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("same type"); + + FoldClause clause = new FoldClause(dataset, "newDimension", "newMeasure", validElements); + softly.assertThat(clause.getDataStructure()).isNotNull(); + } + } + + @Test + public void testUnfold() throws Exception { + + Dataset dataset = mock(Dataset.class); + DataStructure structure = DataStructure.of((o, aClass) -> o, + "id1", IDENTIFIER, String.class, + "id2", IDENTIFIER, String.class, // TODO: What if the dataset already contains id2? + "measure1", MEASURE, String.class, + "measure2", MEASURE, String.class, + "attribute1", ATTRIBUTE, String.class // TODO: Okay with attributes? + ); + when(dataset.getDataStructure()).thenReturn(structure); + + when(dataset.get()).then(invocation -> Stream.of( + tuple(structure, "id1-1", "id2-1", "measure1-1", "measure2-1", "attribute1-1"), + tuple(structure, "id1-1", "id2-2", null, "measure2-2", "attribute1-2"), + tuple(structure, "id1-2", "id2-1", "measure1-3", null, "attribute1-3"), + tuple(structure, "id1-2", "id2-2", "measure1-4", "measure2-4", null), + tuple(structure, "id1-3", "id2-1", null, null, null) + )); + + // TODO: Order should be irrelevant! + // Set elements = Sets.newLinkedHashSet(Lists.newArrayList("measure2", "measure1", "attribute1")); + Set elements = Sets.newLinkedHashSet( + Lists.newArrayList( + structure.get("measure1"), + structure.get("measure2"), + structure.get("attribute1") + ) + ); + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + FoldClause clause = new FoldClause(dataset, "newId", "newMeasure", elements); + + softly.assertThat(clause.getDataStructure()).containsOnlyKeys( + "id1", "id2", "newId", "newMeasure" + ); + + softly.assertThat(clause.get()).flatExtracting(input -> input).extracting(DataPoint::get) + .containsExactly( + "id1-1", "id2-1", "measure1", "measure1-1", + "id1-1", "id2-1", "measure2", "measure2-1", + "id1-1", "id2-1", "attribute1", "attribute1-1", + + // null + "id1-1", "id2-2", "measure2", "measure2-2", + "id1-1", "id2-2", "attribute1", "attribute1-2", + + "id1-2", "id2-1", "measure1", "measure1-3", + // null + "id1-2", "id2-1", "attribute1", "attribute1-3", + + "id1-2", "id2-2", "measure1", "measure1-4", + "id1-2", "id2-2", "measure2", "measure2-4" + // null + ); + } + } +} diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/RenameOperationTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/RenameOperationTest.java index 1537e301..abbbad7d 100644 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/RenameOperationTest.java +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/RenameOperationTest.java @@ -21,12 +21,14 @@ */ import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.DataPoint; import no.ssb.vtl.model.DataStructure; import no.ssb.vtl.model.Dataset; import org.junit.Test; -import java.util.Collections; +import java.util.Map; import java.util.stream.Stream; import static no.ssb.vtl.model.Component.Role; @@ -49,70 +51,71 @@ public Stream get() { } }; - @Test(expected = IllegalArgumentException.class) - public void testNotDuplicates() throws Exception { - ImmutableMap names = ImmutableMap.of("a1", "a2", "b1", "a2"); - try { - new RenameOperation(notNullDataset, names, Collections.emptyMap()); - } catch (Throwable t) { - assertThat(t).hasMessageContaining("a2"); - throw t; - } - } - - @Test(expected = IllegalArgumentException.class) - public void testConsistentArguments() throws Exception { - ImmutableMap names = ImmutableMap.of("a1", "a2", "b1", "b2"); - ImmutableMap roles; - roles = ImmutableMap.of("nothere", Role.IDENTIFIER); - try { - new RenameOperation(notNullDataset, names, roles); - } catch (Throwable t) { - assertThat(t).hasMessageContaining("nothere"); - throw t; - } - } - - @Test() - public void testKeyNotFound() throws Exception { - - Dataset dataset = mock(Dataset.class); - when(dataset.getDataStructure()).thenReturn( - DataStructure.of((s, o) -> null, "notfound", Role.IDENTIFIER, String.class) - ); - - Throwable ex = null; - try { - RenameOperation renameOperation = new RenameOperation(dataset, ImmutableMap.of("1a", "1b"), Collections.emptyMap()); - renameOperation.getDataStructure(); - } catch (Throwable t) { - ex = t; - } - assertThat(ex).hasMessageContaining("1a"); - - } +// @Test(expected = IllegalArgumentException.class) +// public void testNotDuplicates() throws Exception { +// ImmutableMap names = ImmutableMap.of("a1", "a2", "b1", "a2"); +// try { +// new RenameOperation(notNullDataset, names, Collections.emptyMap()); +// } catch (Throwable t) { +// assertThat(t).hasMessageContaining("a2"); +// throw t; +// } +// } + +// @Test(expected = IllegalArgumentException.class) +// public void testConsistentArguments() throws Exception { +// ImmutableMap names = ImmutableMap.of("a1", "a2", "b1", "b2"); +// ImmutableMap roles; +// roles = ImmutableMap.of("nothere", Role.IDENTIFIER); +// try { +// new RenameOperation(notNullDataset, names, roles); +// } catch (Throwable t) { +// assertThat(t).hasMessageContaining("nothere"); +// throw t; +// } +// } + +// @Test() +// public void testKeyNotFound() throws Exception { +// +// Dataset dataset = mock(Dataset.class); +// when(dataset.getDataStructure()).thenReturn( +// DataStructure.of((s, o) -> null, "notfound", Role.IDENTIFIER, String.class) +// ); +// +// Throwable ex = null; +// try { +// RenameOperation renameOperation = new RenameOperation(dataset, ImmutableMap.of("1a", "1b"), Collections.emptyMap()); +// renameOperation.getDataStructure(); +// } catch (Throwable t) { +// ex = t; +// } +// assertThat(ex).hasMessageContaining("1a"); +// +// } @Test public void testRename() throws Exception { Dataset dataset = mock(Dataset.class); - when(dataset.getDataStructure()).thenReturn( - DataStructure.of((s, o) -> null, - "Ia", Role.IDENTIFIER, String.class, - "Ma", Role.MEASURE, String.class, - "Aa", Role.ATTRIBUTE, String.class - ) + DataStructure structure = DataStructure.of((s, o) -> null, + "Ia", Role.IDENTIFIER, String.class, + "Ma", Role.MEASURE, String.class, + "Aa", Role.ATTRIBUTE, String.class + ); + when(dataset.getDataStructure()).thenReturn(structure); + + Map newNames = ImmutableMap.of( + structure.get("Ia"), "Ib", + structure.get("Ma"), "Mb", + structure.get("Aa"), "Ab" ); RenameOperation rename; rename = new RenameOperation( dataset, - ImmutableMap.of( - "Ia", "Ib", - "Ma", "Mb", - "Aa", "Ab" - ), Collections.emptyMap() + newNames ); assertThat(rename.getDataStructure().getRoles()).contains( @@ -127,34 +130,41 @@ public void testRename() throws Exception { public void testRenameAndCast() throws Exception { Dataset dataset = mock(Dataset.class); - when(dataset.getDataStructure()).thenReturn(DataStructure.of((o, aClass) -> o, - "Identifier1", Role.IDENTIFIER, Object.class, - "Identifier2", Role.IDENTIFIER, Object.class, - "Measure1", Role.MEASURE, Object.class, - "Measure2", Role.MEASURE, Object.class, - "Attribute1", Role.ATTRIBUTE, Object.class, - "Attribute2", Role.ATTRIBUTE, Object.class - )); + DataStructure structure = DataStructure.of((o, aClass) -> o, + "Identifier1", Role.IDENTIFIER, String.class, + "Identifier2", Role.IDENTIFIER, String.class, + "Measure1", Role.MEASURE, String.class, + "Measure2", Role.MEASURE, String.class, + "Attribute1", Role.ATTRIBUTE, String.class, + "Attribute2", Role.ATTRIBUTE, String.class + ); + when(dataset.getDataStructure()).thenReturn(structure); + when(dataset.get()).then(invocation -> { + return Stream.of( + structure.wrap(Maps.asMap(structure.keySet(), input -> (Object) input)) + ); + }); + + ImmutableMap newNames = new ImmutableMap.Builder() + .put(structure.get("Identifier1"), "Identifier1Measure") + .put(structure.get("Identifier2"), "Identifier2Attribute") + .put(structure.get("Measure1"), "Measure1Identifier") + .put(structure.get("Measure2"), "Measure2Attribute") + .put(structure.get("Attribute1"), "Attribute1Identifier") + .put(structure.get("Attribute2"), "Attribute2Measure") + .build(); + + ImmutableMap newRoles = new ImmutableMap.Builder() + .put(structure.get("Identifier1"), Role.MEASURE) + .put(structure.get("Identifier2"), Role.ATTRIBUTE) + .put(structure.get("Measure1"), Role.IDENTIFIER) + .put(structure.get("Measure2"), Role.ATTRIBUTE) + .put(structure.get("Attribute1"), Role.IDENTIFIER) + .put(structure.get("Attribute2"), Role.MEASURE) + .build(); RenameOperation rename; - rename = new RenameOperation( - dataset, - new ImmutableMap.Builder() - .put("Identifier1", "Identifier1Measure") - .put("Identifier2", "Identifier2Attribute") - .put("Measure1", "Measure1Identifier") - .put("Measure2", "Measure2Attribute") - .put("Attribute1", "Attribute1Identifier") - .put("Attribute2", "Attribute2Measure") - .build() - , new ImmutableMap.Builder() - .put("Identifier1", Role.MEASURE) - .put("Identifier2", Role.ATTRIBUTE) - .put("Measure1", Role.IDENTIFIER) - .put("Measure2", Role.ATTRIBUTE) - .put("Attribute1", Role.IDENTIFIER) - .put("Attribute2", Role.MEASURE).build() - ); + rename = new RenameOperation(dataset, newNames, newRoles); assertThat(rename.getDataStructure().getRoles()).contains( entry("Identifier1Measure", Role.MEASURE), @@ -165,5 +175,9 @@ public void testRenameAndCast() throws Exception { entry("Attribute2Measure", Role.MEASURE) ); + assertThat(rename.get()).flatExtracting(input -> input).extracting(DataPoint::get) + .containsOnlyElementsOf( + structure.keySet() + ); } } diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/SumOperationTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/SumOperationTest.java index def4239a..2b9eea78 100644 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/SumOperationTest.java +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/SumOperationTest.java @@ -7,17 +7,14 @@ import no.ssb.vtl.model.DataStructure; import no.ssb.vtl.model.Dataset; import no.ssb.vtl.script.operations.join.InnerJoinOperation; -import no.ssb.vtl.script.operations.join.JoinClause; -import no.ssb.vtl.script.operations.join.WorkingDataset; import org.assertj.core.api.SoftAssertions; import java.util.Arrays; import java.util.List; import java.util.stream.Stream; -import static no.ssb.vtl.model.Component.Role; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static no.ssb.vtl.model.Component.*; +import static org.mockito.Mockito.*; /** * Created by hadrien on 21/11/2016. @@ -242,28 +239,9 @@ public void testSumEx1() throws Exception { tuple -> tuple.get(3), ld, tuple -> tuple.get(3), rd ); - join.getClauses().add(new JoinClause() { - - - @Override - public WorkingDataset apply(WorkingDataset workingDataset) { - return new WorkingDataset() { - @Override - public DataStructure getDataStructure() { - return sumOperation.getDataStructure(); - } - - @Override - public Stream get() { - return workingDataset.get().map(tuple -> sumOperation.apply(tuple, null)); - - } - }; - } - }); softly.assertThat( - join.getDataStructure() + join.workDataset().getDataStructure() ).as("data structure of the sum operation of %s and %s", left, right) .isNotNull(); // TODO: Better check. @@ -345,13 +323,13 @@ public void testSumEx2() throws Exception { SumOperation sumOperation = new SumOperation(tuple -> tuple.get(3), ld, 1); softly.assertThat( - join.getDataStructure() + join.workDataset().getDataStructure() ).as("data structure of the sum operation of %s and 1", left) - .isEqualTo(join.getDataStructure()); + .isEqualTo(join.workDataset().getDataStructure()); DataStructure sumDs = sumOperation.getDataStructure(); softly.assertThat( - join.get() + join.workDataset().get() ).as("data of the sum operation of %s and 1", left) .containsExactly( tuple(sumDs.wrap("TIME", "2010"), @@ -447,32 +425,15 @@ public void testSumEx3() throws Exception { tuple -> tuple.get(3), ld ); - join.getClauses().add(new JoinClause() { - @Override - public WorkingDataset apply(WorkingDataset workingDataset) { - return new WorkingDataset() { - @Override - public DataStructure getDataStructure() { - return workingDataset.getDataStructure(); - } - - @Override - public Stream get() { - return workingDataset.get().map(tuple -> sumOperation.apply(tuple, null)); - } - - }; - } - }); softly.assertThat( sumOperation.getDataStructure() ).as("data structure of the sum operation of %s and %s", left, right) .isNotEqualTo(left.getDataStructure()); - DataStructure sumDs = join.getDataStructure(); + DataStructure sumDs = join.workDataset().getDataStructure(); softly.assertThat( - join.get() + join.workDataset().get() ).as("data of the sum operation of %s and %s", left, right) .containsExactly( tuple(sumDs.wrap("TIME", "2010"), diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/UnfoldClauseTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/UnfoldClauseTest.java new file mode 100644 index 00000000..e6901934 --- /dev/null +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/UnfoldClauseTest.java @@ -0,0 +1,163 @@ +package no.ssb.vtl.script.operations; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.DataPoint; +import no.ssb.vtl.model.DataStructure; +import no.ssb.vtl.model.Dataset; +import org.assertj.core.api.AutoCloseableSoftAssertions; +import org.junit.Test; + +import java.util.*; +import java.util.stream.Stream; + +import static com.google.common.base.Preconditions.checkArgument; +import static no.ssb.vtl.model.Component.Role.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class UnfoldClauseTest { + + private static Dataset.Tuple tuple(DataStructure structure, Object... values) { + checkArgument(values.length == structure.size()); + Map map = Maps.newHashMap(); + Iterator iterator = Lists.newArrayList(values).iterator(); + for (String name : structure.keySet()) { + map.put(name, iterator.next()); + } + return structure.wrap(map); + } + + @Test + public void testArguments() throws Exception { + + Dataset dataset = mock(Dataset.class); + Component validIdentifierReference = mock(Component.class); + Component validMeasureReference = mock(Component.class); + Set validElements = Sets.newHashSet("element1, element2"); + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + + softly.assertThatThrownBy(() -> new UnfoldClause(null, validIdentifierReference, validMeasureReference, validElements)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("dataset") + .hasMessageContaining("null"); + + softly.assertThatThrownBy(() -> new UnfoldClause(dataset, null, validMeasureReference, validElements)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("dimensionReference") + .hasMessageContaining("null"); + + softly.assertThatThrownBy(() -> new UnfoldClause(dataset, validIdentifierReference, null, validElements)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("measureReference") + .hasMessageContaining("null"); + + softly.assertThatThrownBy(() -> new UnfoldClause(dataset, validIdentifierReference, validMeasureReference, null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("elements") + .hasMessageContaining("null"); + + softly.assertThatThrownBy(() -> new UnfoldClause(dataset, validIdentifierReference, validMeasureReference, Collections.emptySet())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("elements") + .hasMessageContaining("empty"); + } + } + + @Test + public void testConstraint() throws Exception { + + Set validElements = Sets.newHashSet("some value"); + DataStructure structure = DataStructure.of((o, aClass) -> o, + "id1", IDENTIFIER, String.class, + "id2", IDENTIFIER, String.class, + "measure1", MEASURE, String.class + ); + Dataset dataset = mock(Dataset.class); + when(dataset.getDataStructure()).thenReturn(structure); + + Component validDimension = structure.get("id2"); + Component validMeasure = structure.get("measure1"); + Component invalidReference = mock(Component.class); + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + softly.assertThatThrownBy(() -> { + UnfoldClause clause = new UnfoldClause(dataset, invalidReference, validMeasure, validElements); + clause.getDataStructure(); + }) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("dimension") + .hasMessageContaining(invalidReference.toString()) + .hasMessageContaining("not found"); + + softly.assertThatThrownBy(() -> { + UnfoldClause clause = new UnfoldClause(dataset, validDimension, invalidReference, validElements); + clause.getDataStructure(); + }) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("measure") + .hasMessageContaining(invalidReference.toString()) + .hasMessageContaining("not found"); + + softly.assertThatThrownBy(() -> { + UnfoldClause clause = new UnfoldClause(dataset, validMeasure, validDimension, validElements); + clause.getDataStructure(); + }) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(validMeasure.toString()) + .hasMessageContaining("was not a dimension"); + + + UnfoldClause clause = new UnfoldClause(dataset, validDimension, validMeasure, validElements); + clause.getDataStructure(); + } + } + + @Test + public void testUnfold() throws Exception { + + Set elements = Sets.newLinkedHashSet(Arrays.asList("id2-1", "id2-2")); + Dataset dataset = mock(Dataset.class); + DataStructure structure = DataStructure.of((o, aClass) -> o, + "id1", IDENTIFIER, String.class, + "id2", IDENTIFIER, String.class, + "measure1", MEASURE, String.class, + "measure2", MEASURE, String.class, + "attribute1", ATTRIBUTE, String.class + ); + when(dataset.getDataStructure()).thenReturn(structure); + + when(dataset.get()).then(invocation -> Stream.of( + tuple(structure, "id1-1", "id2-1", "measure1-1", "measure2-1", "attribute1-1"), + tuple(structure, "id1-1", "id2-2", "measure1-2", "measure2-2", "attribute1-2"), + tuple(structure, "id1-2", "id2-1", "measure1-3", "measure2-3", "attribute1-3"), + tuple(structure, "id1-2", "id2-2", "measure1-4", "measure2-4", "attribute1-4"), + tuple(structure, "id1-3", "id2-1", "measure1-5", "measure2-5", "attribute1-5") + )); + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + UnfoldClause clause = new UnfoldClause(dataset, structure.get("id2"), structure.get("measure1"), elements); + + softly.assertThat(clause.getDataStructure()).containsOnlyKeys( + "id1", "id2-1", "id2-2" + ); + + softly.assertThat(clause.get()).flatExtracting(input -> input).extracting(DataPoint::getName) + .contains( + "id1", "id2-1", "id2-2", + "id1", "id2-1", "id2-2", + "id1", "id2-1", "id2-2" + ); + + softly.assertThat(clause.get()).flatExtracting(input -> input).extracting(DataPoint::get) + .contains( + "id1-1", "measure1-1", "measure1-2", + "id1-2", "measure1-3", "measure1-4", + "id1-3", "measure1-5", null + ); + } + } +} diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/join/AbstractJoinOperationTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/join/AbstractJoinOperationTest.java index d6ff3e14..a200a627 100644 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/join/AbstractJoinOperationTest.java +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/operations/join/AbstractJoinOperationTest.java @@ -56,7 +56,7 @@ public void testSameDatasetShouldFail() throws Exception { ) { @Override - WorkingDataset workDataset() { + public WorkingDataset workDataset() { return new WorkingDataset() { @Override public DataStructure getDataStructure() { @@ -85,7 +85,7 @@ public Stream get() { ) { @Override - WorkingDataset workDataset() { + public WorkingDataset workDataset() { return null; } }; @@ -110,7 +110,7 @@ public void testEmptyFails() throws Exception { try { new AbstractJoinOperation(Collections.emptyMap()) { @Override - WorkingDataset workDataset() { + public WorkingDataset workDataset() { return null; } }; @@ -148,7 +148,7 @@ public void testSimple() throws Exception { AbstractJoinOperation result = new AbstractJoinOperation(ImmutableMap.of("ds1", ds1)) { @Override - WorkingDataset workDataset() { + public WorkingDataset workDataset() { return new WorkingDataset() { @Override public DataStructure getDataStructure() { @@ -163,26 +163,7 @@ public Stream get() { } }; - result.getClauses().add(new JoinClause() { - - @Override - public WorkingDataset apply(WorkingDataset workingDataset) { - return new WorkingDataset() { - @Override - public DataStructure getDataStructure() { - return workingDataset.getDataStructure(); - } - - @Override - public Stream get() { - return workingDataset.get(); - } - }; - } - - }); - - assertThat(result.get()) + assertThat(result.workDataset().get()) .containsAll(ds1.get().collect(Collectors.toList())); } diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/ReferenceVisitorTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/ReferenceVisitorTest.java new file mode 100644 index 00000000..4f144de4 --- /dev/null +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/ReferenceVisitorTest.java @@ -0,0 +1,124 @@ +package no.ssb.vtl.script.visitors; + + +import com.google.common.collect.ImmutableMap; +import no.ssb.vtl.model.Component; +import no.ssb.vtl.model.DataStructure; +import no.ssb.vtl.model.Dataset; +import no.ssb.vtl.parser.VTLLexer; +import no.ssb.vtl.parser.VTLParser; +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CommonTokenStream; +import org.assertj.core.api.AutoCloseableSoftAssertions; +import org.assertj.core.util.Maps; +import org.junit.Before; +import org.junit.Test; + +import javax.script.Bindings; +import javax.script.SimpleBindings; + +import static no.ssb.vtl.model.Component.Role.IDENTIFIER; +import static org.mockito.Mockito.*; + +public class ReferenceVisitorTest { + + private SimpleBindings bindings; + private Component component; + private Dataset dataset; + + private static VTLParser parse(String expression) { + VTLLexer lexer = new VTLLexer(new ANTLRInputStream(expression)); + VTLParser parser = new VTLParser(new CommonTokenStream(lexer)); + parser.setErrorHandler(new BailErrorStrategy()); + return parser; + } + + @Before + public void setUp() throws Exception { + DataStructure structure = DataStructure.of((o, aClass) -> o, + "component", IDENTIFIER, String.class + ); + component = structure.get("component"); + dataset = mock(Dataset.class); + bindings = new SimpleBindings(ImmutableMap.of( + "component", component, + "dataset", dataset + )); + + when(dataset.getDataStructure()).thenReturn(structure); + } + + @Test + public void testNotFound() throws Exception { + Bindings emptyBindings = new SimpleBindings( + Maps.newHashMap("dataset", dataset)); + ReferenceVisitor referenceVisitor = new ReferenceVisitor(emptyBindings); + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + softly.assertThatThrownBy(() -> referenceVisitor.visit(parse("componentNotFound").componentRef())) + .describedAs("exception when component not found") + .hasMessageContaining("variable") + .hasMessageContaining("componentNotFound") + .hasMessageContaining("not found"); + + softly.assertThatThrownBy(() -> referenceVisitor.visit(parse("dataset.componentNotFound").componentRef())) + .describedAs("exception when component not found") + .hasMessageContaining("variable") + .hasMessageContaining("componentNotFound") + .hasMessageContaining("not found"); + + softly.assertThatThrownBy(() -> referenceVisitor.visit(parse("datasetNotFound").datasetRef())) + .describedAs("exception when component not found") + .hasMessageContaining("variable") + .hasMessageContaining("datasetNotFound") + .hasMessageContaining("not found"); + + softly.assertThatThrownBy(() -> referenceVisitor.visit(parse("variableNotFound").variableRef())) + .describedAs("exception when component not found") + .hasMessageContaining("variable") + .hasMessageContaining("variableNotFound") + .hasMessageContaining("not found"); + + } + } + + @Test + public void testComponents() throws Exception { + String[] components = new String[]{ + "component", + "'component'", + "dataset.'component'", + "'dataset'.component" + }; + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + ReferenceVisitor referenceVisitor = new ReferenceVisitor(bindings); + for (String componentExpression : components) { + VTLParser parser = parse(componentExpression); + softly.assertThat(referenceVisitor.visit(parser.componentRef())) + .describedAs("resolved componentRef for [%s]", componentExpression) + .isSameAs(component); + } + } + } + + @Test + public void testDatasets() throws Exception { + String[] datasets = new String[]{ + "dataset", + "'dataset'", + }; + + try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { + ReferenceVisitor referenceVisitor = new ReferenceVisitor(bindings); + for (String datasetExpression : datasets) { + VTLParser parser = parse(datasetExpression); + softly.assertThat(referenceVisitor.visit(parser.datasetRef())) + .describedAs("resolved datasetRef for [%s]", datasetExpression) + .isNotNull() + .isSameAs(dataset); + } + } + } +} diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinCalcClauseVisitorTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinCalcClauseVisitorTest.java index 72de45e2..64cf5c75 100644 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinCalcClauseVisitorTest.java +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinCalcClauseVisitorTest.java @@ -7,6 +7,7 @@ import no.ssb.vtl.model.Dataset; import no.ssb.vtl.parser.VTLLexer; import no.ssb.vtl.parser.VTLParser; +import no.ssb.vtl.script.visitors.ReferenceVisitor; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CommonTokenStream; @@ -18,6 +19,7 @@ import java.util.function.Function; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class JoinCalcClauseVisitorTest { @@ -29,7 +31,7 @@ public void testNumericalLiteralsSum() throws Exception { VTLParser parser = new VTLParser(new CommonTokenStream(lexer)); parser.setErrorHandler(new BailErrorStrategy()); - JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(); + JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(null); Function result = visitor.visit(parser.joinCalcExpression()); @@ -45,7 +47,7 @@ public void testNumericalLiteralsSumWithParenthesis() throws Exception { VTLParser parser = new VTLParser(new CommonTokenStream(lexer)); parser.setErrorHandler(new BailErrorStrategy()); - JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(); + JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(null); Function result = visitor.visit(parser.joinCalcExpression()); @@ -61,7 +63,7 @@ public void testNumericalLiteralsProduct() throws Exception { VTLParser parser = new VTLParser(new CommonTokenStream(lexer)); parser.setErrorHandler(new BailErrorStrategy()); - JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(); + JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(null); Function result = visitor.visit(parser.joinCalcExpression()); @@ -92,7 +94,10 @@ public void testNumericalVariableReference() { Dataset.Tuple tuple = ds.wrap(variables); - JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(); + Map scope = Maps.newHashMap(); + scope.putAll(ds); + + JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(new ReferenceVisitor(scope)); Function result = visitor.visit(parser.joinCalcExpression()); @@ -111,7 +116,7 @@ public void testNumericalLiteralsProductWithParenthesis() throws Exception { // Setup fake map. - JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(); + JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(null); Function result = visitor.visit(parser.joinCalcExpression()); @@ -126,11 +131,9 @@ public void testNumericalReferenceNotFound() throws Exception { VTLLexer lexer = new VTLLexer(new ANTLRInputStream(test)); VTLParser parser = new VTLParser(new CommonTokenStream(lexer)); parser.setErrorHandler(new BailErrorStrategy()); - JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(); - + JoinCalcClauseVisitor visitor = new JoinCalcClauseVisitor(new ReferenceVisitor(Collections.emptyMap())); - Throwable ex = null; - try { + assertThatThrownBy(() -> { // TODO: This should happen during execution (when data is computed). Function result = visitor.visit(parser.joinCalcExpression()); result.apply(new Dataset.AbstractTuple() { @@ -139,10 +142,7 @@ protected List delegate() { return Collections.emptyList(); } }); - } catch (Throwable t) { - ex = t; - } - assertThat(ex).hasMessageContaining("variable") + }).hasMessageContaining("variable") .hasMessageContaining("notFoundVariable"); } diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinRenameClauseVisitorTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinRenameClauseVisitorTest.java deleted file mode 100644 index 18c425c8..00000000 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/visitors/join/JoinRenameClauseVisitorTest.java +++ /dev/null @@ -1,71 +0,0 @@ -package no.ssb.vtl.script.visitors.join; - -import no.ssb.vtl.model.DataStructure; -import no.ssb.vtl.parser.VTLLexer; -import no.ssb.vtl.parser.VTLParser; -import no.ssb.vtl.script.operations.RenameOperation; -import no.ssb.vtl.script.operations.join.WorkingDataset; -import org.antlr.v4.runtime.*; -import org.junit.Test; - -import java.util.stream.Stream; - -import static no.ssb.vtl.model.Component.Role; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class JoinRenameClauseVisitorTest { - - @Test - public void testRename() throws Exception { - - WorkingDataset dataset = mock(WorkingDataset.class); - DataStructure structure = DataStructure.of((o, aClass) -> o, - "foo.identifier", Role.IDENTIFIER, String.class, - "foo.measure", Role.MEASURE, String.class, - "foo.attribute", Role.ATTRIBUTE, String.class, - "identifier", Role.IDENTIFIER, String.class, - "measure", Role.MEASURE, String.class, - "attribute", Role.ATTRIBUTE, String.class - ); - when(dataset.getDataStructure()).thenReturn(structure); - when(dataset.get()).thenReturn(Stream.empty()); - - - String test = "rename" + - " foo.identifier to renamedFooIdentifier," + - " foo.measure to renamedFooMeasure," + - " foo.attribute to renamedFooAttribute," + - " identifier to renamedIdentifier," + - " measure to renamedMeasure, " + - " attribute to renamedAttribute"; - VTLLexer lexer = new VTLLexer(new ANTLRInputStream(test)); - VTLParser parser = new VTLParser(new CommonTokenStream(lexer)); - - lexer.addErrorListener(new ConsoleErrorListener()); - lexer.addErrorListener(new BaseErrorListener() { - @Override - public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { - throw new RuntimeException(msg, e); - } - }); - parser.addErrorListener(new ConsoleErrorListener()); - parser.addErrorListener(new BaseErrorListener() { - @Override - public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { - throw new RuntimeException(msg, e); - } - }); - - JoinRenameClauseVisitor visitor = new JoinRenameClauseVisitor(dataset); - - RenameOperation visit = visitor.visit(parser.joinRenameExpression()); - - assertThat(visit.getDataStructure()).containsOnlyKeys( - "renamedFooIdentifier", "renamedFooMeasure", "renamedFooAttribute", - "renamedIdentifier", "renamedMeasure", "renamedAttribute" - ); - - } -} diff --git a/pom.xml b/pom.xml index bbf3ef00..fe601537 100644 --- a/pom.xml +++ b/pom.xml @@ -253,6 +253,20 @@ ${project.version} + + + org.antlr + antlr4-runtime + 4.6 + + + + org.antlr + antlr4 + 4.6 + test + + com.google.guava guava diff --git a/websocket/pom.xml b/websocket/pom.xml index c296ede5..ed620f8e 100644 --- a/websocket/pom.xml +++ b/websocket/pom.xml @@ -19,6 +19,12 @@ java-vtl-script + + org.antlr + antlr4 + compile + + no.ssb.vtl java-vtl-ssb-api-connector @@ -67,11 +73,6 @@ termd-core 1.0.0 - - org.antlr - antlr4 - RELEASE -