diff --git a/entitydb-eql/README.md b/entitydb-eql/README.md new file mode 100644 index 0000000..893c002 --- /dev/null +++ b/entitydb-eql/README.md @@ -0,0 +1,52 @@ +# Idyl NLP Entity Query Language + +The Entity Query Language, or EQL, provides a SQL-like syntax for querying entities. EQL provides a means of filtering entities that meet given conditions. This project includes a Pig UDF for using EQL in your Pig jobs. + +## Syntax + +The EQL query `select * from entities` will return all entities from the entity store. + +A where clause can be added to only retrieve entities meeting some condition: + +`select * from entities where text = 'George Washington'` + +This query returns all entities having the text "George Washington." Other queryable fields are confidence, documentId, and context. Multiple fields can be combined with the and keyword. + +`select * from entities where text = 'George Washington' and confidence > 50` + +EQL does not support OR conditionals. Use multiple EQL queries to accomplish an OR condition. EQL queries can be executed through the Idyl E3 API when the entity store is enabled. + +### Example queries + +To find or filter entities with a given text: + +`select * from entities where text = "George Washington"` + +To find or filter entities with a given text in a specific context: + +`select * from entities where text = "George Washington" and context = "book1"` + +#### Queryable Fields + +Now that you see it's a lot like SQL, here are the queryable fields: + +| Field | Description | Examples | Remarks | +| ----- | ----------- | -------- | ------- | +| `id` | The entity's ID. | | | +| `text` | The text of the entity. | "George Washington" | Supports wildcards `*` in the text but not as the first character. | +| `type` | The type of the entity. | "person" | | +| `confidence` | The confidence of the entity - integer values between 0 and 100, inclusive. | 50 | | +| `language` | The language of the entity. | en | | +| `context` | The entity's context. | | | +| `documentId` | The entity's document ID. | | | +| `uri` | The entity's URI. | | | + +#### Paging + +Paging can be achieved using the `limit` and `offset` keywords: + +`select * from entities limit 10 offset 50` + +This query returns the first 10 entities after the first 50 entities. + +The `limit` and `offset` keywords can also be used independently. Note that by default the limit is 25. Use caution when setting large limits. diff --git a/entitydb-eql/eql-filters/pom.xml b/entitydb-eql/eql-filters/pom.xml new file mode 100644 index 0000000..dcac2d5 --- /dev/null +++ b/entitydb-eql/eql-filters/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + + ai.idylnlp + eql + 1.3.0-SNAPSHOT + + eql-filters + eql-filters + + + ai.idylnlp + eql-language + ${project.version} + + + ai.idylnlp + idylnlp-model + ${project.version} + + + org.apache.commons + commons-collections4 + ${commons.collections.version} + + + junit + junit + test + + + org.apache.logging.log4j + log4j-core + + + diff --git a/entitydb-eql/eql-filters/src/main/java/ai/idylnlp/eql/filters/EqlFilters.java b/entitydb-eql/eql-filters/src/main/java/ai/idylnlp/eql/filters/EqlFilters.java new file mode 100644 index 0000000..903e0b3 --- /dev/null +++ b/entitydb-eql/eql-filters/src/main/java/ai/idylnlp/eql/filters/EqlFilters.java @@ -0,0 +1,285 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.filters; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import ai.idylnlp.eql.Eql; +import ai.idylnlp.eql.exceptions.QueryGenerationException; +import ai.idylnlp.eql.filters.comparisons.DateComparison; +import ai.idylnlp.eql.model.EntityQuery; +import ai.idylnlp.model.entity.Entity; + +/** + * Static functions for applying EQL statements to entities. + * + * @author Mountain Fog, Inc. + * + */ +public class EqlFilters { + + private static final Logger LOGGER = LogManager.getLogger(EqlFilters.class); + + /** + * Determines if an entity satisfies (matches) an EQL statement. Note that this function + * internally calls the filterEntities function for evaluation by wrapping + * the entity and the EQL statements in collections. + * @param entity The {@link Entity entity} being tested. + * @param eql The EQL statement. + * @return true if the entity satisfies the EQL statement. Otherwise, false. + * @throws QueryGenerationException Thrown if the EQL statement is malformed. + */ + public static boolean isMatch(Entity entity, String eql) throws QueryGenerationException { + + Collection entities = new ArrayList(); + entities.add(entity); + + List eqlStatements = new ArrayList(); + eqlStatements.add(eql); + + Collection matchedEntities = filterEntities(entities, eqlStatements); + + // If the collection is NOT empty the entity was matched. + return !(matchedEntities.isEmpty()); + + } + + /** + * Filters date entities. + * Non-date entities are not filtered out. + * @param entities The collection of entities. + * @param date The target date. + * @param dateComparison How to compare the dates. + * @return A filtered collection of date entities. + */ + public static Collection filterEntities(Collection entities, Date date, DateComparison dateComparison) { + + Collection matchedEntities = new LinkedList<>(); + + for(Entity entity : entities ) { + + if(entity.getType().equals("date")) { + + // The exact milliseconds of the date is stored in the metadata of the date. + String milliseconds = entity.getMetadata().get("time"); + + if(milliseconds != null) { + + Date entityDate = new Date(Long.valueOf(milliseconds)); + + if(dateComparison.equals(DateComparison.BEFORE) && entityDate.before(date)) { + + matchedEntities.add(entity); + + } else if(dateComparison.equals(DateComparison.AFTER) && entityDate.after(date)) { + + matchedEntities.add(entity); + + } else { + + // For some reason this date is missing its milliseconds metadata value. Do not include it since we don't know. + } + + } + + } else { + + // Include it since it is not a date. + matchedEntities.add(entity); + + } + + } + + return matchedEntities; + + } + + /** + * Filters date entities in a date window centered on a target date. + * Non-date entities are not filtered out. + * @param entities The collection of entities. + * @param targetDate The target date. + * @param minutes The size of the window in minutes. + * @param value The size of the window. + * @return A filtered collection of date entities that exist in the given window. + */ + public static Collection filterEntities(Collection entities, Date targetDate, int minutes) { + + Date startDate = new Date(targetDate.getTime() - 5*60*1000); + Date endDate = new Date(targetDate.getTime() + 5*60*1000); + + Collection matchedEntities = new LinkedList<>(); + + for(Entity entity : entities ) { + + if(entity.getType().equals("date")) { + + // The exact milliseconds of the date is stored in an metadata of the date. + String milliseconds = entity.getMetadata().get("time"); + + if(milliseconds != null) { + + Date entityDate = new Date(Long.valueOf(milliseconds)); + + if(entityDate.after(startDate) && entityDate.before(endDate)) { + + matchedEntities.add(entity); + + } + + } else { + + // For some reason this date is missing its milliseconds metadata value. Do not include it since we don't know. + } + + } else { + + // Include it since it is not a date. + matchedEntities.add(entity); + + } + + } + + return matchedEntities; + + } + + /** + * Filter a collection of entities based on given EQL statements. + * @param entities The collection of {@link Entity entities}. + * @param eqlStatement An EQL statement. + * @return A filtered collection of {@link Entity entities} containing only those + * entities that meet the criteria of at least one EQL statement. + * @throws QueryGenerationException + */ + public static Collection filterEntities(Collection entities, String eqlStatement) throws QueryGenerationException { + + return filterEntities(entities, Arrays.asList(eqlStatement)); + + } + + /** + * Filter a collection of entities based on given EQL statements. + * @param entities The collection of {@link Entity entities}. + * @param eqlStatements A list of EQL statements. + * @return A filtered collection of {@link Entity entities} containing only those + * entities that meet the criteria of at least one EQL statement. + * @throws QueryGenerationException + */ + public static Collection filterEntities(Collection entities, List eqlStatements) throws QueryGenerationException { + + // A universalMatch is when all entities match the filter. When this happens + // there is no need to check each individual entity. + boolean universalMatch = false; + + Collection matchedEntities = new LinkedList<>(); + + if(CollectionUtils.isEmpty(eqlStatements)) { + + // There are no statements so this is a universal match. + universalMatch = true; + + } else { + + for(String eql : eqlStatements) { + + if("select * from entities".equalsIgnoreCase(eql)) { + + universalMatch = true; + + } else { + + EntityQuery entityQuery = Eql.generate(eql); + + if(!universalMatch) { + + for(Entity entity : entities) { + + if(entityQuery.isMatch(entity)) { + + if(passNotConditions(entity, entityQuery)) { + matchedEntities.add(entity); + } + + } + } + + } + + } + + } + + } + + // If it is a universalMatch we return all entities. + // Otherwise we just return the entities that matched at least one EQL statement. + + if(universalMatch) { + + return entities; + + } else { + + return matchedEntities; + + } + + } + + private static boolean passNotConditions(Entity entity, EntityQuery entityQuery) { + + // Determine if the entity passes the NOT conditions of the query. + + if(StringUtils.isNotEmpty(entityQuery.getNotText()) && StringUtils.equals(entity.getText(), entityQuery.getNotText())) return false; + if(StringUtils.isNotEmpty(entityQuery.getNotType()) && StringUtils.equals(entity.getType(), entityQuery.getNotType())) return false; + if(StringUtils.isNotEmpty(entityQuery.getNotContext()) && StringUtils.equals(entity.getContext(), entityQuery.getNotContext())) return false; + if(StringUtils.isNotEmpty(entityQuery.getNotDocumentId()) && StringUtils.equals(entity.getDocumentId(), entityQuery.getNotDocumentId())) return false; + if(StringUtils.isNotEmpty(entityQuery.getNotLanguageCode()) && StringUtils.equals(entity.getLanguageCode(), entityQuery.getNotLanguageCode())) return false; + if(StringUtils.isNotEmpty(entityQuery.getNotUri()) && StringUtils.equals(entity.getUri(), entityQuery.getNotUri())) return false; + + return true; + + } + +} diff --git a/entitydb-eql/eql-filters/src/main/java/ai/idylnlp/eql/filters/comparisons/DateComparison.java b/entitydb-eql/eql-filters/src/main/java/ai/idylnlp/eql/filters/comparisons/DateComparison.java new file mode 100644 index 0000000..e76cf07 --- /dev/null +++ b/entitydb-eql/eql-filters/src/main/java/ai/idylnlp/eql/filters/comparisons/DateComparison.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.filters.comparisons; + +import org.apache.commons.lang3.StringUtils; + +public enum DateComparison { + + BEFORE("before"), AFTER("after"); + + private String dateComparison; + + private DateComparison(String dateComparison) { + + this.dateComparison = dateComparison; + + } + + /** + * Gets the enumeration from a string value. + * @param dateComparison The value to look up. + * @return A {@link DateComparison comparison}. + */ + public static DateComparison fromValue(String dateComparison) { + + if(StringUtils.isNotEmpty(dateComparison)) { + + for (DateComparison d : DateComparison.values()) { + + if (dateComparison.equalsIgnoreCase(d.getDateComparison())) { + return d; + } + + } + + } + + throw new IllegalArgumentException("No date comparison found with value: " + dateComparison); + + } + + @Override + public String toString() { + + return dateComparison; + + } + + public String getDateComparison() { + + return dateComparison; + + } + +} diff --git a/entitydb-eql/eql-filters/src/test/java/ai/idylnlp/test/eql/filters/EqlFiltersTest.java b/entitydb-eql/eql-filters/src/test/java/ai/idylnlp/test/eql/filters/EqlFiltersTest.java new file mode 100644 index 0000000..cc75587 --- /dev/null +++ b/entitydb-eql/eql-filters/src/test/java/ai/idylnlp/test/eql/filters/EqlFiltersTest.java @@ -0,0 +1,331 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.test.eql.filters; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Test; + +import ai.idylnlp.eql.filters.EqlFilters; +import ai.idylnlp.eql.filters.comparisons.DateComparison; +import ai.idylnlp.model.entity.Entity; + +public class EqlFiltersTest { + + @Test + public void isMatchTest0() throws Exception { + + Entity entity = new Entity("George Washington", 0.5, "person", "[0, 2)", "context", "docid"); + + entity.setContext("context"); + entity.setDocumentId("documentid"); + + String eqlStatement = "select * from entities"; + + boolean isMatch = EqlFilters.isMatch(entity, eqlStatement); + + assertTrue(isMatch); + + } + + @Test + public void isMatchTest1() throws Exception { + + Entity entity = new Entity("entity1", 0.5, "person", "[0, 2)", "context", "docid"); + + entity.setContext("context"); + entity.setDocumentId("documentid"); + + String eqlStatement = "select * from entities where text = \"entity1\""; + + boolean isMatch = EqlFilters.isMatch(entity, eqlStatement); + + assertTrue(isMatch); + + } + + @Test + public void isMatchTest2() throws Exception { + + Entity entity = new Entity("entity2", 0.5, "person", "[0, 2)", "context", "docid"); + + entity.setContext("context"); + entity.setDocumentId("documentid"); + + String eqlStatement = "select * from entities where text = \"entity1\""; + + boolean isMatch = EqlFilters.isMatch(entity, eqlStatement); + + assertFalse(isMatch); + + } + + @Test + public void isMatchTest3() throws Exception { + + Entity entity1 = new Entity("entity2", 0.5, "person", "[0, 2)", "context", "docid"); + Entity entity2 = new Entity("blah", 0.5, "person", "[0, 2)", "context", "docid"); + + List entities = new ArrayList(); + entities.add(entity1); + entities.add(entity2); + + String eqlStatement = "select * from entities where text = \"entity1\" and text != \"blah\""; + + Collection matchedEntities = EqlFilters.filterEntities(entities, eqlStatement); + + assertEquals(0, matchedEntities.size()); + + } + + @Test + public void isMatchTest4() throws Exception { + + Entity entity = new Entity("entity", 0.5, "person", "[0, 2)", "context", "docid"); + + String eqlStatement = "select * from entities where text = \"entity\" and type = \"place\""; + + boolean isMatch = EqlFilters.isMatch(entity, eqlStatement); + + assertFalse(isMatch); + + } + + @Test + public void filterEntitiesText() throws Exception { + + Collection entities = new ArrayList<>(); + + Entity entity1 = new Entity("entity1", 0.5, "person", "[0, 2)", "context", "docid"); + entity1.setContext("context"); + entity1.setDocumentId("documentid"); + + Entity entity2 = new Entity("entity2", 0.5, "person", "[0, 2)", "context", "docid"); + entity2.setContext("context"); + entity2.setDocumentId("documentid"); + + Entity entity3 = new Entity("entity3", 0.5, "person", "[0, 2)", "context", "docid"); + entity3.setContext("context"); + entity3.setDocumentId("documentid"); + + entities.add(entity1); + entities.add(entity2); + entities.add(entity3); + + List eqlStatements = new ArrayList<>(); + eqlStatements.add("select * from entities where text = \"entity2\""); + + Collection filteredEntities = EqlFilters.filterEntities(entities, eqlStatements); + + assertEquals(1, filteredEntities.size()); + assertEquals("entity2", filteredEntities.iterator().next().getText()); + + } + + @Test + public void filterEntitiesConfidence() throws Exception { + + Collection entities = new ArrayList<>(); + + Entity entity1 = new Entity("entity1", 0.5, "person", "[0, 2)", "context", "docid"); + entity1.setContext("context"); + entity1.setDocumentId("documentid"); + + Entity entity2 = new Entity("entity2", 0.6, "person", "[0, 2)", "context", "docid"); + entity2.setContext("context"); + entity2.setDocumentId("documentid"); + + Entity entity3 = new Entity("entity3", 0.7, "person", "[0, 2)", "context", "docid"); + entity3.setContext("context"); + entity3.setDocumentId("documentid"); + + entities.add(entity1); + entities.add(entity2); + entities.add(entity3); + + List eqlStatements = new ArrayList<>(); + eqlStatements.add("select * from entities where confidence = 50"); + + Collection filteredEntities = EqlFilters.filterEntities(entities, eqlStatements); + + assertEquals(1, filteredEntities.size()); + assertEquals("entity1", filteredEntities.iterator().next().getText()); + + } + + @Test + public void filterEntitiesConfidenceBetween() throws Exception { + + Collection entities = new ArrayList<>(); + + Entity entity1 = new Entity("entity1", 0.5, "person", "[0, 2)", "context", "docid"); + entity1.setContext("context"); + entity1.setDocumentId("documentid"); + + Entity entity2 = new Entity("entity2", 0.6, "person", "[0, 2)", "context", "docid"); + entity2.setContext("context"); + entity2.setDocumentId("documentid"); + + Entity entity3 = new Entity("entity3", 0.7, "person", "[0, 2)", "context", "docid"); + entity3.setContext("context"); + entity3.setDocumentId("documentid"); + + entities.add(entity1); + entities.add(entity2); + entities.add(entity3); + + List eqlStatements = new ArrayList<>(); + eqlStatements.add("select * from entities where confidence between 45 and 55"); + + Collection filteredEntities = EqlFilters.filterEntities(entities, eqlStatements); + + assertEquals(1, filteredEntities.size()); + assertEquals("entity1", filteredEntities.iterator().next().getText()); + + } + + @Test + public void filterEntitiesContext() throws Exception { + + Collection entities = new ArrayList<>(); + + Entity entity1 = new Entity("entity1", 0.5, "person", "[0, 2)", "context", "docid"); + entity1.setContext("context"); + entity1.setDocumentId("documentid"); + + Entity entity2 = new Entity("entity2", 0.6, "person", "[0, 2)", "context", "docid"); + entity2.setContext("context"); + entity2.setDocumentId("documentid"); + + Entity entity3 = new Entity("entity3", 0.7, "person", "[0, 2)", "context", "docid"); + entity3.setContext("context"); + entity3.setDocumentId("documentid"); + + entities.add(entity1); + entities.add(entity2); + entities.add(entity3); + + List eqlStatements = new ArrayList<>(); + eqlStatements.add("select * from entities where context = \"test\""); + + Collection filteredEntities = EqlFilters.filterEntities(entities, eqlStatements); + + assertEquals(0, filteredEntities.size()); + + } + + @Test + public void datesFilter1() { + + Map metadata = new HashMap(); + metadata.put("time", "1464027942965"); + + Entity entity = new Entity("entity1", 0.5, "date", "[0, 2)", "context", "docid"); + entity.setMetadata(metadata); + + Collection entities = new ArrayList(); + entities.add(entity); + + Collection filteredEntities = EqlFilters.filterEntities(entities, new Date(), DateComparison.BEFORE); + + assertEquals(1, filteredEntities.size()); + + } + + @Test + public void datesFilter2() { + + Map metadata = new HashMap(); + metadata.put("time", "1464027942965"); + + Entity entity = new Entity("entity1", 0.5, "date", "[0, 2)", "context", "docid"); + entity.setMetadata(metadata); + + Collection entities = new ArrayList(); + entities.add(entity); + + Collection filteredEntities = EqlFilters.filterEntities(entities, new Date(), DateComparison.AFTER); + + assertEquals(0, filteredEntities.size()); + + } + + @Test + public void datesFilter3() { + + // Person entities should not be filtered out by a date filter. + + Entity entity = new Entity("entity1", 0.5, "person", "[0, 2)", "context", "docid"); + + Collection entities = new ArrayList(); + entities.add(entity); + + Collection filteredEntities = EqlFilters.filterEntities(entities, new Date(), DateComparison.BEFORE); + + assertEquals(1, filteredEntities.size()); + + } + + @Test + public void datesFilter4() { + + Map metadata = new HashMap(); + metadata.put("time", "1464027942965"); + + Entity entity = new Entity("entity1", 0.5, "date", "[0, 2)", "context", "docid"); + entity.setMetadata(metadata); + + Map metadata2 = new HashMap(); + metadata2.put("time", "1464020000000"); + + Entity entity2 = new Entity("entity1", 0.5, "date", "[0, 2)", "context", "docid"); + entity2.setMetadata(metadata2); + + Collection entities = new ArrayList(); + entities.add(entity); + entities.add(entity2); + + Collection filteredEntities = EqlFilters.filterEntities(entities, new Date(Long.valueOf("1464027942910")), 30); + + assertEquals(1, filteredEntities.size()); + + } + +} diff --git a/entitydb-eql/eql-filters/src/test/resources/log4j2.xml b/entitydb-eql/eql-filters/src/test/resources/log4j2.xml new file mode 100644 index 0000000..8a1918f --- /dev/null +++ b/entitydb-eql/eql-filters/src/test/resources/log4j2.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + diff --git a/entitydb-eql/eql-language/pom.xml b/entitydb-eql/eql-language/pom.xml new file mode 100644 index 0000000..825f936 --- /dev/null +++ b/entitydb-eql/eql-language/pom.xml @@ -0,0 +1,96 @@ + + + + 4.0.0 + + ai.idylnlp + eql + 1.3.0-SNAPSHOT + + eql-language + eql-language + + + + org.antlr + antlr4-maven-plugin + 4.0 + + ${basedir}/src/main/java/ai/idylnlp/eql/antlr + + + + + antlr4 + + + + + + net.alchim31.maven + scala-maven-plugin + 3.4.6 + + + *.java + + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + + + ai.idylnlp + idylnlp-model + ${project.version} + + + org.antlr + antlr4-runtime + ${antlr.version} + + + org.apache.commons + commons-collections4 + ${commons.collections.version} + + + org.apache.commons + commons-lang3 + + + org.apache.logging.log4j + log4j-core + + + junit + junit + test + + + diff --git a/entitydb-eql/eql-language/src/main/antlr4/Eql.g4 b/entitydb-eql/eql-language/src/main/antlr4/Eql.g4 new file mode 100644 index 0000000..1ea0a30 --- /dev/null +++ b/entitydb-eql/eql-language/src/main/antlr4/Eql.g4 @@ -0,0 +1,56 @@ +grammar Eql; + +@header { + package ai.idylnlp.eql.antlr; +} + +command: + select (sort)? (';')? +// | graph (';')? + ; + +select: + 'select * from entities' + | 'select * from entities' (option)* + | 'select * from entities where' condition (WS 'and' condition)* + | 'select * from entities where' condition (WS 'and' condition)* (option)* + ; + +//graph: +// | 'select graph depth' WS value=INTEGERS WS 'from entities where' WS GRAPH_FIELD (WS)? '=' (WS)? value=STRING +// | 'select graph depth' WS value=INTEGERS WS 'from entities where' WS GRAPH_FIELD (WS)? '=' (WS)? value=STRING (WS 'and' condition)* (option)* +// ; + +condition: + WS NUMERIC_FIELD (WS)? operator=('='|'>'|'<'|'>='|'<=') (WS)? value=INTEGERS + | WS NUMERIC_FIELD WS 'between' WS value1=INTEGERS WS 'and' WS value2=INTEGERS + | WS STRING_FIELD (WS)? operator=('='|'!=') (WS)? value=STRING + | WS METADATA_FIELD WS value1=STRING (WS)? '=' (WS)? value2=STRING + ; + +option: + WS OPTION_FIELD WS value=INTEGERS + | WS OPTION_FIELD WS value=INTEGERS + ; + +sort: + WS 'order by' WS (NUMERIC_FIELD | STRING_FIELD) (WS SORT_ORDER_FIELD)? + ; + +METADATA_FIELD: 'metadata'; +NUMERIC_FIELD: 'confidence'; +OPTION_FIELD: 'limit' | 'offset' ; +STRING_FIELD: 'id' | 'context' | 'documentid' | 'text' | 'type' | 'uri' | 'language' ; +SORT_ORDER_FIELD: 'asc' | 'desc' ; +INTEGERS: ('0'..'9')+ ; + +STRING: '"' ('A'..'Z' | 'a'..'z' | '0'..'9' | '_' | '-' | '/' | ' ' | ':' | [.] )+ '"' + { + String s = getText(); + s = s.substring(1, s.length() - 1); // strip the leading and trailing quotes + s = s.replace("\"\"", "\""); // replace all double quotes with single quotes + setText(s); + } + ; + +WS: (' ' | '\t')+; \ No newline at end of file diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/Eql.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/Eql.java new file mode 100644 index 0000000..dd9a0e1 --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/Eql.java @@ -0,0 +1,345 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import ai.idylnlp.eql.antlr.EqlBaseListener; +import ai.idylnlp.eql.antlr.EqlLexer; +import ai.idylnlp.eql.antlr.EqlParser; +import ai.idylnlp.eql.antlr.EqlParser.ConditionContext; +import ai.idylnlp.eql.antlr.EqlParser.OptionContext; +import ai.idylnlp.eql.exceptions.QueryGenerationException; +import ai.idylnlp.eql.model.ConfidenceRange; +import ai.idylnlp.eql.model.EntityMetadataFilter; +import ai.idylnlp.eql.model.EntityOrder; +import ai.idylnlp.eql.model.EntityQuery; +import ai.idylnlp.eql.model.SortOrder; + +/** + * Entity Query Language (EQL) for querying entity stores. + * EQL is a domain-specific language (DSL) that provides + * a SQL-like interface for querying entity stores. + * + * The purpose of EQL is to provide a generic interface that + * can be used to query across all entity stores. + * + * @author Mountain Fog, Inc. + * + */ +public class Eql { + + private static final Logger LOGGER = LogManager.getLogger(Eql.class); + + private static final String CONTEXT = "context"; + private static final String DOCUMENTID = "documentid"; + private static final String TEXT = "text"; + private static final String CONFIDENCE = "confidence"; + private static final String LANGUAGE = "language"; + private static final String LIMIT = "limit"; + private static final String OFFSET = "offset"; + private static final String URI = "uri"; + private static final String TYPE = "type"; + + private Eql() { + // This is a utility class. + } + + /** + * Generates a {@link EntityQuery query} from an EQL query. + * @param eqlQuery An Entity Query Language query. + * @return A {@link EntityQuery query}. + */ + public static EntityQuery generate(String eqlQuery) throws QueryGenerationException { + + LOGGER.trace("Generating entity query from EQL statement: {}", eqlQuery); + + EntityQuery entityQuery = new EntityQuery(); + + List entityMetadataFilters = new LinkedList(); + + try { + + InputStream stream = new ByteArrayInputStream(eqlQuery.getBytes(StandardCharsets.UTF_8)); + + EqlLexer lexer = new EqlLexer(new ANTLRInputStream(stream)); + EqlParser parser = new EqlParser(new CommonTokenStream(lexer)); + + parser.addErrorListener(new BaseErrorListener() { + + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException ex) { + throw new IllegalStateException("Failed to parse EQL at line " + line + ". Message: " + msg + ".", ex); + } + + }); + + final AtomicReference command = new AtomicReference<>(); + + parser.addParseListener(new EqlBaseListener() { + + @Override + public void exitSort(EqlParser.SortContext ctx) { + + command.set(ctx.getText()); + + if(ctx.NUMERIC_FIELD() != null) { + + entityQuery.setEntityOrder(EntityOrder.CONFIDENCE); + + } else { + + // It is a STRING_FIELD. + + if(ctx.STRING_FIELD().getText().equalsIgnoreCase("text")) { + + entityQuery.setEntityOrder(EntityOrder.TEXT); + + } else if(ctx.STRING_FIELD().getText().equalsIgnoreCase("id")) { + + entityQuery.setEntityOrder(EntityOrder.ID); + + } else if(ctx.STRING_FIELD().getText().equalsIgnoreCase("extractiondate")) { + + entityQuery.setEntityOrder(EntityOrder.EXTRACTION_DATE); + + } else if(ctx.STRING_FIELD().getText().equalsIgnoreCase("type")) { + + entityQuery.setEntityOrder(EntityOrder.TYPE); + + } + + } + + if(ctx.SORT_ORDER_FIELD() != null) { + + if(ctx.SORT_ORDER_FIELD().getText().equalsIgnoreCase("asc")) { + + entityQuery.setSortOrder(SortOrder.ASC); + + } else if(ctx.SORT_ORDER_FIELD().getText().equalsIgnoreCase("desc")) { + + entityQuery.setSortOrder(SortOrder.DESC); + + } + + } + + } + + @Override + public void exitCommand(EqlParser.CommandContext ctx) { + + //LOGGER.debug("Command: " + ctx.getText()); + + command.set(ctx.getText()); + + } + + @Override + public void exitOption(OptionContext ctx) { + + if(ctx.OPTION_FIELD() != null && !StringUtils.isEmpty(ctx.OPTION_FIELD().getText())) { + + if(ctx.OPTION_FIELD().getText().equalsIgnoreCase(LIMIT)) { + + int limit = Integer.valueOf(ctx.INTEGERS().getText()); + + entityQuery.setLimit(limit); + + } else if(ctx.OPTION_FIELD().getText().equalsIgnoreCase(OFFSET)) { + + int limit = Integer.valueOf(ctx.INTEGERS().getText()); + + entityQuery.setOffset(limit); + + } + + } + + } + + @Override + public void exitCondition(ConditionContext ctx) { + + if(ctx.NUMERIC_FIELD() != null && !StringUtils.isEmpty(ctx.NUMERIC_FIELD().getText())) { + + if(ctx.NUMERIC_FIELD().getText().equalsIgnoreCase(CONFIDENCE)) { + + if(ctx.INTEGERS().size() == 1) { + + String operator = ctx.operator.getText(); + + double confidence = Double.valueOf(ctx.INTEGERS(0).getText()) / 100; + + if(operator.equals("=")) { + + // For when "confidence = 50" + entityQuery.setConfidenceRange(new ConfidenceRange(confidence, confidence)); + + } else if(operator.equals(">")) { + + // For when "confidence > 50" + confidence = Math.min(1, confidence + 0.01); + entityQuery.setConfidenceRange(new ConfidenceRange(confidence, 1.0)); + + } else if(operator.equalsIgnoreCase("<")) { + + // For when "confidence < 50" + confidence = Math.max(0, confidence - 0.01); + entityQuery.setConfidenceRange(new ConfidenceRange(0.0, confidence)); + + } else if(operator.equalsIgnoreCase("<=")) { + + // For when "confidence <= 50" + entityQuery.setConfidenceRange(new ConfidenceRange(0.0, confidence)); + + } else if(operator.equalsIgnoreCase(">=")) { + + // For when "confidence >= 50" + entityQuery.setConfidenceRange(new ConfidenceRange(confidence, 1.0)); + + } + + } else if(ctx.INTEGERS().size() == 2) { + + // For when "confidence between 10 and 50" is in the query. + double minConfidence = Double.valueOf(ctx.INTEGERS(0).getText()) / 100; + double maxConfidence = Double.valueOf(ctx.INTEGERS(1).getText()) / 100; + entityQuery.setConfidenceRange(new ConfidenceRange(minConfidence, maxConfidence)); + + } + + } + + } else if(ctx.STRING_FIELD() != null) { + + //LOGGER.debug("Field: " + ctx.STRING_FIELD().getText()); + //LOGGER.debug("Value: " + ctx.STRING(0).getText()); + + String operator = ctx.operator.getText(); + + if(ctx.STRING_FIELD().getText().equalsIgnoreCase(CONTEXT)) { + + if(operator.equals("=")) { + entityQuery.setContext(ctx.STRING(0).getText()); + } else { + entityQuery.setNotContext(ctx.STRING(0).getText()); + } + + } else if(ctx.STRING_FIELD().getText().equalsIgnoreCase(DOCUMENTID)) { + + if(operator.equals("=")) { + entityQuery.setDocumentId(ctx.STRING(0).getText()); + } else { + entityQuery.setNotDocumentId(ctx.STRING(0).getText()); + } + + } else if(ctx.STRING_FIELD().getText().equalsIgnoreCase(TEXT)) { + + if(operator.equals("=")) { + entityQuery.setText(ctx.STRING(0).getText()); + } else { + entityQuery.setNotText(ctx.STRING(0).getText()); + } + + } else if(ctx.STRING_FIELD().getText().equalsIgnoreCase(TYPE)) { + + if(operator.equals("=")) { + entityQuery.setType(ctx.STRING(0).getText()); + } else { + entityQuery.setNotType(ctx.STRING(0).getText()); + } + + } else if(ctx.STRING_FIELD().getText().equalsIgnoreCase(LANGUAGE)) { + + if(operator.equals("=")) { + entityQuery.setLanguageCode(ctx.STRING(0).getText()); + } else { + entityQuery.setNotLanguageCode(ctx.STRING(0).getText()); + } + + } else if(ctx.STRING_FIELD().getText().equalsIgnoreCase(URI)) { + + if(operator.equals("=")) { + entityQuery.setUri(ctx.STRING(0).getText()); + } else { + entityQuery.setNotUri(ctx.STRING(0).getText()); + } + + } + + } else if(ctx.METADATA_FIELD() != null && !StringUtils.isEmpty(ctx.METADATA_FIELD().getText())) { + + String metadataName = ctx.STRING().get(0).getText(); + String metadataValue = ctx.STRING().get(1).getText(); + + entityMetadataFilters.add(new EntityMetadataFilter(metadataName, metadataValue)); + + } + + } + + }); + + entityQuery.setEntityMetadataFilters(entityMetadataFilters); + + // Parse the EQL. + parser.command(); + + stream.close(); + + } catch (IOException ex) { + + throw new QueryGenerationException("Unable to generate query.", ex); + + } + + return entityQuery; + + } + +} diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/Eql.tokens b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/Eql.tokens new file mode 100644 index 0000000..d81e5d9 --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/Eql.tokens @@ -0,0 +1,34 @@ +T__9=3 +T__8=4 +T__7=5 +T__6=6 +T__5=7 +T__4=8 +SORT_ORDER_FIELD=17 +STRING_FIELD=16 +NUMERIC_FIELD=14 +OPTION_FIELD=15 +INTEGERS=18 +T__11=1 +T__3=9 +T__2=10 +STRING=19 +T__1=11 +T__0=12 +WS=20 +T__10=2 +METADATA_FIELD=13 +'metadata'=13 +'and'=6 +'between'=5 +'<='=1 +'>'=11 +'select * from entities where'=12 +'='=10 +'<'=9 +'>='=7 +';'=8 +'order by'=3 +'!='=4 +'confidence'=14 +'select * from entities'=2 diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlBaseListener.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlBaseListener.java new file mode 100644 index 0000000..1bcff9d --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlBaseListener.java @@ -0,0 +1,31 @@ +// Generated from Eql.g4 by ANTLR 4.0 + + package ai.idylnlp.eql.antlr; + + +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.antlr.v4.runtime.tree.ErrorNode; + +public class EqlBaseListener implements EqlListener { + @Override public void enterCondition(EqlParser.ConditionContext ctx) { } + @Override public void exitCondition(EqlParser.ConditionContext ctx) { } + + @Override public void enterSelect(EqlParser.SelectContext ctx) { } + @Override public void exitSelect(EqlParser.SelectContext ctx) { } + + @Override public void enterSort(EqlParser.SortContext ctx) { } + @Override public void exitSort(EqlParser.SortContext ctx) { } + + @Override public void enterCommand(EqlParser.CommandContext ctx) { } + @Override public void exitCommand(EqlParser.CommandContext ctx) { } + + @Override public void enterOption(EqlParser.OptionContext ctx) { } + @Override public void exitOption(EqlParser.OptionContext ctx) { } + + @Override public void enterEveryRule(ParserRuleContext ctx) { } + @Override public void exitEveryRule(ParserRuleContext ctx) { } + @Override public void visitTerminal(TerminalNode node) { } + @Override public void visitErrorNode(ErrorNode node) { } +} \ No newline at end of file diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlLexer.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlLexer.java new file mode 100644 index 0000000..a90a5eb --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlLexer.java @@ -0,0 +1,155 @@ +// Generated from Eql.g4 by ANTLR 4.0 + + package ai.idylnlp.eql.antlr; + +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.misc.*; + +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) +public class EqlLexer extends Lexer { + protected static final DFA[] _decisionToDFA; + protected static final PredictionContextCache _sharedContextCache = + new PredictionContextCache(); + public static final int + T__11=1, T__10=2, T__9=3, T__8=4, T__7=5, T__6=6, T__5=7, T__4=8, T__3=9, + T__2=10, T__1=11, T__0=12, METADATA_FIELD=13, NUMERIC_FIELD=14, OPTION_FIELD=15, + STRING_FIELD=16, SORT_ORDER_FIELD=17, INTEGERS=18, STRING=19, WS=20; + public static String[] modeNames = { + "DEFAULT_MODE" + }; + + public static final String[] tokenNames = { + "", + "'<='", "'select * from entities'", "'order by'", "'!='", "'between'", + "'and'", "'>='", "';'", "'<'", "'='", "'>'", "'select * from entities where'", + "'metadata'", "'confidence'", "OPTION_FIELD", "STRING_FIELD", "SORT_ORDER_FIELD", + "INTEGERS", "STRING", "WS" + }; + public static final String[] ruleNames = { + "T__11", "T__10", "T__9", "T__8", "T__7", "T__6", "T__5", "T__4", "T__3", + "T__2", "T__1", "T__0", "METADATA_FIELD", "NUMERIC_FIELD", "OPTION_FIELD", + "STRING_FIELD", "SORT_ORDER_FIELD", "INTEGERS", "STRING", "WS" + }; + + + public EqlLexer(CharStream input) { + super(input); + _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache); + } + + @Override + public String getGrammarFileName() { return "Eql.g4"; } + + @Override + public String[] getTokenNames() { return tokenNames; } + + @Override + public String[] getRuleNames() { return ruleNames; } + + @Override + public String[] getModeNames() { return modeNames; } + + @Override + public ATN getATN() { return _ATN; } + + @Override + public void action(RuleContext _localctx, int ruleIndex, int actionIndex) { + switch (ruleIndex) { + case 18: STRING_action((RuleContext)_localctx, actionIndex); break; + } + } + private void STRING_action(RuleContext _localctx, int actionIndex) { + switch (actionIndex) { + case 0: + String s = getText(); + s = s.substring(1, s.length() - 1); // strip the leading and trailing quotes + s = s.replace("\"\"", "\""); // replace all double quotes with single quotes + setText(s); + break; + } + } + + public static final String _serializedATN = + "\2\4\26\u00ea\b\1\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b"+ + "\t\b\4\t\t\t\4\n\t\n\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20"+ + "\t\20\4\21\t\21\4\22\t\22\4\23\t\23\4\24\t\24\4\25\t\25\3\2\3\2\3\2\3"+ + "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+ + "\3\3\3\3\3\3\3\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\5\3\5\3\5\3"+ + "\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\7\3\7\3\7\3\7\3\b\3\b\3\b\3\t\3\t\3\n"+ + "\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r"+ + "\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3"+ + "\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3"+ + "\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3"+ + "\20\3\20\3\20\5\20\u00a5\n\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+ + "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+ + "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+ + "\3\21\3\21\5\21\u00cd\n\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22\u00d6"+ + "\n\22\3\23\6\23\u00d9\n\23\r\23\16\23\u00da\3\24\3\24\6\24\u00df\n\24"+ + "\r\24\16\24\u00e0\3\24\3\24\3\24\3\25\6\25\u00e7\n\25\r\25\16\25\u00e8"+ + "\2\26\3\3\1\5\4\1\7\5\1\t\6\1\13\7\1\r\b\1\17\t\1\21\n\1\23\13\1\25\f"+ + "\1\27\r\1\31\16\1\33\17\1\35\20\1\37\21\1!\22\1#\23\1%\24\1\'\25\2)\26"+ + "\1\3\2\4\7\"\"/\2\2,-\7?\2\2-\4\3\2\2\2./\7u\2\2/\60\7g\2\2\60\61\7n\2\2\61\62\7"+ + "g\2\2\62\63\7e\2\2\63\64\7v\2\2\64\65\7\"\2\2\65\66\7,\2\2\66\67\7\"\2"+ + "\2\678\7h\2\289\7t\2\29:\7q\2\2:;\7o\2\2;<\7\"\2\2<=\7g\2\2=>\7p\2\2>"+ + "?\7v\2\2?@\7k\2\2@A\7v\2\2AB\7k\2\2BC\7g\2\2CD\7u\2\2D\6\3\2\2\2EF\7q"+ + "\2\2FG\7t\2\2GH\7f\2\2HI\7g\2\2IJ\7t\2\2JK\7\"\2\2KL\7d\2\2LM\7{\2\2M"+ + "\b\3\2\2\2NO\7#\2\2OP\7?\2\2P\n\3\2\2\2QR\7d\2\2RS\7g\2\2ST\7v\2\2TU\7"+ + "y\2\2UV\7g\2\2VW\7g\2\2WX\7p\2\2X\f\3\2\2\2YZ\7c\2\2Z[\7p\2\2[\\\7f\2"+ + "\2\\\16\3\2\2\2]^\7@\2\2^_\7?\2\2_\20\3\2\2\2`a\7=\2\2a\22\3\2\2\2bc\7"+ + ">\2\2c\24\3\2\2\2de\7?\2\2e\26\3\2\2\2fg\7@\2\2g\30\3\2\2\2hi\7u\2\2i"+ + "j\7g\2\2jk\7n\2\2kl\7g\2\2lm\7e\2\2mn\7v\2\2no\7\"\2\2op\7,\2\2pq\7\""+ + "\2\2qr\7h\2\2rs\7t\2\2st\7q\2\2tu\7o\2\2uv\7\"\2\2vw\7g\2\2wx\7p\2\2x"+ + "y\7v\2\2yz\7k\2\2z{\7v\2\2{|\7k\2\2|}\7g\2\2}~\7u\2\2~\177\7\"\2\2\177"+ + "\u0080\7y\2\2\u0080\u0081\7j\2\2\u0081\u0082\7g\2\2\u0082\u0083\7t\2\2"+ + "\u0083\u0084\7g\2\2\u0084\32\3\2\2\2\u0085\u0086\7o\2\2\u0086\u0087\7"+ + "g\2\2\u0087\u0088\7v\2\2\u0088\u0089\7c\2\2\u0089\u008a\7f\2\2\u008a\u008b"+ + "\7c\2\2\u008b\u008c\7v\2\2\u008c\u008d\7c\2\2\u008d\34\3\2\2\2\u008e\u008f"+ + "\7e\2\2\u008f\u0090\7q\2\2\u0090\u0091\7p\2\2\u0091\u0092\7h\2\2\u0092"+ + "\u0093\7k\2\2\u0093\u0094\7f\2\2\u0094\u0095\7g\2\2\u0095\u0096\7p\2\2"+ + "\u0096\u0097\7e\2\2\u0097\u0098\7g\2\2\u0098\36\3\2\2\2\u0099\u009a\7"+ + "n\2\2\u009a\u009b\7k\2\2\u009b\u009c\7o\2\2\u009c\u009d\7k\2\2\u009d\u00a5"+ + "\7v\2\2\u009e\u009f\7q\2\2\u009f\u00a0\7h\2\2\u00a0\u00a1\7h\2\2\u00a1"+ + "\u00a2\7u\2\2\u00a2\u00a3\7g\2\2\u00a3\u00a5\7v\2\2\u00a4\u0099\3\2\2"+ + "\2\u00a4\u009e\3\2\2\2\u00a5 \3\2\2\2\u00a6\u00a7\7k\2\2\u00a7\u00cd\7"+ + "f\2\2\u00a8\u00a9\7e\2\2\u00a9\u00aa\7q\2\2\u00aa\u00ab\7p\2\2\u00ab\u00ac"+ + "\7v\2\2\u00ac\u00ad\7g\2\2\u00ad\u00ae\7z\2\2\u00ae\u00cd\7v\2\2\u00af"+ + "\u00b0\7f\2\2\u00b0\u00b1\7q\2\2\u00b1\u00b2\7e\2\2\u00b2\u00b3\7w\2\2"+ + "\u00b3\u00b4\7o\2\2\u00b4\u00b5\7g\2\2\u00b5\u00b6\7p\2\2\u00b6\u00b7"+ + "\7v\2\2\u00b7\u00b8\7k\2\2\u00b8\u00cd\7f\2\2\u00b9\u00ba\7v\2\2\u00ba"+ + "\u00bb\7g\2\2\u00bb\u00bc\7z\2\2\u00bc\u00cd\7v\2\2\u00bd\u00be\7v\2\2"+ + "\u00be\u00bf\7{\2\2\u00bf\u00c0\7r\2\2\u00c0\u00cd\7g\2\2\u00c1\u00c2"+ + "\7w\2\2\u00c2\u00c3\7t\2\2\u00c3\u00cd\7k\2\2\u00c4\u00c5\7n\2\2\u00c5"+ + "\u00c6\7c\2\2\u00c6\u00c7\7p\2\2\u00c7\u00c8\7i\2\2\u00c8\u00c9\7w\2\2"+ + "\u00c9\u00ca\7c\2\2\u00ca\u00cb\7i\2\2\u00cb\u00cd\7g\2\2\u00cc\u00a6"+ + "\3\2\2\2\u00cc\u00a8\3\2\2\2\u00cc\u00af\3\2\2\2\u00cc\u00b9\3\2\2\2\u00cc"+ + "\u00bd\3\2\2\2\u00cc\u00c1\3\2\2\2\u00cc\u00c4\3\2\2\2\u00cd\"\3\2\2\2"+ + "\u00ce\u00cf\7c\2\2\u00cf\u00d0\7u\2\2\u00d0\u00d6\7e\2\2\u00d1\u00d2"+ + "\7f\2\2\u00d2\u00d3\7g\2\2\u00d3\u00d4\7u\2\2\u00d4\u00d6\7e\2\2\u00d5"+ + "\u00ce\3\2\2\2\u00d5\u00d1\3\2\2\2\u00d6$\3\2\2\2\u00d7\u00d9\4\62;\2"+ + "\u00d8\u00d7\3\2\2\2\u00d9\u00da\3\2\2\2\u00da\u00d8\3\2\2\2\u00da\u00db"+ + "\3\2\2\2\u00db&\3\2\2\2\u00dc\u00de\7$\2\2\u00dd\u00df\t\2\2\2\u00de\u00dd"+ + "\3\2\2\2\u00df\u00e0\3\2\2\2\u00e0\u00de\3\2\2\2\u00e0\u00e1\3\2\2\2\u00e1"+ + "\u00e2\3\2\2\2\u00e2\u00e3\7$\2\2\u00e3\u00e4\b\24\2\2\u00e4(\3\2\2\2"+ + "\u00e5\u00e7\t\3\2\2\u00e6\u00e5\3\2\2\2\u00e7\u00e8\3\2\2\2\u00e8\u00e6"+ + "\3\2\2\2\u00e8\u00e9\3\2\2\2\u00e9*\3\2\2\2\n\2\u00a4\u00cc\u00d5\u00da"+ + "\u00de\u00e0\u00e8"; + public static final ATN _ATN = + ATNSimulator.deserialize(_serializedATN.toCharArray()); + static { + _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()]; + } +} \ No newline at end of file diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlLexer.tokens b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlLexer.tokens new file mode 100644 index 0000000..d81e5d9 --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlLexer.tokens @@ -0,0 +1,34 @@ +T__9=3 +T__8=4 +T__7=5 +T__6=6 +T__5=7 +T__4=8 +SORT_ORDER_FIELD=17 +STRING_FIELD=16 +NUMERIC_FIELD=14 +OPTION_FIELD=15 +INTEGERS=18 +T__11=1 +T__3=9 +T__2=10 +STRING=19 +T__1=11 +T__0=12 +WS=20 +T__10=2 +METADATA_FIELD=13 +'metadata'=13 +'and'=6 +'between'=5 +'<='=1 +'>'=11 +'select * from entities where'=12 +'='=10 +'<'=9 +'>='=7 +';'=8 +'order by'=3 +'!='=4 +'confidence'=14 +'select * from entities'=2 diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlListener.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlListener.java new file mode 100644 index 0000000..29c3b67 --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlListener.java @@ -0,0 +1,23 @@ +// Generated from Eql.g4 by ANTLR 4.0 + + package ai.idylnlp.eql.antlr; + +import org.antlr.v4.runtime.tree.*; +import org.antlr.v4.runtime.Token; + +public interface EqlListener extends ParseTreeListener { + void enterCondition(EqlParser.ConditionContext ctx); + void exitCondition(EqlParser.ConditionContext ctx); + + void enterSelect(EqlParser.SelectContext ctx); + void exitSelect(EqlParser.SelectContext ctx); + + void enterSort(EqlParser.SortContext ctx); + void exitSort(EqlParser.SortContext ctx); + + void enterCommand(EqlParser.CommandContext ctx); + void exitCommand(EqlParser.CommandContext ctx); + + void enterOption(EqlParser.OptionContext ctx); + void exitOption(EqlParser.OptionContext ctx); +} \ No newline at end of file diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlParser.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlParser.java new file mode 100644 index 0000000..513bf1b --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/antlr/EqlParser.java @@ -0,0 +1,572 @@ +// Generated from Eql.g4 by ANTLR 4.0 + + package ai.idylnlp.eql.antlr; + +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.misc.*; +import org.antlr.v4.runtime.tree.*; +import java.util.List; +import java.util.Iterator; +import java.util.ArrayList; + +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) +public class EqlParser extends Parser { + protected static final DFA[] _decisionToDFA; + protected static final PredictionContextCache _sharedContextCache = + new PredictionContextCache(); + public static final int + T__11=1, T__10=2, T__9=3, T__8=4, T__7=5, T__6=6, T__5=7, T__4=8, T__3=9, + T__2=10, T__1=11, T__0=12, METADATA_FIELD=13, NUMERIC_FIELD=14, OPTION_FIELD=15, + STRING_FIELD=16, SORT_ORDER_FIELD=17, INTEGERS=18, STRING=19, WS=20; + public static final String[] tokenNames = { + "", "'<='", "'select * from entities'", "'order by'", "'!='", + "'between'", "'and'", "'>='", "';'", "'<'", "'='", "'>'", "'select * from entities where'", + "'metadata'", "'confidence'", "OPTION_FIELD", "STRING_FIELD", "SORT_ORDER_FIELD", + "INTEGERS", "STRING", "WS" + }; + public static final int + RULE_command = 0, RULE_select = 1, RULE_condition = 2, RULE_option = 3, + RULE_sort = 4; + public static final String[] ruleNames = { + "command", "select", "condition", "option", "sort" + }; + + @Override + public String getGrammarFileName() { return "Eql.g4"; } + + @Override + public String[] getTokenNames() { return tokenNames; } + + @Override + public String[] getRuleNames() { return ruleNames; } + + @Override + public ATN getATN() { return _ATN; } + + public EqlParser(TokenStream input) { + super(input); + _interp = new ParserATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache); + } + public static class CommandContext extends ParserRuleContext { + public SelectContext select() { + return getRuleContext(SelectContext.class,0); + } + public SortContext sort() { + return getRuleContext(SortContext.class,0); + } + public CommandContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_command; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).enterCommand(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).exitCommand(this); + } + } + + public final CommandContext command() throws RecognitionException { + CommandContext _localctx = new CommandContext(_ctx, getState()); + enterRule(_localctx, 0, RULE_command); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(10); select(); + setState(12); + _la = _input.LA(1); + if (_la==WS) { + { + setState(11); sort(); + } + } + + setState(15); + _la = _input.LA(1); + if (_la==8) { + { + setState(14); match(8); + } + } + + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class SelectContext extends ParserRuleContext { + public ConditionContext condition(int i) { + return getRuleContext(ConditionContext.class,i); + } + public List condition() { + return getRuleContexts(ConditionContext.class); + } + public OptionContext option(int i) { + return getRuleContext(OptionContext.class,i); + } + public TerminalNode WS(int i) { + return getToken(EqlParser.WS, i); + } + public List WS() { return getTokens(EqlParser.WS); } + public List option() { + return getRuleContexts(OptionContext.class); + } + public SelectContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_select; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).enterSelect(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).exitSelect(this); + } + } + + public final SelectContext select() throws RecognitionException { + SelectContext _localctx = new SelectContext(_ctx, getState()); + enterRule(_localctx, 2, RULE_select); + try { + int _alt; + setState(51); + switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(17); match(2); + } + break; + + case 2: + enterOuterAlt(_localctx, 2); + { + setState(18); match(2); + setState(22); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,2,_ctx); + while ( _alt!=2 && _alt!=-1 ) { + if ( _alt==1 ) { + { + { + setState(19); option(); + } + } + } + setState(24); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,2,_ctx); + } + } + break; + + case 3: + enterOuterAlt(_localctx, 3); + { + setState(25); match(12); + setState(26); condition(); + setState(32); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,3,_ctx); + while ( _alt!=2 && _alt!=-1 ) { + if ( _alt==1 ) { + { + { + setState(27); match(WS); + setState(28); match(6); + setState(29); condition(); + } + } + } + setState(34); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,3,_ctx); + } + } + break; + + case 4: + enterOuterAlt(_localctx, 4); + { + setState(35); match(12); + setState(36); condition(); + setState(42); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,4,_ctx); + while ( _alt!=2 && _alt!=-1 ) { + if ( _alt==1 ) { + { + { + setState(37); match(WS); + setState(38); match(6); + setState(39); condition(); + } + } + } + setState(44); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,4,_ctx); + } + setState(48); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,5,_ctx); + while ( _alt!=2 && _alt!=-1 ) { + if ( _alt==1 ) { + { + { + setState(45); option(); + } + } + } + setState(50); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,5,_ctx); + } + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ConditionContext extends ParserRuleContext { + public Token operator; + public Token value; + public Token value1; + public Token value2; + public TerminalNode STRING_FIELD() { return getToken(EqlParser.STRING_FIELD, 0); } + public TerminalNode NUMERIC_FIELD() { return getToken(EqlParser.NUMERIC_FIELD, 0); } + public List INTEGERS() { return getTokens(EqlParser.INTEGERS); } + public TerminalNode INTEGERS(int i) { + return getToken(EqlParser.INTEGERS, i); + } + public List STRING() { return getTokens(EqlParser.STRING); } + public TerminalNode STRING(int i) { + return getToken(EqlParser.STRING, i); + } + public TerminalNode WS(int i) { + return getToken(EqlParser.WS, i); + } + public List WS() { return getTokens(EqlParser.WS); } + public TerminalNode METADATA_FIELD() { return getToken(EqlParser.METADATA_FIELD, 0); } + public ConditionContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_condition; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).enterCondition(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).exitCondition(this); + } + } + + public final ConditionContext condition() throws RecognitionException { + ConditionContext _localctx = new ConditionContext(_ctx, getState()); + enterRule(_localctx, 4, RULE_condition); + int _la; + try { + setState(95); + switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(53); match(WS); + setState(54); match(NUMERIC_FIELD); + setState(56); + _la = _input.LA(1); + if (_la==WS) { + { + setState(55); match(WS); + } + } + + setState(58); + ((ConditionContext)_localctx).operator = _input.LT(1); + _la = _input.LA(1); + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << 1) | (1L << 7) | (1L << 9) | (1L << 10) | (1L << 11))) != 0)) ) { + ((ConditionContext)_localctx).operator = (Token)_errHandler.recoverInline(this); + } + consume(); + setState(60); + _la = _input.LA(1); + if (_la==WS) { + { + setState(59); match(WS); + } + } + + setState(62); ((ConditionContext)_localctx).value = match(INTEGERS); + } + break; + + case 2: + enterOuterAlt(_localctx, 2); + { + setState(63); match(WS); + setState(64); match(NUMERIC_FIELD); + setState(65); match(WS); + setState(66); match(5); + setState(67); match(WS); + setState(68); ((ConditionContext)_localctx).value1 = match(INTEGERS); + setState(69); match(WS); + setState(70); match(6); + setState(71); match(WS); + setState(72); ((ConditionContext)_localctx).value2 = match(INTEGERS); + } + break; + + case 3: + enterOuterAlt(_localctx, 3); + { + setState(73); match(WS); + setState(74); match(STRING_FIELD); + setState(76); + _la = _input.LA(1); + if (_la==WS) { + { + setState(75); match(WS); + } + } + + setState(78); + ((ConditionContext)_localctx).operator = _input.LT(1); + _la = _input.LA(1); + if ( !(_la==4 || _la==10) ) { + ((ConditionContext)_localctx).operator = (Token)_errHandler.recoverInline(this); + } + consume(); + setState(80); + _la = _input.LA(1); + if (_la==WS) { + { + setState(79); match(WS); + } + } + + setState(82); ((ConditionContext)_localctx).value = match(STRING); + } + break; + + case 4: + enterOuterAlt(_localctx, 4); + { + setState(83); match(WS); + setState(84); match(METADATA_FIELD); + setState(85); match(WS); + setState(86); ((ConditionContext)_localctx).value1 = match(STRING); + setState(88); + _la = _input.LA(1); + if (_la==WS) { + { + setState(87); match(WS); + } + } + + setState(90); match(10); + setState(92); + _la = _input.LA(1); + if (_la==WS) { + { + setState(91); match(WS); + } + } + + setState(94); ((ConditionContext)_localctx).value2 = match(STRING); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class OptionContext extends ParserRuleContext { + public Token value; + public TerminalNode OPTION_FIELD() { return getToken(EqlParser.OPTION_FIELD, 0); } + public TerminalNode INTEGERS() { return getToken(EqlParser.INTEGERS, 0); } + public TerminalNode WS(int i) { + return getToken(EqlParser.WS, i); + } + public List WS() { return getTokens(EqlParser.WS); } + public OptionContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_option; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).enterOption(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).exitOption(this); + } + } + + public final OptionContext option() throws RecognitionException { + OptionContext _localctx = new OptionContext(_ctx, getState()); + enterRule(_localctx, 6, RULE_option); + try { + setState(105); + switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(97); match(WS); + setState(98); match(OPTION_FIELD); + setState(99); match(WS); + setState(100); ((OptionContext)_localctx).value = match(INTEGERS); + } + break; + + case 2: + enterOuterAlt(_localctx, 2); + { + setState(101); match(WS); + setState(102); match(OPTION_FIELD); + setState(103); match(WS); + setState(104); ((OptionContext)_localctx).value = match(INTEGERS); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class SortContext extends ParserRuleContext { + public TerminalNode STRING_FIELD() { return getToken(EqlParser.STRING_FIELD, 0); } + public TerminalNode NUMERIC_FIELD() { return getToken(EqlParser.NUMERIC_FIELD, 0); } + public TerminalNode WS(int i) { + return getToken(EqlParser.WS, i); + } + public List WS() { return getTokens(EqlParser.WS); } + public TerminalNode SORT_ORDER_FIELD() { return getToken(EqlParser.SORT_ORDER_FIELD, 0); } + public SortContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_sort; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).enterSort(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EqlListener ) ((EqlListener)listener).exitSort(this); + } + } + + public final SortContext sort() throws RecognitionException { + SortContext _localctx = new SortContext(_ctx, getState()); + enterRule(_localctx, 8, RULE_sort); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(107); match(WS); + setState(108); match(3); + setState(109); match(WS); + setState(110); + _la = _input.LA(1); + if ( !(_la==NUMERIC_FIELD || _la==STRING_FIELD) ) { + _errHandler.recoverInline(this); + } + consume(); + setState(113); + _la = _input.LA(1); + if (_la==WS) { + { + setState(111); match(WS); + setState(112); match(SORT_ORDER_FIELD); + } + } + + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static final String _serializedATN = + "\2\3\26v\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\3\2\3\2\5\2\17\n\2\3"+ + "\2\5\2\22\n\2\3\3\3\3\3\3\7\3\27\n\3\f\3\16\3\32\13\3\3\3\3\3\3\3\3\3"+ + "\3\3\7\3!\n\3\f\3\16\3$\13\3\3\3\3\3\3\3\3\3\3\3\7\3+\n\3\f\3\16\3.\13"+ + "\3\3\3\7\3\61\n\3\f\3\16\3\64\13\3\5\3\66\n\3\3\4\3\4\3\4\5\4;\n\4\3\4"+ + "\3\4\5\4?\n\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4"+ + "\5\4O\n\4\3\4\3\4\5\4S\n\4\3\4\3\4\3\4\3\4\3\4\3\4\5\4[\n\4\3\4\3\4\5"+ + "\4_\n\4\3\4\5\4b\n\4\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5l\n\5\3\6\3\6"+ + "\3\6\3\6\3\6\3\6\5\6t\n\6\3\6\2\7\2\4\6\b\n\2\5\5\3\3\t\t\13\r\4\6\6\f"+ + "\f\4\20\20\22\22\u0084\2\f\3\2\2\2\4\65\3\2\2\2\6a\3\2\2\2\bk\3\2\2\2"+ + "\nm\3\2\2\2\f\16\5\4\3\2\r\17\5\n\6\2\16\r\3\2\2\2\16\17\3\2\2\2\17\21"+ + "\3\2\2\2\20\22\7\n\2\2\21\20\3\2\2\2\21\22\3\2\2\2\22\3\3\2\2\2\23\66"+ + "\7\4\2\2\24\30\7\4\2\2\25\27\5\b\5\2\26\25\3\2\2\2\27\32\3\2\2\2\30\26"+ + "\3\2\2\2\30\31\3\2\2\2\31\66\3\2\2\2\32\30\3\2\2\2\33\34\7\16\2\2\34\""+ + "\5\6\4\2\35\36\7\26\2\2\36\37\7\b\2\2\37!\5\6\4\2 \35\3\2\2\2!$\3\2\2"+ + "\2\" \3\2\2\2\"#\3\2\2\2#\66\3\2\2\2$\"\3\2\2\2%&\7\16\2\2&,\5\6\4\2\'"+ + "(\7\26\2\2()\7\b\2\2)+\5\6\4\2*\'\3\2\2\2+.\3\2\2\2,*\3\2\2\2,-\3\2\2"+ + "\2-\62\3\2\2\2.,\3\2\2\2/\61\5\b\5\2\60/\3\2\2\2\61\64\3\2\2\2\62\60\3"+ + "\2\2\2\62\63\3\2\2\2\63\66\3\2\2\2\64\62\3\2\2\2\65\23\3\2\2\2\65\24\3"+ + "\2\2\2\65\33\3\2\2\2\65%\3\2\2\2\66\5\3\2\2\2\678\7\26\2\28:\7\20\2\2"+ + "9;\7\26\2\2:9\3\2\2\2:;\3\2\2\2;<\3\2\2\2<>\t\2\2\2=?\7\26\2\2>=\3\2\2"+ + "\2>?\3\2\2\2?@\3\2\2\2@b\7\24\2\2AB\7\26\2\2BC\7\20\2\2CD\7\26\2\2DE\7"+ + "\7\2\2EF\7\26\2\2FG\7\24\2\2GH\7\26\2\2HI\7\b\2\2IJ\7\26\2\2Jb\7\24\2"+ + "\2KL\7\26\2\2LN\7\22\2\2MO\7\26\2\2NM\3\2\2\2NO\3\2\2\2OP\3\2\2\2PR\t"+ + "\3\2\2QS\7\26\2\2RQ\3\2\2\2RS\3\2\2\2ST\3\2\2\2Tb\7\25\2\2UV\7\26\2\2"+ + "VW\7\17\2\2WX\7\26\2\2XZ\7\25\2\2Y[\7\26\2\2ZY\3\2\2\2Z[\3\2\2\2[\\\3"+ + "\2\2\2\\^\7\f\2\2]_\7\26\2\2^]\3\2\2\2^_\3\2\2\2_`\3\2\2\2`b\7\25\2\2"+ + "a\67\3\2\2\2aA\3\2\2\2aK\3\2\2\2aU\3\2\2\2b\7\3\2\2\2cd\7\26\2\2de\7\21"+ + "\2\2ef\7\26\2\2fl\7\24\2\2gh\7\26\2\2hi\7\21\2\2ij\7\26\2\2jl\7\24\2\2"+ + "kc\3\2\2\2kg\3\2\2\2l\t\3\2\2\2mn\7\26\2\2no\7\5\2\2op\7\26\2\2ps\t\4"+ + "\2\2qr\7\26\2\2rt\7\23\2\2sq\3\2\2\2st\3\2\2\2t\13\3\2\2\2\22\16\21\30"+ + "\",\62\65:>NRZ^aks"; + public static final ATN _ATN = + ATNSimulator.deserialize(_serializedATN.toCharArray()); + static { + _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()]; + } +} \ No newline at end of file diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/exceptions/QueryGenerationException.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/exceptions/QueryGenerationException.java new file mode 100644 index 0000000..ad3061d --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/exceptions/QueryGenerationException.java @@ -0,0 +1,61 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.exceptions; + +/** + * An exception thrown when generating an {@link EntityQuery query} + * from an EQL query. + * + * @author Mountain Fog, Inc. + * + */ +public class QueryGenerationException extends RuntimeException { + + private static final long serialVersionUID = 2407358322163930971L; + + /** + * Creates a new exception. + * @param message The message of the exception. + */ + public QueryGenerationException(String message) { + super(message); + } + + /** + * Creates a new exception. + * @param message The message of the exception. + * @param throwable The exception. + */ + public QueryGenerationException(String message, Throwable throwable) { + super(message, throwable); + } + +} diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/ConfidenceRange.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/ConfidenceRange.java new file mode 100644 index 0000000..362a0c1 --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/ConfidenceRange.java @@ -0,0 +1,100 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.model; + +/** + * A confidence range for a query. + * + * @author Mountain Fog, Inc. + * + */ +public class ConfidenceRange { + + private double minimum = 0.0; + private double maximum = 100.0; + + @Override + public String toString() { + return "Minimum: " + minimum + "; Maximum: " + maximum; + } + + /** + * Creates a new confidence range with the + * minimum value set to 0.0 and the maximum + * value set to 100.0. + */ + public ConfidenceRange() { + + } + + /** + * Creates a new confidence range using a single value. + * @param confidence The confidence value. + */ + public ConfidenceRange(double confidence) { + + this.minimum = confidence; + this.maximum = confidence; + + } + + /** + * Creates a new confidence range. The provided + * minimum and maximum values are inclusive. + * @param minimum The minimum value. + * @param maximum The maximum value. + */ + public ConfidenceRange(double minimum, double maximum) { + + this.minimum = minimum; + this.maximum = maximum; + + } + + /** + * Gets the minimum value. When used in a query this value + * is used as "greater than or equal to" this value. + * @return The minimum value. + */ + public double getMinimum() { + return minimum; + } + + /** + * Gets the maximum value. When used in a query this value + * is used as "less than or equal to" this value. + * @return The maximum value. + */ + public double getMaximum() { + return maximum; + } + +} diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/EntityMetadataFilter.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/EntityMetadataFilter.java new file mode 100644 index 0000000..b982a4d --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/EntityMetadataFilter.java @@ -0,0 +1,168 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.model; + +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; + +/** + * Used to query on entity metadata in an entity store. + * + * @author Mountain Fog, Inc. + * + */ +public class EntityMetadataFilter { + + private String name; + private String value; + private boolean isCaseSensitive = false; + private String comparator = "equals"; + + /** + * Creates a new {@link StoredEntityMetadata}. + * @param name The name of the attribute. + * @param value The value of the attribute. + */ + public EntityMetadataFilter(String name, String value) { + + this.name = name; + this.value = value; + + } + + /** + * Creates a new {@link StoredEntityMetadata}. + * @param name The name of the attribute. + * @param value The value of the attribute. + * @param isCaseSensitive Set to true to do a case-sensitive query. + */ + public EntityMetadataFilter(String name, String value, boolean isCaseSensitive) { + + this.name = name; + this.value = value; + this.isCaseSensitive = isCaseSensitive; + + } + + /** + * Creates a new {@link EntityMetadataFilter}. + * @param name The name of the attribute. + * @param value The value of the attribute. + * @param isCaseSensitive Set to true to do a case-sensitive query. + * @param comparator How to do the comparison. + */ + public EntityMetadataFilter(String name, String value, boolean isCaseSensitive, String comparator) { + + this.name = name; + this.value = value; + this.isCaseSensitive = isCaseSensitive; + this.comparator = comparator; + + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return ReflectionToStringBuilder.toString(this); + } + + /** + * Gets the name of the attribute. + * @return The name of the attribute. + */ + public String getName() { + return name; + } + + /** + * Sets the name of the attribute. + * @param name The name of the attribute. + */ + public void setName(String name) { + this.name = name; + } + + /** + * Gets the value of the attribute. + * @return The value of the attribute. + */ + public String getValue() { + return value; + } + + /** + * Sets the value of the attribute. + * @param value The value of the attribute. + */ + public void setValue(String value) { + this.value = value; + } + + /** + * Gets if the query on metadata should be case-sensitive. Not all + * entity stores will be able to respect this value. Refer to the + * entity store implementation documentation to see if the entity store + * supports case-sensitive querying. + * @return True if the query on metadata should be case-sensitive. + */ + public boolean isCaseSensitive() { + return isCaseSensitive; + } + + /** + * Sets if the query on metadata should be case-sensitive. Not all + * entity stores will be able to respect this value. Refer to the + * entity store implementation documentation to see if the entity store + * supports case-sensitive querying. + * @param isCaseSensitive True if the query on metadata should be case-sensitive. + */ + public void setCaseSensitive(boolean isCaseSensitive) { + this.isCaseSensitive = isCaseSensitive; + } + + /** + * Gets the comparator for the query. + * @return The comparator for the query. + */ + public String getComparator() { + return comparator; + } + + /** + * Sets the comparator for the query. + * @param comparator THe comparator for the query. + */ + public void setComparator(String comparator) { + this.comparator = comparator; + } + +} diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/EntityOrder.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/EntityOrder.java new file mode 100644 index 0000000..75006f6 --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/EntityOrder.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.model; + +/** + * Specifies the field to order by in an + * {@link EntityQuery query}. + * + * @author Mountain Fog, Inc. + * + */ +public enum EntityOrder { + + /** + * Sort by the database-assigned ID. + */ + ID("id"), + + /** + * Sort by the entity text. + */ + TEXT("text"), + + /** + * Sort by the entity confidence. + */ + CONFIDENCE("confidence"), + + /** + * Sort by the type of the entity. + */ + TYPE("type"), + + /** + * Sort by the entity's extraction date. + */ + EXTRACTION_DATE("extractionDate"); + + private String property; + + private EntityOrder(String property) { + + this.property = property; + + } + + /** + * Gets the name of the property to sort by. + * @return The name of the property. + */ + public String getProperty() { + return property; + } + +} diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/EntityQuery.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/EntityQuery.java new file mode 100644 index 0000000..2b3ad15 --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/EntityQuery.java @@ -0,0 +1,455 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.model; + +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import ai.idylnlp.model.entity.Entity; + +/** + * A query to be executed by an {@link EntityStore}. At least one property + * of this class must be provided or all stored entities will be returned + * by the query. If the {@link EntityOrder} is not set the default sort + * order is the entity text. Searches are not case-sensitive by default. + * + * @author Mountain Fog, Inc. + * + */ +public class EntityQuery { + + private static final Logger LOGGER = LogManager.getLogger(EntityQuery.class); + + private ConfidenceRange confidenceRange; + + private String text; + private String notText; + + private String type; + private String notType; + + private String languageCode; + private String notLanguageCode; + + private String context; + private String notContext; + + private String documentId; + private String notDocumentId; + + private String uri; + private String notUri; + + private int limit = 25; + private int offset = 0; + + private List entityMetadataFilters; + private EntityOrder entityOrder = EntityOrder.ID; + private SortOrder sortOrder = SortOrder.DESC; + + public boolean isMatch(Entity entity) { + + boolean match = false; + + if(confidenceRange != null + && confidenceRange.getMinimum() <= entity.getConfidence() + && entity.getConfidence() <= confidenceRange.getMaximum()) { + + LOGGER.debug("Entity confidence: {}", entity.getConfidence()); + LOGGER.debug("Range: {} to {}", confidenceRange.getMinimum(), confidenceRange.getMaximum()); + + match = true; + + } + + if(StringUtils.isNotEmpty(text)) { + + if(StringUtils.equalsIgnoreCase(text, entity.getText())) { + + match = true; + + } else { + + match = false; + + } + + } + + if(StringUtils.isNotEmpty(type)) { + + if(StringUtils.equalsIgnoreCase(text, entity.getType())) { + + match = true; + + } else { + + match = false; + + } + + } + + if(StringUtils.isNotEmpty(context)) { + + if(StringUtils.equalsIgnoreCase(context, entity.getContext())) { + + match = true; + + } else { + + match = false; + + } + + } + + if(StringUtils.isNotEmpty(documentId)) { + + if(StringUtils.equalsIgnoreCase(documentId, entity.getDocumentId())) { + + match = true; + + } else { + + match = false; + + } + + } + + if(StringUtils.isNotEmpty(uri)) { + + if(StringUtils.equalsIgnoreCase(uri, entity.getUri())) { + + match = true; + + } else { + + match = false; + + } + + } + + if(StringUtils.isNotEmpty(languageCode)) { + + if(StringUtils.equalsIgnoreCase(languageCode, entity.getLanguageCode())) { + + match = true; + + } else { + + match = false; + + } + + } + + return match; + + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return ReflectionToStringBuilder.toString(this); + } + + /** + * Gets the confidence range for the query. + * @return The {@link ConfidenceRange}. + */ + public ConfidenceRange getConfidenceRange() { + return confidenceRange; + } + + /** + * Sets the confidence range for the query. + * @param confidenceRange The {@link ConfidenceRange}. + */ + public void setConfidenceRange(ConfidenceRange confidenceRange) { + this.confidenceRange = confidenceRange; + } + + /** + * Sets the confidence range for the query. + * @param minimum The minimum confidence value. + * @param maximum The maximum confidence value. + */ + public void setConfidenceRange(double minimum, double maximum) { + this.confidenceRange = new ConfidenceRange(minimum, maximum); + } + + /** + * Gets the entity text for the query. + * @return The entity text. + */ + public String getText() { + return text; + } + + /** + * Sets the entity text for the query. Refer to the implementation + * of the {@link EntityStore} to see if wildcard characters + * are allowed. + * @param text The entity text. + */ + public void setText(String text) { + this.text = text; + } + + /** + * The type of entities to be queried. + * @return The entity type. + */ + public String getType() { + return type; + } + + /** + * Set the type of entities to be queried. + * @param type The entity type. + */ + public void setType(String type) { + this.type = type.toLowerCase(); + } + + /** + * Gets the sort {@link EntityOrder order}. If null the ordering is by the entity ID. + * @return The sort order. + */ + public EntityOrder getEntityOrder() { + + if(entityOrder == null) { + return EntityOrder.ID; + } else { + return entityOrder; + } + } + + /** + * Sets the sort {@link EntityOrder order}. + * @param entityOrder The sort order. + */ + public void setEntityOrder(EntityOrder entityOrder) { + this.entityOrder = entityOrder; + } + + /** + * Gets the context. + * @return The context. + */ + public String getContext() { + return context; + } + + /** + * Sets the context. + * @param context The context. + */ + public void setContext(String context) { + this.context = context; + } + + /** + * Gets the list of entity metadata filters for the query. + * @return A list of {@link EntityMetadataFilter}. + */ + public List getEntityMetadataFilters() { + return entityMetadataFilters; + } + + /** + * Sets the list of entity metadata filters for the query. + * @param entityMetadataFilters A list of {@link EntityMetadataFilter}. + */ + public void setEntityMetadataFilters(List entityMetadataFilters) { + this.entityMetadataFilters = entityMetadataFilters; + } + + /** + * Gets the document ID. + * @return A document ID. + */ + public String getDocumentId() { + return documentId; + } + + /** + * Sets the document ID. + * @param documentId The document ID. + */ + public void setDocumentId(String documentId) { + this.documentId = documentId; + } + + /** + * Gets the limit of results to return. + * @return The limit of results to return. + */ + public int getLimit() { + return limit; + } + + /** + * Sets the limit of results to return. + * @param limit The limit of results to return. + */ + public void setLimit(int limit) { + this.limit = limit; + } + + /** + * Gets the offset for results. + * @return The offset for results. + */ + public int getOffset() { + return offset; + } + + /** + * Sets the offset for results. + * @param offset The offset for results. + */ + public void setOffset(int offset) { + this.offset = offset; + } + + /** + * Gets the URI. + * @return The URI. + */ + public String getUri() { + return uri; + } + + /** + * Sets the URI. + * @param uri The URI. + */ + public void setUri(String uri) { + this.uri = uri; + } + + /** + * Gets the language; + * @return The language. + */ + public String getLanguageCode() { + return languageCode; + } + + /** + * Sets the language. + * @param language The language. + */ + public void setLanguageCode(String languageCode) { + this.languageCode = languageCode; + } + + public String getNotContext() { + return notContext; + } + + public void setNotContext(String notContext) { + this.notContext = notContext; + } + + public String getNotDocumentId() { + return notDocumentId; + } + + public void setNotDocumentId(String notDocumentId) { + this.notDocumentId = notDocumentId; + } + + public String getNotText() { + return notText; + } + + public void setNotText(String notText) { + this.notText = notText; + } + + public String getNotType() { + return notType; + } + + public void setNotType(String notType) { + this.notType = notType; + } + + public String getNotUri() { + return notUri; + } + + public void setNotUri(String notUri) { + this.notUri = notUri; + } + + public String getNotLanguageCode() { + return notLanguageCode; + } + + public void setNotLanguageCode(String notLanguage) { + this.notLanguageCode = notLanguage; + } + + /** + * Gets the {@link SortOrder}. If null the sort order is descending. + * @return The {@link SortOrder}. + */ + public SortOrder getSortOrder() { + + if(sortOrder == null) { + return SortOrder.DESC; + } else { + return sortOrder; + } + + } + + /** + * Sets the {@link SortOrder}. + * @param sortOrder The {@link SortOrder}. + */ + public void setSortOrder(SortOrder sortOrder) { + this.sortOrder = sortOrder; + } + +} diff --git a/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/SortOrder.java b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/SortOrder.java new file mode 100644 index 0000000..15a615e --- /dev/null +++ b/entitydb-eql/eql-language/src/main/java/ai/idylnlp/eql/model/SortOrder.java @@ -0,0 +1,66 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.model; + +/** + * Specifies entity ordering for use in an + * {@link EntityQuery query}. + * + * @author Mountain Fog, Inc. + * + */ +public enum SortOrder { + + ASC("asc"), + DESC("desc"); + + private String property; + + private SortOrder(String property) { + + this.property = property; + + } + + @Override + public String toString() { + return property; + } + + /** + * Gets the name of the property to sort by. + * @return The name of the property. + */ + public String getSortOrder() { + return property; + } + +} diff --git a/entitydb-eql/eql-language/src/test/java/ai/idylnlp/test/eql/EqlTest.java b/entitydb-eql/eql-language/src/test/java/ai/idylnlp/test/eql/EqlTest.java new file mode 100644 index 0000000..fe4df2e --- /dev/null +++ b/entitydb-eql/eql-language/src/test/java/ai/idylnlp/test/eql/EqlTest.java @@ -0,0 +1,529 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.test.eql; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Test; + +import ai.idylnlp.eql.Eql; +import ai.idylnlp.eql.model.EntityOrder; +import ai.idylnlp.eql.model.EntityQuery; +import ai.idylnlp.eql.model.SortOrder; + +public class EqlTest { + + private static final Logger LOGGER = LogManager.getLogger(EqlTest.class); + + @Test + public void semicolon1() throws Exception { + + String uri = "http://mtnfog.com/1.0/George_Washington"; + String query = String.format("select * from entities;", uri); + LOGGER.debug("Query: {}", query); + + EntityQuery entityQuery = Eql.generate(query); + + assertNotNull(entityQuery); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void notContext() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where context!=\"ctx\""); + + assertNotNull(entityQuery); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertEquals("ctx", entityQuery.getNotContext()); + + } + + @Test + public void notContextAndContext() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where context!=\"ctx\" and context=\"c\""); + + assertNotNull(entityQuery); + assertEquals("c", entityQuery.getContext()); + assertEquals("ctx", entityQuery.getNotContext()); + + } + + public void sort1() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities order by text"); + + assertNotNull(entityQuery); + assertEquals(EntityOrder.TEXT, entityQuery.getEntityOrder()); + + } + + public void sort2() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities order by id asc"); + + assertNotNull(entityQuery); + assertEquals(EntityOrder.ID, entityQuery.getEntityOrder()); + assertEquals(SortOrder.ASC, entityQuery.getSortOrder()); + + } + + @Test + public void sort3() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities order by id desc"); + + assertNotNull(entityQuery); + assertEquals(EntityOrder.ID, entityQuery.getEntityOrder()); + assertEquals(SortOrder.DESC, entityQuery.getSortOrder()); + + } + + @Test + public void sort4() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities"); + + assertNotNull(entityQuery); + assertEquals(EntityOrder.ID, entityQuery.getEntityOrder()); + assertEquals(SortOrder.DESC, entityQuery.getSortOrder()); + + } + + @Test(expected = IllegalStateException.class) + public void sort5() throws Exception { + + LOGGER.info("The space between the sort field and the semicolon is invalid."); + + Eql.generate("select * from entities order by text ;"); + + } + + @Test(expected = IllegalStateException.class) + public void sort6() throws Exception { + + LOGGER.info("Invalid sort order."); + + Eql.generate("select * from entities order by id asdf"); + + } + + @Test + public void query1() throws Exception { + + LOGGER.info("Testing lack of spaces around the equals sign"); + + EntityQuery entityQuery = Eql.generate("select * from entities where type=\"Person\""); + + assertNotNull(entityQuery); + assertEquals("person", entityQuery.getType()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void query2() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where type=\"Person\" and language = \"en\""); + + assertNotNull(entityQuery); + assertEquals("person", entityQuery.getType()); + assertEquals("en", entityQuery.getLanguageCode()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void type1() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where type = \"Person\""); + + assertNotNull(entityQuery); + assertEquals("person", entityQuery.getType()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void type2() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where type = \"Place\""); + + assertNotNull(entityQuery); + assertEquals("place", entityQuery.getType()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void empty() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities"); + + assertNotNull(entityQuery); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void limit1() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities limit 20"); + + assertNotNull(entityQuery); + assertEquals(20, entityQuery.getLimit()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void limit2() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities limit 1500"); + + assertNotNull(entityQuery); + assertEquals(1500, entityQuery.getLimit()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void limit3() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where metadata \"birth_date\" = \"10-20-1945\" limit 3"); + + assertNotNull(entityQuery); + assertEquals(3, entityQuery.getLimit()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + assertEquals(1, entityQuery.getEntityMetadataFilters().size()); + assertEquals("birth_date", entityQuery.getEntityMetadataFilters().get(0).getName()); + assertEquals("10-20-1945", entityQuery.getEntityMetadataFilters().get(0).getValue()); + + } + + @Test + public void limit4() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities limit 20 offset 100"); + + assertNotNull(entityQuery); + assertEquals(20, entityQuery.getLimit()); + assertEquals(100, entityQuery.getOffset()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void offset1() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where metadata \"birth_date\" = \"10-20-1945\" limit 3 offset 10"); + + assertNotNull(entityQuery); + assertEquals(3, entityQuery.getLimit()); + assertEquals(10, entityQuery.getOffset()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + assertEquals(1, entityQuery.getEntityMetadataFilters().size()); + assertEquals("birth_date", entityQuery.getEntityMetadataFilters().get(0).getName()); + assertEquals("10-20-1945", entityQuery.getEntityMetadataFilters().get(0).getValue()); + + } + + @Test + public void offset2() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities offset 10"); + + assertNotNull(entityQuery); + assertEquals(10, entityQuery.getOffset()); + assertTrue(StringUtils.isEmpty(entityQuery.getContext())); + assertTrue(StringUtils.isEmpty(entityQuery.getDocumentId())); + assertTrue(StringUtils.isEmpty(entityQuery.getText())); + + } + + @Test + public void metadata1() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where metadata \"birth_date\" = \"10-20-1945\""); + + assertNotNull(entityQuery); + assertEquals(1, entityQuery.getEntityMetadataFilters().size()); + assertEquals("birth_date", entityQuery.getEntityMetadataFilters().get(0).getName()); + assertEquals("10-20-1945", entityQuery.getEntityMetadataFilters().get(0).getValue()); + + } + + @Test + public void metadata2() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where metadata \"birth_date\" = \"10-20-1945\" and metadata \"death_date\" = \"7-13-1987\""); + + assertNotNull(entityQuery); + assertTrue(entityQuery.getEntityMetadataFilters().size() == 2); + assertEquals("birth_date", entityQuery.getEntityMetadataFilters().get(0).getName()); + assertEquals("10-20-1945", entityQuery.getEntityMetadataFilters().get(0).getValue()); + assertTrue(entityQuery.getEntityMetadataFilters().get(1).getName().equalsIgnoreCase("death_date")); + assertTrue(entityQuery.getEntityMetadataFilters().get(1).getValue().equalsIgnoreCase("7-13-1987")); + + } + + @Test + public void metadata3() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where metadata \"birth_date\" = \"10/20/1945\""); + + assertNotNull(entityQuery); + assertEquals(1, entityQuery.getEntityMetadataFilters().size()); + assertEquals("birth_date", entityQuery.getEntityMetadataFilters().get(0).getName()); + assertEquals("10/20/1945", entityQuery.getEntityMetadataFilters().get(0).getValue()); + + } + + @Test + public void compound1() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where confidence = 50 and text = \"test\""); + + assertNotNull(entityQuery); + assertNotNull(entityQuery.getConfidenceRange()); + assertNotNull(entityQuery.getConfidenceRange().getMaximum()); + assertEquals(0.5, entityQuery.getConfidenceRange().getMaximum(), 0); + assertEquals("test", entityQuery.getText()); + + } + + @Test + public void compound2() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where context = \"ctx\" and text = \"test\""); + + assertNotNull(entityQuery); + assertEquals("ctx", entityQuery.getContext()); + assertEquals("test", entityQuery.getText()); + + } + + @Test + public void compound3() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where context = \"ctx\" and text = \"test\" and documentid = \"docid\""); + + assertNotNull(entityQuery); + assertEquals("ctx", entityQuery.getContext()); + assertEquals("test", entityQuery.getText()); + assertTrue(entityQuery.getDocumentId().contains("docid")); + + } + + @Test + public void compound4() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where context = \"ctx\" and text = \"test\" and documentid = \"docid\" and confidence = 30"); + + assertNotNull(entityQuery); + assertEquals("ctx", entityQuery.getContext()); + assertEquals("test", entityQuery.getText()); + assertTrue(entityQuery.getDocumentId().contains("docid")); + assertTrue(entityQuery.getConfidenceRange().getMaximum() == 0.3); + + } + + @Test + public void confidence1() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where confidence = 50"); + + assertNotNull(entityQuery); + assertNotNull(entityQuery.getConfidenceRange()); + assertNotNull(entityQuery.getConfidenceRange().getMaximum()); + assertTrue(entityQuery.getConfidenceRange().getMaximum() == 0.5); + + } + + @Test + public void confidence2() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where confidence between 25 and 50"); + + assertNotNull(entityQuery); + assertNotNull(entityQuery.getConfidenceRange()); + assertNotNull(entityQuery.getConfidenceRange().getMaximum()); + assertTrue(entityQuery.getConfidenceRange().getMaximum() == 0.5); + assertNotNull(entityQuery.getConfidenceRange().getMinimum()); + assertTrue(entityQuery.getConfidenceRange().getMinimum() == 0.25); + + } + + @Test + public void confidence3() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where confidence > 25"); + + LOGGER.info(entityQuery.getConfidenceRange().toString()); + + assertNotNull(entityQuery); + assertNotNull(entityQuery.getConfidenceRange()); + assertNotNull(entityQuery.getConfidenceRange().getMaximum()); + assertTrue(entityQuery.getConfidenceRange().getMaximum() == 1.0); + assertNotNull(entityQuery.getConfidenceRange().getMinimum()); + assertTrue(entityQuery.getConfidenceRange().getMinimum() == 0.26); + + } + + @Test + public void confidence4() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where confidence < 60"); + + LOGGER.info(entityQuery.getConfidenceRange().toString()); + + assertNotNull(entityQuery); + assertNotNull(entityQuery.getConfidenceRange()); + assertNotNull(entityQuery.getConfidenceRange().getMaximum()); + assertTrue(entityQuery.getConfidenceRange().getMaximum() == 0.59); + assertNotNull(entityQuery.getConfidenceRange().getMinimum()); + assertTrue(entityQuery.getConfidenceRange().getMinimum() == 0.0); + + } + + @Test + public void confidence5() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where confidence >= 60"); + + assertNotNull(entityQuery); + assertNotNull(entityQuery.getConfidenceRange()); + assertNotNull(entityQuery.getConfidenceRange().getMaximum()); + assertTrue(entityQuery.getConfidenceRange().getMaximum() == 1.0); + assertNotNull(entityQuery.getConfidenceRange().getMinimum()); + assertTrue(entityQuery.getConfidenceRange().getMinimum() == 0.60); + + } + + @Test + public void confidence6() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where confidence >= 60 and text = \"George Washington\""); + + assertNotNull(entityQuery); + assertNotNull(entityQuery.getConfidenceRange()); + assertNotNull(entityQuery.getConfidenceRange().getMaximum()); + assertTrue(entityQuery.getConfidenceRange().getMaximum() == 1.0); + assertNotNull(entityQuery.getConfidenceRange().getMinimum()); + assertTrue(entityQuery.getConfidenceRange().getMinimum() == 0.60); + assertEquals("George Washington", entityQuery.getText()); + + } + + @Test + public void context() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where context = \"test\""); + + assertEquals("test", entityQuery.getContext()); + + } + + @Test + public void documentid() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where documentid = \"test\""); + + assertEquals("test", entityQuery.getDocumentId()); + + } + + @Test + public void text() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where text = \"test\""); + + assertEquals("test", entityQuery.getText()); + + } + + @Test + public void textSpace1() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where text = \"GeorgeWashington\""); + + assertEquals("GeorgeWashington", entityQuery.getText()); + + } + + @Test + public void textSpace2() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where text = \"George Washington\""); + + assertEquals("George Washington", entityQuery.getText()); + + } + + @Test + public void textSpace3() throws Exception { + + EntityQuery entityQuery = Eql.generate("select * from entities where text=\"George Washington\""); + + assertEquals("George Washington", entityQuery.getText()); + + } + +} diff --git a/entitydb-eql/eql-language/src/test/java/ai/idylnlp/test/eql/model/test/EntityQueryTest.java b/entitydb-eql/eql-language/src/test/java/ai/idylnlp/test/eql/model/test/EntityQueryTest.java new file mode 100644 index 0000000..0ce8acc --- /dev/null +++ b/entitydb-eql/eql-language/src/test/java/ai/idylnlp/test/eql/model/test/EntityQueryTest.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.test.eql.model.test; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import ai.idylnlp.eql.model.EntityQuery; + +public class EntityQueryTest { + + /*@Test + public void build() { + + EntityQuery entityQuery = EntityQuery.builder() + .entityText("George Washington") + .confidence(50) + .context("context") + .build(); + + assertEquals("George Washington", entityQuery.getEntityText()); + assertEquals(50, entityQuery.getConfidenceRange().getMaximum(), 0); + assertEquals("context", entityQuery.getContext()); + + }*/ + + @Test + public void type() { + + EntityQuery entityQuery = new EntityQuery(); + + assertNull(entityQuery.getType()); + assertEquals(0, entityQuery.getOffset()); + assertEquals(25, entityQuery.getLimit()); + + } + +} diff --git a/entitydb-eql/eql-language/src/test/resources/log4j2.xml b/entitydb-eql/eql-language/src/test/resources/log4j2.xml new file mode 100644 index 0000000..ef38e1f --- /dev/null +++ b/entitydb-eql/eql-language/src/test/resources/log4j2.xml @@ -0,0 +1,32 @@ + + + + + logs + + + + + + + + + + + + + diff --git a/entitydb-eql/eql-pig-udf/pig/eql.pig b/entitydb-eql/eql-pig-udf/pig/eql.pig new file mode 100644 index 0000000..0ad1837 --- /dev/null +++ b/entitydb-eql/eql-pig-udf/pig/eql.pig @@ -0,0 +1,11 @@ +REGISTER eql-pig-udf.jar; +DEFINE EqlFilter ai.idylnlp.eql.pig.EqlFilterFunc('select * from entities'); +DEFINE EqlMatch ai.idylnlp.eql.pig.EqlMatchFunc('select * from entities'); + +rawDS = load 'input' using PigStorage() as (entity:chararray); + +filteredDS = foreach rawDS generate EqlFilter(entity); +STORE filteredDS INTO 'output-filtered' using PigStorage(';'); + +matchDS = foreach rawDS generate EqlMatch(entity); +STORE matchDS INTO 'output-match' using PigStorage(';'); \ No newline at end of file diff --git a/entitydb-eql/eql-pig-udf/pig/input b/entitydb-eql/eql-pig-udf/pig/input new file mode 100644 index 0000000..3e5f7be --- /dev/null +++ b/entitydb-eql/eql-pig-udf/pig/input @@ -0,0 +1 @@ +{"text":"George Washington","confidence":0.0,"extractionDate":0,"metadata":{}} \ No newline at end of file diff --git a/entitydb-eql/eql-pig-udf/pig/run.sh b/entitydb-eql/eql-pig-udf/pig/run.sh new file mode 100644 index 0000000..e9a5b4a --- /dev/null +++ b/entitydb-eql/eql-pig-udf/pig/run.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# TODO: Set for your system. +export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ +export PIG_HOME=/opt/pig-0.16.0/ + +$PIG_HOME/bin/pig -x local eql.pig diff --git a/entitydb-eql/eql-pig-udf/pom.xml b/entitydb-eql/eql-pig-udf/pom.xml new file mode 100644 index 0000000..ba64a5a --- /dev/null +++ b/entitydb-eql/eql-pig-udf/pom.xml @@ -0,0 +1,111 @@ + + + + 4.0.0 + + ai.idylnlp + eql + 1.3.0-SNAPSHOT + + eql-pig-udf + eql-pig-udf + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + package + + shade + + + eql-pig-udf + + + + + + + + + + maven-resources-plugin + 3.0.2 + + + copy-resources + validate + + copy-resources + + + ${basedir}/target + + + ${basedir}/pig/ + false + + + + + + + + + + + ai.idylnlp + eql-language + ${project.version} + + + ai.idylnlp + eql-filters + ${project.version} + + + com.google.code.gson + gson + + + org.apache.commons + commons-lang3 + 3.3 + + + org.apache.pig + pig + 0.16.0 + + + org.apache.hadoop + hadoop-common + 2.7.3 + + + junit + junit + test + + + diff --git a/entitydb-eql/eql-pig-udf/src/main/java/ai/idylnlp/eql/pig/EqlFilterFunc.java b/entitydb-eql/eql-pig-udf/src/main/java/ai/idylnlp/eql/pig/EqlFilterFunc.java new file mode 100644 index 0000000..c6595ad --- /dev/null +++ b/entitydb-eql/eql-pig-udf/src/main/java/ai/idylnlp/eql/pig/EqlFilterFunc.java @@ -0,0 +1,89 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.pig; + +import java.io.IOException; + +import org.apache.pig.EvalFunc; +import org.apache.pig.data.Tuple; + +import com.google.gson.Gson; + +import ai.idylnlp.eql.filters.EqlFilters; +import ai.idylnlp.model.entity.Entity; + +/** + * Pig UDF that executes an EQL statement on an entity. + * + * @author Mountain Fog, Inc. + * + */ +public class EqlFilterFunc extends EvalFunc { + + private String eql; + private Gson gson; + + public EqlFilterFunc(String eql) { + + this.eql = eql; + gson = new Gson(); + + } + + @Override + public String exec(Tuple input) throws IOException { + + if (input == null || input.size() == 0) { + + return null; + + } else { + + final String entityJson = input.get(0).toString(); + + Entity entity = gson.fromJson(entityJson, Entity.class); + + if(EqlFilters.isMatch(entity, eql)) { + + return entityJson; + + } else { + + return ""; + + } + + } + + } + + +} diff --git a/entitydb-eql/eql-pig-udf/src/main/java/ai/idylnlp/eql/pig/EqlMatchFunc.java b/entitydb-eql/eql-pig-udf/src/main/java/ai/idylnlp/eql/pig/EqlMatchFunc.java new file mode 100644 index 0000000..29f7623 --- /dev/null +++ b/entitydb-eql/eql-pig-udf/src/main/java/ai/idylnlp/eql/pig/EqlMatchFunc.java @@ -0,0 +1,78 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/* + * (C) Copyright 2017 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.idylnlp.eql.pig; + +import java.io.IOException; + +import org.apache.pig.FilterFunc; +import org.apache.pig.data.Tuple; + +import com.google.gson.Gson; + +import ai.idylnlp.eql.filters.EqlFilters; +import ai.idylnlp.model.entity.Entity; + +/** + * Pig UDF that executes an EQL statement on an entity. + * + * @author Mountain Fog, Inc. + * + */ +public class EqlMatchFunc extends FilterFunc { + + private String eql; + private Gson gson; + + public EqlMatchFunc(String eql) { + + this.eql = eql; + gson = new Gson(); + + } + + @Override + public Boolean exec(Tuple input) throws IOException { + + if (input == null || input.size() == 0) { + + return null; + + } else { + + Entity entity = gson.fromJson(input.get(0).toString(), Entity.class); + + return EqlFilters.isMatch(entity, eql); + + } + + } + +} diff --git a/entitydb-eql/eql-pig-udf/src/test/java/ai/idylnlp/test/eql/pig/EqlMatchFuncTest.java b/entitydb-eql/eql-pig-udf/src/test/java/ai/idylnlp/test/eql/pig/EqlMatchFuncTest.java new file mode 100644 index 0000000..fc79f6c --- /dev/null +++ b/entitydb-eql/eql-pig-udf/src/test/java/ai/idylnlp/test/eql/pig/EqlMatchFuncTest.java @@ -0,0 +1,63 @@ +/******************************************************************************* + * Copyright 2019 Mountain Fog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package ai.idylnlp.test.eql.pig; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.pig.FilterFunc; +import org.apache.pig.data.DefaultTuple; +import org.junit.Before; +import org.junit.Test; + +import com.google.gson.Gson; + +import ai.idylnlp.eql.pig.EqlMatchFunc; +import ai.idylnlp.model.entity.Entity; + +public class EqlMatchFuncTest { + + private static final String SELECT_ALL = "select * from entities"; + private Gson gson; + + @Before + public void before() { + + gson = new Gson(); + + } + + @Test + public void classExtendsFilterFunc() { + assertTrue(new EqlMatchFunc(SELECT_ALL) instanceof FilterFunc); + } + + @Test + public void filter1() throws IOException { + + Entity entity = new Entity("George Washington"); + String jsonEntity = gson.toJson(entity); + + DefaultTuple input = new DefaultTuple(); + input.append(jsonEntity); + + assertEquals(true, new EqlMatchFunc(SELECT_ALL).exec(input)); + + } + +} diff --git a/entitydb-eql/eql-pig-udf/src/test/resources/log4j2.xml b/entitydb-eql/eql-pig-udf/src/test/resources/log4j2.xml new file mode 100644 index 0000000..8a1918f --- /dev/null +++ b/entitydb-eql/eql-pig-udf/src/test/resources/log4j2.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + diff --git a/entitydb-eql/pom.xml b/entitydb-eql/pom.xml new file mode 100644 index 0000000..8ec4488 --- /dev/null +++ b/entitydb-eql/pom.xml @@ -0,0 +1,151 @@ + + + + 4.0.0 + + ai.idylnlp + idylnlp + 1.3.0-SNAPSHOT + + pom + eql + eql + Entity Query Language (EQL) is a SQL-like language for querying entities. + https://github.com/idylnlp/idylnlp-eql + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + scm:git:https://github.com/idylnlp/idylnlp-eql.git + scm:git:https://github.com/idylnlp/idylnlp-eql.git + https://github.com/idylnlp/idylnlp-eql.git + + + + Apache License, Version 2 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + Mountain Fog + help@idylnlp.ai + Mountain Fog + http://www.mtnfog.com + + + + UTF-8 + UTF-8 + 4.0 + 4.1 + 3.6 + + + eql-language + eql-filters + eql-pig-udf + + + + + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + + + org.codehaus.mojo + versions-maven-plugin + 2.1 + + false + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.3 + true + + ossrh + https://oss.sonatype.org/ + false + + + + + + + release + + + + org.apache.maven.plugins + maven-source-plugin + 2.4 + + + attach-sources + + jar + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + sign-artifacts + verify + + sign + + + + + + + + +