Skip to content

Commit

Permalink
SOLR-12490: Introducing json.queries to define many named queries in …
Browse files Browse the repository at this point in the history
…Query DSL.
  • Loading branch information
mkhludnev committed Jan 3, 2020
1 parent 479e736 commit 8fba8eb
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 35 deletions.
6 changes: 4 additions & 2 deletions solr/CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,14 @@ Upgrade Notes
If you prefer to keep the old (but insecure) serialization strategy, you can start your nodes using the
property: `-Dsolr.useUnsafeOverseerResponse=true`. Keep in mind that this will be removed in future version of Solr.

* SOLR-13808: add cache=false into uderneath BoolQParser's filter clause or {"bool":{"filter":..}} to avoid caching in
* SOLR-13808: add cache=false into underneath BoolQParser's filter clause or {"bool":{"filter":..}} to avoid caching in
filterCache. (Mikhail Khludnev)

New Features
---------------------
(No changes)
* SOLR-12490: Introducing json.queries in JSON Request API. Every property of this object holds one or many named
Query DSL queries. It's optional and doesn't impact response without explicit referencing these queries by names
(Anatolii Siuniaev via Mikhail Khludnev)

Improvements
---------------------
Expand Down
74 changes: 45 additions & 29 deletions solr/core/src/java/org/apache/solr/request/json/RequestUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ public static void processParams(SolrRequestHandler handler, SolrQueryRequest re

// implement compat for existing components...
JsonQueryConverter jsonQueryConverter = new JsonQueryConverter();

if (json != null && !isShard) {
for (Map.Entry<String,Object> entry : json.entrySet()) {
String key = entry.getKey();
Expand All @@ -214,48 +215,62 @@ public static void processParams(SolrRequestHandler handler, SolrQueryRequest re
out = "rows";
} else if (SORT.equals(key)) {
out = SORT;
} else if ("queries".equals(key)) {
Object queriesJsonObj = entry.getValue();
if (queriesJsonObj instanceof Map && queriesJsonObj != null) {
@SuppressWarnings("unchecked")
final Map<String,Object> queriesAsMap = (Map<String,Object>) queriesJsonObj;
for (Map.Entry<String,Object> queryJsonProperty : queriesAsMap.entrySet()) {
out = queryJsonProperty.getKey();
arr = true;
isQuery = true;
convertJsonPropertyToLocalParams(newMap, jsonQueryConverter, queryJsonProperty, out, isQuery, arr);
}
continue;
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Expected Map for 'queries', received " + queriesJsonObj);
}
} else if ("params".equals(key) || "facet".equals(key) ) {
// handled elsewhere
continue;
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown top-level key in JSON request : " + key);
}

Object val = entry.getValue();

if (arr) {
String[] existing = newMap.get(out);
List lst = val instanceof List ? (List)val : null;
int existingSize = existing==null ? 0 : existing.length;
int jsonSize = lst==null ? 1 : lst.size();
String[] newval = new String[ existingSize + jsonSize ];
for (int i=0; i<existingSize; i++) {
newval[i] = existing[i];
}
if (lst != null) {
for (int i = 0; i < jsonSize; i++) {
Object v = lst.get(i);
newval[existingSize + i] = isQuery ? jsonQueryConverter.toLocalParams(v, newMap) : v.toString();
}
} else {
newval[newval.length-1] = isQuery ? jsonQueryConverter.toLocalParams(val, newMap) : val.toString();
}
newMap.put(out, newval);
} else {
newMap.put(out, new String[]{isQuery ? jsonQueryConverter.toLocalParams(val, newMap) : val.toString()});
}

convertJsonPropertyToLocalParams(newMap, jsonQueryConverter, entry, out, isQuery, arr);
}


}

if (json != null) {
req.setJSON(json);
}

}

private static void convertJsonPropertyToLocalParams(Map<String, String[]> outMap, JsonQueryConverter jsonQueryConverter, Map.Entry<String, Object> jsonProperty, String outKey, boolean isQuery, boolean arr) {
Object val = jsonProperty.getValue();

if (arr) {
String[] existing = outMap.get(outKey);
List<?> lst = val instanceof List ? (List<?>)val : null;
int existingSize = existing==null ? 0 : existing.length;
int jsonSize = lst==null ? 1 : lst.size();
String[] newval = new String[ existingSize + jsonSize ];
for (int i=0; i<existingSize; i++) {
newval[i] = existing[i];
}
if (lst != null) {
for (int i = 0; i < jsonSize; i++) {
Object v = lst.get(i);
newval[existingSize + i] = isQuery ? jsonQueryConverter.toLocalParams(v, outMap) : v.toString();
}
} else {
newval[newval.length-1] = isQuery ? jsonQueryConverter.toLocalParams(val, outMap) : val.toString();
}
outMap.put(outKey, newval);
} else {
outMap.put(outKey, new String[]{isQuery ? jsonQueryConverter.toLocalParams(val, outMap) : val.toString()});
}
}


// queryParamName is something like json.facet or json.query, or just json...
Expand Down Expand Up @@ -295,6 +310,7 @@ private static void getParamsFromJSON(Map<String, String[]> params, String json)

Object o = ObjectBuilder.getVal(parser);
if (!(o instanceof Map)) return;
@SuppressWarnings("unchecked")
Map<String,Object> map = (Map<String,Object>)o;
// To make consistent with json.param handling, we should make query params come after json params (i.e. query params should
// appear to overwrite json params.
Expand All @@ -310,7 +326,7 @@ private static void getParamsFromJSON(Map<String, String[]> params, String json)
if (val == null) {
params.remove(key);
} else if (val instanceof List) {
List lst = (List) val;
List<?> lst = (List<?>) val;
String[] vals = new String[lst.size()];
for (int i = 0; i < vals.length; i++) {
vals[i] = lst.get(i).toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
private static int origTableSize;
private static FacetField.FacetMethod origDefaultFacetMethod;

@SuppressWarnings("deprecation")
@BeforeClass
public static void beforeTests() throws Exception {
systemSetPropertySolrDisableShardsWhitelist("true");
Expand Down Expand Up @@ -83,6 +84,7 @@ public static void initServers() throws Exception {
}
}

@SuppressWarnings("deprecation")
@AfterClass
public static void afterTests() throws Exception {
systemClearPropertySolrDisableShardsWhitelist();
Expand Down Expand Up @@ -1063,7 +1065,6 @@ public void doStats(Client client, ModifiableSolrParams p) throws Exception {
}

public static void doStatsTemplated(Client client, ModifiableSolrParams p) throws Exception {
int numShards = client.local() ? 1 : client.getClientProvider().all().size();
p.set("Z_num_i", "Z_" + p.get("num_i") );
p.set("Z_num_l", "Z_" + p.get("num_l") );
p.set("sparse_num_d", "sparse_" + p.get("num_d") );
Expand Down Expand Up @@ -2290,6 +2291,19 @@ public static void doStatsTemplated(Client client, ModifiableSolrParams p) throw
"}"
);

//test filter using queries from json.queries
client.testJQ(params(p, "q", "*:*"
, "json.queries", "{catS:{'#cat_sA': '${cat_s}:A'}, ff:[{'#id_1':'-id:1'},{'#id_2':'-id:2'}]}"
, "json.facet", "{" +
",t_filt1:{${terms} type:terms, field:${cat_s}, domain:{filter:{param:catS} } }" + // test filter via "param" type from .queries
",t_filt2:{${terms} type:terms, field:${cat_s}, domain:{filter:{param:ff}} }" + // test multi-valued query parameter from .queries
"}"
)
, "facets=={ count:6, " +
",t_filt1:{ buckets:[ {val:A, count:2}] } " +
",t_filt2:{ buckets:[ {val:B, count:2}, {val:A, count:1}] } " +
"}"
);

// test acc reuse (i.e. reset() method). This is normally used for stats that are not calculated in the first phase,
// currently non-sorting stats.
Expand Down Expand Up @@ -2907,7 +2921,7 @@ public void doBigger(Client client, ModifiableSolrParams p) throws Exception {
int commitPercent = 10;
int ndocs=1000;

Map<Integer, Map<Integer, List<Integer>>> model = new HashMap(); // cat->where->list<ids>
Map<Integer, Map<Integer, List<Integer>>> model = new HashMap<>(); // cat->where->list<ids>
for (int i=0; i<ndocs; i++) {
Integer cat = r.nextInt(numCat);
Integer where = r.nextInt(numWhere);
Expand Down Expand Up @@ -3328,7 +3342,6 @@ public void testErrors() throws Exception {
}

public void doTestErrors(Client client) throws Exception {
ModifiableSolrParams p = params("rows", "0");
client.deleteByQuery("*:*", null);

try {
Expand Down Expand Up @@ -3646,11 +3659,18 @@ public void testOtherErrorCases() throws Exception {
req("q", "*:*", "rows", "0", "json.facet", "{cat_s:{type:terms,field:cat_s,sort:[\"count desc\"]}}"),
SolrException.ErrorCode.BAD_REQUEST);


assertQEx("Should fail as facet is not of type map",
"Expected Map for 'facet', received ArrayList=[{}]",
req("q", "*:*", "rows", "0", "json.facet", "[{}]"), SolrException.ErrorCode.BAD_REQUEST);

assertQEx("Should fail as queries is not of type map",
"Expected Map for 'queries', received [{}]",
req("q", "*:*", "rows", "0", "json.queries", "[{}]"), SolrException.ErrorCode.BAD_REQUEST);

assertQEx("Should fail as queries are null in JSON",
"Expected Map for 'queries', received null",
req("json", "{query:\"*:*\", queries:null}"), SolrException.ErrorCode.BAD_REQUEST);

// range facets
assertQEx("Should fail as 'other' is of type Map",
"Expected list of string or comma separated string values for 'other', " +
Expand Down

0 comments on commit 8fba8eb

Please sign in to comment.