From 1d8443245cb4dabec408a340606c405e560a2577 Mon Sep 17 00:00:00 2001 From: iilyak Date: Mon, 29 Jan 2024 13:13:39 -0800 Subject: [PATCH] Modernize scala syntax (#87) * Do not brackets for function without args * Do not use postfix notation * Do not use procedural syntax * Use case class to get hashCode and equals The `ValueSource` requires overriding `equals` and `hashCode()`. These two are automatically derived for case classes. * Collection is removed from recent scala be more explicit * Refactor request parsing to validate the types of nested structures * Use unchecked for inner types we cannot match on * Use `case e: Throwable => ...` idiom to catch all possible exceptions. The previous implementation `case e => ...` produces following warning on modern scala ``` warning: This catches all Throwables. If this is really intended, use `case e : Throwable` to clear this warning. case e => ^ ``` * Add return type where asked by compiler --- .../clouseau/ClouseauQueryParser.scala | 4 +- .../clouseau/ClouseauSupervisor.scala | 2 +- .../clouseau/ClouseauTypeFactory.scala | 94 ++--- .../com/cloudant/clouseau/Converters.scala | 62 +++ .../clouseau/DistanceValueSource.scala | 10 +- .../ExternalSnapshotDeletionPolicy.scala | 2 +- .../clouseau/IndexCleanupService.scala | 6 +- .../clouseau/IndexManagerService.scala | 12 +- .../com/cloudant/clouseau/IndexService.scala | 374 ++++++++++++------ .../scala/com/cloudant/clouseau/Main.scala | 2 +- .../cloudant/clouseau/QueryExplainer.scala | 16 +- .../scala/com/cloudant/clouseau/Utils.scala | 51 +++ .../cloudant/clouseau/IndexServiceSpec.scala | 188 ++++++++- .../com/cloudant/clouseau/UtilsSpec.scala | 85 ++++ 14 files changed, 718 insertions(+), 190 deletions(-) create mode 100644 src/main/scala/com/cloudant/clouseau/Converters.scala create mode 100644 src/test/scala/com/cloudant/clouseau/UtilsSpec.scala diff --git a/src/main/scala/com/cloudant/clouseau/ClouseauQueryParser.scala b/src/main/scala/com/cloudant/clouseau/ClouseauQueryParser.scala index 00773187..960e581a 100644 --- a/src/main/scala/com/cloudant/clouseau/ClouseauQueryParser.scala +++ b/src/main/scala/com/cloudant/clouseau/ClouseauQueryParser.scala @@ -107,7 +107,7 @@ class ClouseauQueryParser(version: Version, fpRegex.matcher(str).matches() } - private def setLowercaseExpandedTerms(field: String) { + private def setLowercaseExpandedTerms(field: String): Unit = { getAnalyzer match { case a: PerFieldAnalyzer => setLowercaseExpandedTerms(a.getWrappedAnalyzer(field)) @@ -116,7 +116,7 @@ class ClouseauQueryParser(version: Version, } } - private def setLowercaseExpandedTerms(analyzer: Analyzer) { + private def setLowercaseExpandedTerms(analyzer: Analyzer): Unit = { setLowercaseExpandedTerms(!analyzer.isInstanceOf[KeywordAnalyzer]) } diff --git a/src/main/scala/com/cloudant/clouseau/ClouseauSupervisor.scala b/src/main/scala/com/cloudant/clouseau/ClouseauSupervisor.scala index a1723747..a371c5dc 100644 --- a/src/main/scala/com/cloudant/clouseau/ClouseauSupervisor.scala +++ b/src/main/scala/com/cloudant/clouseau/ClouseauSupervisor.scala @@ -26,7 +26,7 @@ class ClouseauSupervisor(ctx: ServiceContext[ConfigurationArgs]) extends Service var cleanup = spawnAndMonitorService[IndexCleanupService, ConfigurationArgs]('cleanup, ctx.args) var analyzer = spawnAndMonitorService[AnalyzerService, ConfigurationArgs]('analyzer, ctx.args) - override def trapMonitorExit(monitored: Any, ref: Reference, reason: Any) { + override def trapMonitorExit(monitored: Any, ref: Reference, reason: Any) = { if (monitored == manager) { logger.warn("manager crashed") manager = spawnAndMonitorService[IndexManagerService, ConfigurationArgs]('main, ctx.args) diff --git a/src/main/scala/com/cloudant/clouseau/ClouseauTypeFactory.scala b/src/main/scala/com/cloudant/clouseau/ClouseauTypeFactory.scala index d546a0c5..3017e695 100644 --- a/src/main/scala/com/cloudant/clouseau/ClouseauTypeFactory.scala +++ b/src/main/scala/com/cloudant/clouseau/ClouseauTypeFactory.scala @@ -120,49 +120,51 @@ object ClouseauTypeFactory extends TypeFactory { result } - private def addFields(doc: Document, field0: Any): Unit = field0 match { - case (name: String, value: String, options: List[(String, Any)]) => - val map = options.toMap - constructField(name, value, toStore(map), toIndex(map), toTermVector(map)) match { - case Some(field) => - map.get("boost") match { - case Some(boost: Number) => - field.setBoost(toFloat(boost)) - case None => - 'ok - } - doc.add(field) - if (isFacet(map) && value.nonEmpty) { - val fp = FacetIndexingParams.DEFAULT - val delim = fp.getFacetDelimChar - if (!name.contains(delim) && !value.contains(delim)) { - val facets = new SortedSetDocValuesFacetFields(fp) - facets.addFields(doc, List(new CategoryPath(name, value)).asJava) + private def addFields(doc: Document, field0: Any): Unit = { + field0 match { + case (name: String, value: String, options: List[(String, Any) @unchecked]) => + val map = options.collect { case t @ (_: String, _: Any) => t }.asInstanceOf[List[(String, Any)]].toMap + constructField(name, value, toStore(map), toIndex(map), toTermVector(map)) match { + case Some(field) => + map.get("boost") match { + case Some(boost: Number) => + field.setBoost(toFloat(boost)) + case None => + 'ok } - } - case None => - 'ok - } - case (name: String, value: Boolean, options: List[(String, Any)]) => - val map = options.toMap - constructField(name, value.toString, toStore(map), Index.NOT_ANALYZED, toTermVector(map)) match { - case Some(field) => - doc.add(field) - case None => - 'ok - } - case (name: String, value: Any, options: List[(String, Any)]) => - val map = options.toMap - toDouble(value) match { - case Some(doubleValue) => - doc.add(new DoubleField(name, doubleValue, toStore(map))) - if (isFacet(map)) { - doc.add(new DoubleDocValuesField(name, doubleValue)) - } - case None => - logger.warn("Unrecognized value: %s".format(value)) - 'ok - } + doc.add(field) + if (isFacet(map) && value.nonEmpty) { + val fp = FacetIndexingParams.DEFAULT + val delim = fp.getFacetDelimChar + if (!name.contains(delim) && !value.contains(delim)) { + val facets = new SortedSetDocValuesFacetFields(fp) + facets.addFields(doc, List(new CategoryPath(name, value)).asJava) + } + } + case None => + 'ok + } + case (name: String, value: Boolean, options: List[(String, Any) @unchecked]) => + val map = options.collect { case t @ (_: String, _: Any) => t }.asInstanceOf[List[(String, Any)]].toMap + constructField(name, value.toString, toStore(map), Index.NOT_ANALYZED, toTermVector(map)) match { + case Some(field) => + doc.add(field) + case None => + 'ok + } + case (name: String, value: Any, options: List[(String, Any) @unchecked]) => + val map = options.collect { case t @ (_: String, _: Any) => t }.asInstanceOf[List[(String, Any)]].toMap + toDouble(value) match { + case Some(doubleValue) => + doc.add(new DoubleField(name, doubleValue, toStore(map))) + if (isFacet(map)) { + doc.add(new DoubleDocValuesField(name, doubleValue)) + } + case None => + logger.warn("Unrecognized value: %s".format(value)) + 'ok + } + } } private def constructField(name: String, value: String, store: Store, index: Index, tv: TermVector): Option[Field] = { @@ -192,7 +194,7 @@ object ClouseauTypeFactory extends TypeFactory { case v: java.lang.Float => Some(v.doubleValue) case v: java.lang.Integer => Some(v.doubleValue) case v: java.lang.Long => Some(v.doubleValue) - case v: scala.math.BigInt => Some(v.doubleValue()) + case v: scala.math.BigInt => Some(v.doubleValue) case _ => None } @@ -212,7 +214,7 @@ object ClouseauTypeFactory extends TypeFactory { case false => Store.NO case str: String => try { - Store.valueOf(str toUpperCase) + Store.valueOf(str.toUpperCase) } catch { case _: IllegalArgumentException => Store.NO @@ -228,7 +230,7 @@ object ClouseauTypeFactory extends TypeFactory { case false => Index.NO case str: String => try { - Index.valueOf(str toUpperCase) + Index.valueOf(str.toUpperCase) } catch { case _: IllegalArgumentException => Index.ANALYZED @@ -240,7 +242,7 @@ object ClouseauTypeFactory extends TypeFactory { def toTermVector(options: Map[String, Any]): TermVector = { val termVector = options.getOrElse("termvector", "no").asInstanceOf[String] - TermVector.valueOf(termVector toUpperCase) + TermVector.valueOf(termVector.toUpperCase) } def isFacet(options: Map[String, Any]) = { diff --git a/src/main/scala/com/cloudant/clouseau/Converters.scala b/src/main/scala/com/cloudant/clouseau/Converters.scala new file mode 100644 index 00000000..411b568f --- /dev/null +++ b/src/main/scala/com/cloudant/clouseau/Converters.scala @@ -0,0 +1,62 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +package com.cloudant.clouseau + +object conversions { + /** + * Scala 2.9.1 doesn't have extractors on PartialFunction. These were introduced in + * + * [scala/scala#7111](https://github.com/scala/scala/pull/7111) and released in 2.13.0 + * + * See also [SIP-38](https://docs.scala-lang.org/sips/converters-among-optional-functions-partialfunctions-and-extractor-objects.html) + * + * with extractors you can do something like + * + * ```scala + * val asString: PartialFunction[Any, String] = + * { case string: String => string } + * + * def test(value: Any): Option[String] = { + * value match { + * case asString(s) => Some(s) + * case _ => None + * } + * } + * ``` + * + * This implicit conversion adds a middle ground solution. It can be used as follows: + * + * ```scala + * import conversions._ /// Here we bring implicit convertors in scope + * + * val asString: PartialFunction[Any, String] = + * { case string: String => string } + * + * def test(value: Any): Option[String] = { + * val asStringExtractor = asString.Extractor /// This is the extra line we need to add + * value match { + * case asStringExtractor(s) => Some(s) /// Use Extractor to get the matching content + * case _ => None + * } + * } + * ``` + * + * @param pf + */ + class ExtendedPF[A, B](val pf: PartialFunction[A, B]) { + object Extractor { + def unapply(a: A): Option[B] = pf.lift(a) + } + } + implicit def extendPartialFunction[A, B](pf: PartialFunction[A, B]): ExtendedPF[A, B] = new ExtendedPF(pf) +} diff --git a/src/main/scala/com/cloudant/clouseau/DistanceValueSource.scala b/src/main/scala/com/cloudant/clouseau/DistanceValueSource.scala index d7449271..3d950b46 100644 --- a/src/main/scala/com/cloudant/clouseau/DistanceValueSource.scala +++ b/src/main/scala/com/cloudant/clouseau/DistanceValueSource.scala @@ -25,11 +25,11 @@ import java.util.Map This is lucene spatial's DistanceValueSource but with configurable x and y field names to better suit our existing API. */ -class DistanceValueSource(ctx: SpatialContext, - lon: String, - lat: String, - multiplier: Double, - from: Point) +case class DistanceValueSource(ctx: SpatialContext, + lon: String, + lat: String, + multiplier: Double, + from: Point) extends ValueSource { def description() = "DistanceValueSource(%s)".format(from) diff --git a/src/main/scala/com/cloudant/clouseau/ExternalSnapshotDeletionPolicy.scala b/src/main/scala/com/cloudant/clouseau/ExternalSnapshotDeletionPolicy.scala index 3bc12c16..bb505ba9 100644 --- a/src/main/scala/com/cloudant/clouseau/ExternalSnapshotDeletionPolicy.scala +++ b/src/main/scala/com/cloudant/clouseau/ExternalSnapshotDeletionPolicy.scala @@ -49,7 +49,7 @@ class ExternalSnapshotDeletionPolicy(dir: FSDirectory) extends IndexDeletionPoli object ExternalSnapshotDeletionPolicy { - def snapshot(originDir: File, snapshotDir: File, files: Collection[String]) { + def snapshot(originDir: File, snapshotDir: File, files: Collection[String]) = { if (!originDir.isAbsolute) { throw new IOException(originDir + " is not an absolute path") } diff --git a/src/main/scala/com/cloudant/clouseau/IndexCleanupService.scala b/src/main/scala/com/cloudant/clouseau/IndexCleanupService.scala index d3368c04..cbc2af4e 100644 --- a/src/main/scala/com/cloudant/clouseau/IndexCleanupService.scala +++ b/src/main/scala/com/cloudant/clouseau/IndexCleanupService.scala @@ -51,7 +51,7 @@ class IndexCleanupService(ctx: ServiceContext[ConfigurationArgs]) extends Servic cleanup(rootDir, pattern, activeSigs) } - private def cleanup(fileOrDir: File, includePattern: Pattern, activeSigs: List[String]) { + private def cleanup(fileOrDir: File, includePattern: Pattern, activeSigs: List[String]): Unit = { if (!fileOrDir.isDirectory) { return } @@ -71,7 +71,7 @@ class IndexCleanupService(ctx: ServiceContext[ConfigurationArgs]) extends Servic } } - private def recursivelyDelete(fileOrDir: File, deleteDir: Boolean) { + private def recursivelyDelete(fileOrDir: File, deleteDir: Boolean): Unit = { if (fileOrDir.isDirectory) { for (file <- fileOrDir.listFiles) recursivelyDelete(file, deleteDir) @@ -82,7 +82,7 @@ class IndexCleanupService(ctx: ServiceContext[ConfigurationArgs]) extends Servic fileOrDir.delete } - private def rename(srcDir: File, destDir: File) { + private def rename(srcDir: File, destDir: File): Unit = { if (!srcDir.isDirectory) { return } diff --git a/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala b/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala index 577ad767..26eff297 100644 --- a/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala +++ b/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala @@ -55,13 +55,13 @@ class IndexManagerService(ctx: ServiceContext[ConfigurationArgs]) extends Servic pid } - def put(path: String, pid: Pid) { + def put(path: String, pid: Pid) = { val prev = pathToPid.put(path, pid) pidToPath.remove(prev) pidToPath.put(pid, path) } - def remove(pid: Pid) { + def remove(pid: Pid) = { val path = pidToPath.remove(pid) pathToPid.remove(path) if (Option(path).isDefined) { @@ -73,13 +73,13 @@ class IndexManagerService(ctx: ServiceContext[ConfigurationArgs]) extends Servic pidToPath.isEmpty } - def close() { + def close() = { pidToPath.asScala foreach { kv => kv._1 ! ('close, 'closing) } } - def closeByPath(path: String) { + def closeByPath(path: String) = { pidToPath.asScala foreach { kv => if (kv._2.startsWith(path)) { @@ -104,7 +104,7 @@ class IndexManagerService(ctx: ServiceContext[ConfigurationArgs]) extends Servic metrics.gauge("NativeFSLock.count")(getNativeFSLockHeldSize(LOCK_HELD.asScala)) } - def getNativeFSLockHeldSize(lockHeld: Collection[String]) = lockHeld.synchronized { + def getNativeFSLockHeldSize(lockHeld: scala.collection.mutable.Set[String]) = lockHeld.synchronized { lockHeld.size } @@ -211,7 +211,7 @@ class IndexManagerService(ctx: ServiceContext[ConfigurationArgs]) extends Servic } } - private def replyAll(path: String, msg: Any) { + private def replyAll(path: String, msg: Any) = { waiters.remove(path) match { case Some(list) => for ((pid, ref) <- list) { diff --git a/src/main/scala/com/cloudant/clouseau/IndexService.scala b/src/main/scala/com/cloudant/clouseau/IndexService.scala index 28625047..358c5691 100644 --- a/src/main/scala/com/cloudant/clouseau/IndexService.scala +++ b/src/main/scala/com/cloudant/clouseau/IndexService.scala @@ -57,6 +57,9 @@ import scalang.Pid import com.spatial4j.core.context.SpatialContext import com.spatial4j.core.distance.DistanceUtils import java.util.HashSet +import conversions._ +import Utils.ensureElementsType +import java.lang.Throwable case class IndexServiceArgs(config: Configuration, name: String, queryParser: QueryParser, writer: IndexWriter) case class HighlightParameters(highlighter: Highlighter, highlightFields: List[String], highlightNumber: Int, analyzers: List[Analyzer]) @@ -66,6 +69,18 @@ case class TopDocs(updateSeq: Long, totalHits: Long, hits: List[Hit]) case class Hit(order: List[Any], fields: List[Any]) class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) with Instrumented { + import IndexService.{ + getDrilldown, + getGroups, + getIncludeFields, + getListOfStringsOption, + getOption, + getRanges, + GroupName, + RangesLabel, + RangesName, + RangesQuery + } var reader = DirectoryReader.open(ctx.args.writer, true) var updateSeq = getCommittedSeq @@ -192,7 +207,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w committing = false } - def countFields() { + def countFields() = { if (countFieldsEnabled) { val leaves = reader.leaves().iterator() val warningThreshold = ctx.args.config. @@ -211,7 +226,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w } } - override def exit(msg: Any) { + override def exit(msg: Any) = { debug("Closed with reason: %.1000s".format(msg)) try { reader.close() @@ -233,7 +248,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w } } - private def commit(newUpdateSeq: Long, newPurgeSeq: Long) { + private def commit(newUpdateSeq: Long, newPurgeSeq: Long) = { if (!committing && (newUpdateSeq > updateSeq || newPurgeSeq > purgeSeq)) { committing = true val index = self @@ -262,45 +277,22 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w val queryString = request.options.getOrElse('query, "*:*").asInstanceOf[String] val refresh = request.options.getOrElse('refresh, true).asInstanceOf[Boolean] val limit = request.options.getOrElse('limit, 25).asInstanceOf[Int] - val partition = request.options.getOrElse('partition, 'nil) match { - case 'nil => - None - case value => - Some(value.asInstanceOf[String]) - } - val counts = request.options.getOrElse('counts, 'nil) match { - case 'nil => - None - case value => - Some(value) - } - val ranges = request.options.getOrElse('ranges, 'nil) match { - case 'nil => - None - case value => - Some(value) - } + val partition: Option[String] = getOption[String](request.options, 'partition) - val includeFields: Set[String] = - request.options.getOrElse('include_fields, 'nil) match { - case 'nil => - null - case value: List[String] => - Set[String]() ++ ("_id" :: value).toSet - case other => - throw new ParseException(other + " is not a valid include_fields query") - } + val counts: Option[List[String]] = getListOfStringsOption(request.options, 'counts) + + val ranges = getRanges(request.options) + + val includeFields: Option[Set[String]] = getIncludeFields(request.options) val legacy = request.options.getOrElse('legacy, false).asInstanceOf[Boolean] parseQuery(queryString, partition) match { case baseQuery: Query => safeSearch { - val query = request.options.getOrElse('drilldown, Nil) match { - case Nil => - baseQuery - case categories: List[List[String]] => + val query = getDrilldown(request.options) match { + case Some(categories: List[_]) => { val drilldownQuery = new DrillDownQuery( FacetIndexingParams.DEFAULT, baseQuery) for (category <- categories) { @@ -326,16 +318,17 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w } } drilldownQuery - case _ => + } + case Some(_) => throw new ParseException("invalid drilldown query") + case None => baseQuery } - val searcher = getSearcher(refresh) val weight = searcher.createNormalizedWeight(query) val docsScoredInOrder = !weight.scoresDocsOutOfOrder val sort = parseSort(request.options.getOrElse('sort, 'relevance)).rewrite(searcher) - val after = toScoreDoc(sort, request.options.getOrElse('after, 'nil)) + val after = toScoreDoc(sort, getOption(request.options, 'after)) val hitsCollector = (limit, after, sort) match { case (0, _, _) => @@ -358,9 +351,9 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w case None => null case Some(rangeList: List[_]) => - val rangeFacetRequests: List[FacetRequest] = for ((name: String, ranges: List[_]) <- rangeList) yield { + val rangeFacetRequests: List[FacetRequest] = for ((name: RangesName, ranges: List[_]) <- rangeList) yield { new RangeFacetRequest(name, ranges.map({ - case (label: String, rangeQuery: String) => + case (label: RangesLabel, rangeQuery: RangesQuery) => ctx.args.queryParser.parse(rangeQuery) match { case q: NumericRangeQuery[_] => new DoubleRange( @@ -433,7 +426,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w } private def getHits(collector: Collector, searcher: IndexSearcher, - includeFields: Set[String], HPs: HighlightParameters = null) = + includeFields: Option[Set[String]], HPs: Option[HighlightParameters] = None) = collector match { case c: TopDocsCollector[_] => c.topDocs.scoreDocs.map({ docToHit(searcher, _, includeFields, HPs) }).toList @@ -441,11 +434,11 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w Nil } - private def createCountsCollector(counts: Option[Any]): FacetsCollector = { + private def createCountsCollector(counts: Option[List[String]]): FacetsCollector = { counts match { case None => null - case Some(counts: List[String]) => + case Some(counts: List[_]) => val state = try { new SortedSetDocValuesReaderState(reader) } catch { @@ -499,19 +492,11 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w val queryString = request.options.getOrElse('query, "*:*").asInstanceOf[String] val field = request.options('field).asInstanceOf[String] val refresh = request.options.getOrElse('refresh, true).asInstanceOf[Boolean] - val groups = request.options('groups).asInstanceOf[List[Any]] + val groups = getGroups(request.options).getOrElse(List()) val groupSort = request.options('group_sort) val docSort = request.options('sort) val docLimit = request.options.getOrElse('limit, 25).asInstanceOf[Int] - val includeFields: Set[String] = - request.options.getOrElse('include_fields, 'nil) match { - case 'nil => - null - case value: List[String] => - Set[String]() ++ ("_id" :: value).toSet - case other => - throw new ParseException(other + " is not a valid include_fields query") - } + val includeFields: Option[Set[String]] = getIncludeFields(request.options) parseQuery(queryString, None) match { case query: Query => val searcher = getSearcher(refresh) @@ -550,38 +535,31 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w } } - private def getHighlightParameters(options: Map[Symbol, Any], query: Query) = - options.getOrElse('highlight_fields, 'nil) match { - case 'nil => - null - case highlightFields: List[String] => { - val preTag = options.getOrElse('highlight_pre_tag, - "").asInstanceOf[String] - val postTag = options.getOrElse('highlight_post_tag, - "").asInstanceOf[String] - val highlightNumber = options.getOrElse('highlight_number, - 1).asInstanceOf[Int] //number of fragments - val highlightSize = options.getOrElse('highlight_size, 0). - asInstanceOf[Int] - val htmlFormatter = new SimpleHTMLFormatter(preTag, postTag) - val highlighter = new Highlighter(htmlFormatter, new QueryScorer(query)) - if (highlightSize > 0) { - highlighter.setTextFragmenter(new SimpleFragmenter(highlightSize)) - } - val analyzers = highlightFields.map { field => - ctx.args.queryParser.getAnalyzer() match { - case a1: PerFieldAnalyzer => - a1.getWrappedAnalyzer(field) - case a2: Analyzer => - a2 - } + private def getHighlightParameters(options: Map[Symbol, Any], query: Query): Option[HighlightParameters] = + getListOfStringsOption(options, 'highlight_fields).map(highlightFields => { + val preTag = options.getOrElse('highlight_pre_tag, + "").asInstanceOf[String] + val postTag = options.getOrElse('highlight_post_tag, + "").asInstanceOf[String] + val highlightNumber = options.getOrElse('highlight_number, + 1).asInstanceOf[Int] //number of fragments + val highlightSize = options.getOrElse('highlight_size, 0). + asInstanceOf[Int] + val htmlFormatter = new SimpleHTMLFormatter(preTag, postTag) + val highlighter = new Highlighter(htmlFormatter, new QueryScorer(query)) + if (highlightSize > 0) { + highlighter.setTextFragmenter(new SimpleFragmenter(highlightSize)) + } + val analyzers = highlightFields.map { field => + ctx.args.queryParser.getAnalyzer() match { + case a1: PerFieldAnalyzer => + a1.getWrappedAnalyzer(field) + case a2: Analyzer => + a2 } - HighlightParameters(highlighter, highlightFields, highlightNumber, - analyzers) } - case other => - throw new ParseException(other + " is not a valid highlight_fields query") - } + HighlightParameters(highlighter, highlightFields, highlightNumber, analyzers) + }) private def validateGroupField(field: String) = { IndexService.SORT_FIELD_RE.findFirstMatchIn(field) match { @@ -598,14 +576,14 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w } private def makeSearchGroup(group: Any): SearchGroup[BytesRef] = group match { - case ('null, order: List[AnyRef]) => + case (None, order: List[AnyRef @unchecked]) => val result: SearchGroup[BytesRef] = new SearchGroup - result.sortValues = order.toArray + result.sortValues = order.collect({ case ref: AnyRef => ref }).toArray result - case (name: String, order: List[AnyRef]) => + case (Some(name: String), order: List[AnyRef @unchecked]) => val result: SearchGroup[BytesRef] = new SearchGroup result.groupValue = name - result.sortValues = order.toArray + result.sortValues = order.collect({ case ref: AnyRef => ref }).toArray result } @@ -632,7 +610,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w ('error, ('bad_request, "Malformed query syntax")) case e: ParseException => ('error, ('bad_request, e.getMessage)) - case e => + case e: Throwable => ('error, e.getMessage) } @@ -643,7 +621,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w new IndexSearcher(reader) } - private def reopenIfChanged() { + private def reopenIfChanged() = { val newReader = DirectoryReader.openIfChanged(reader) if (newReader != null) { reader.close() @@ -696,17 +674,17 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w Sort.RELEVANCE case field: String => new Sort(toSortField(field)) - case fields: List[String] => + case fields: List[String @unchecked] => new Sort(fields.map(toSortField).toArray: _*) } private def docToHit(searcher: IndexSearcher, scoreDoc: ScoreDoc, - includeFields: Set[String] = null, HPs: HighlightParameters = null): Hit = { + includeFields: Option[Set[String]] = null, HPs: Option[HighlightParameters] = None): Hit = { val doc = includeFields match { - case null => + case None => searcher.doc(scoreDoc.doc) - case _ => - searcher.doc(scoreDoc.doc, includeFields.asJava) + case Some(fields) => + searcher.doc(scoreDoc.doc, fields.asJava) } var fields = doc.getFields.asScala.foldLeft(Map[String, Any]())((acc, field) => { @@ -719,7 +697,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w acc.get(field.name) match { case None => acc + (field.name -> value) - case Some(list: List[Any]) => + case Some(list: List[_]) => acc + (field.name -> (value :: list)) case Some(existingValue: Any) => acc + (field.name -> List(value, existingValue)) @@ -731,15 +709,19 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w case _ => List[Any](scoreDoc.score, scoreDoc.doc) } - if (HPs != null) { - val highlights = (HPs.highlightFields zip HPs.analyzers).map { - case (field, analyzer) => - (field, doc.getValues(field).flatMap { v => - HPs.highlighter.getBestFragments(analyzer, field, - v, HPs.highlightNumber).toList - }.toList) + + HPs match { + case Some(parameters: HighlightParameters) => { + val highlights = (parameters.highlightFields zip parameters.analyzers).map { + case (field, analyzer) => + (field, doc.getValues(field).flatMap { v => + parameters.highlighter.getBestFragments(analyzer, field, + v, parameters.highlightNumber).toList + }.toList) + } + fields += "_highlights" -> highlights.toList } - fields += "_highlights" -> highlights.toList + case None => () } Hit(order, fields.toList) } @@ -769,7 +751,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w case null => DistanceUtils.EARTH_EQUATORIAL_RADIUS_KM } val ctx = SpatialContext.GEO - val point = ctx.makePoint(lon toDouble, lat toDouble) + val point = ctx.makePoint(lon.toDouble, lat.toDouble) val degToKm = DistanceUtils.degrees2Dist(1, radius) val valueSource = new DistanceValueSource(ctx, fieldLon, fieldLat, degToKm, point) valueSource.getSortField(fieldOrder == "-") @@ -805,13 +787,13 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w (node.label.components.toList, node.value, children) } - private def toScoreDoc(sort: Sort, after: Any): Option[ScoreDoc] = after match { - case 'nil => + private def toScoreDoc(sort: Sort, after: Option[Any]): Option[ScoreDoc] = after match { + case None => None - case (score: Any, doc: Any) => + case Some((score: Any, doc: Any)) => Some(new ScoreDoc(ClouseauTypeFactory.toInteger(doc), ClouseauTypeFactory.toFloat(score))) - case list: List[Object] => + case Some(list: List[Object @unchecked]) => val doc = list.last sort.getSort match { case Array(SortField.FIELD_SCORE) => @@ -824,7 +806,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w throw new ParseException("sort order not compatible with given bookmark") } Some(new FieldDoc(ClouseauTypeFactory.toInteger(doc), - Float.NaN, (sortfields zip fields) map { + Float.NaN, sortfields.zip(fields).map { case (_, 'null) => null case (_, str: String) => @@ -839,7 +821,7 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w java.lang.Integer.valueOf(number.intValue()) case (_, field) => field - } toArray)) + }.toArray)) } } @@ -859,23 +841,23 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w ctx.args.writer.getConfig().getIndexDeletionPolicy().asInstanceOf[ExternalSnapshotDeletionPolicy] } - private def debug(str: String) { + private def debug(str: String) = { IndexService.logger.debug(prefix_name(str)) } - private def info(str: String) { + private def info(str: String) = { IndexService.logger.info(prefix_name(str)) } - private def warn(str: String) { + private def warn(str: String) = { IndexService.logger.warn(prefix_name(str)) } - private def warn(str: String, e: Throwable) { + private def warn(str: String, e: Throwable) = { IndexService.logger.warn(prefix_name(str), e) } - private def error(str: String, e: Throwable) { + private def error(str: String, e: Throwable) = { IndexService.logger.error(prefix_name(str), e) } @@ -932,4 +914,168 @@ object IndexService { dirCtor.newInstance(path, lockFactory).asInstanceOf[FSDirectory] } + /** + * Returns the Option[List[List[String]]] of a `drilldown` key from given map of options if it is in correct format. + * + * @param options: Map[Symbol, Any] - map of options to extract `drilldown` key + * @return An optional value of a `drilldown` key as `Option[List[List[String]]]` + * @throws ParseException + * + */ + + private[clouseau] def getDrilldown(options: Map[Symbol, Any]): Option[List[List[String]]] = { + val asListOfStrings: PartialFunction[Any, List[String]] = ensureElementsType( + { case string: String => string }, + { case (container, element) => throw new ParseException(container.toString + " contains non-string element " + element.toString) } + ) + + val asListofListsOfStrings: PartialFunction[Any, List[List[String]]] = ensureElementsType( + asListOfStrings, + { case (container, element) => throw new ParseException("invalid drilldown query " + container.toString) } + ) + + val extractor = asListofListsOfStrings.Extractor + getOption[Any](options, 'drilldown) match { + case Some(extractor(value)) => + Some(value) + case None => + None + } + } + + /** + * Returns the Option[value] of a `ranges` key from given map of options if it is in correct format. + * + * @param options: Map[Symbol, Any] - map of options to extract `ranges` key + * @return An optional value of a `ranges` key as `Option[List[Product2[RangesName, List[Product2[RangesLabel, RangesQuery]]]]]` + * @throws ParseException + * + * The function expects the `ranges` value to be in a following format + * + * ```erlang + * -type name :: string(). + * -type label :: string(). + * -type query :: string(). + * [{name(), [ + * {label(), query()} + * ]}] + * ``` + * + * or the same in scala format + * + * ```scala + * type Name = String + * type Label = String + * type Query = String + * List[(Name, List( + * (Label, Query) + * ))] + * ``` + * + */ + private[clouseau]type RangesLabel = String + private[clouseau]type RangesQuery = String + private[clouseau]type RangesName = String + + private[clouseau] def getRanges(options: Map[Symbol, Any]): Option[List[Product2[RangesName, List[Product2[RangesLabel, RangesQuery]]]]] = { + val asQueries: PartialFunction[Any, List[Product2[RangesLabel, RangesQuery]]] = ensureElementsType( + { case (label: RangesLabel, query: RangesQuery) => (label, query) }, + { case (_container, _element) => throw new ParseException("invalid ranges query") } + ) + + val asQueriesExtractor = asQueries.Extractor + + val asRange: PartialFunction[Any, List[Product2[RangesName, List[Product2[RangesLabel, RangesQuery]]]]] = ensureElementsType( + { case (name: RangesName, asQueriesExtractor(queries)) => (name, queries) }, + { case (container, _element) => throw new ParseException(container.toString + " is not a valid ranges query") } + ).orElse({ case notAList => throw new ParseException("invalid ranges query") }) + + val extractor = asRange.Extractor + + getOption[Any](options, 'ranges) match { + case Some(extractor(value)) => + Some(value) + case None => + None + } + } + + /** + * Returns the Option[List[String]] of a given option key from a map of options if each element of a list is indeed a String. + * + * @param options: Map[Symbol, Any] - map of options to extract given key + * @return An optional value of a given key as `Option[List[String]]` + * @throws ParseException + * + */ + private[clouseau] def getListOfStringsOption(options: Map[Symbol, Any], field: Symbol): Option[List[String]] = { + val asListOfStrings: PartialFunction[Any, List[String]] = ensureElementsType( + { case string: String => string }, + { case (container, _element) => throw new ParseException(container.toString + " is not a valid " + field + " query") } + ).orElse({ case notAList => throw new ParseException(notAList.toString + " is not a valid " + field + " query") }) + + val extractor = asListOfStrings.Extractor + getOption[Any](options, field) match { + case Some(extractor(value)) => + Some(value) + case None => + None + } + } + + /** + * Returns the Option[Set[String]] of a `include_fields` key from given map of options if each element of a list + * is indeed a String. + * + * It does convert List[String] to Set[String] before returning the result. It also injects an "_id" key into Set[String]. + * + * @param options: Map[Symbol, Any] - map of options to extract given key + * @return An optional value of a `include_fields` key as `Option[Set[String]]` + * @throws ParseException + * + */ + + private[clouseau] def getIncludeFields(options: Map[Symbol, Any]): Option[Set[String]] = { + getListOfStringsOption(options, 'include_fields).map(value => Set[String]() ++ ("_id" :: value).toSet) + } + + /** + * Returns the Option[List[Product2[Option[GroupName], List[Any]]]] of a `groups` key from given map of options + * if each element of a list of tuples with arity 2 and first element is a String. + * It does not check whether the second element of the tuple whether it is AnyRef or Any because of auto unboxing in scala. + * + * @param options: Map[Symbol, Any] - map of options to extract given key from + * @return An optional value of a `groups` key as `Option[List[Product2[Option[GroupName], List[Any]]]]` + * @throws ParseException + * + */ + + private[clouseau]type GroupName = String + private[clouseau] def getGroups(options: Map[Symbol, Any]): Option[List[Product2[Option[GroupName], List[Any]]]] = { + val asGroup: PartialFunction[Any, List[Product2[Option[GroupName], List[Any]]]] = ensureElementsType( + { + case (name: GroupName, order: List[_]) => (Some(name), order) + case ('null, order: List[_]) => (None, order) + }, + { case (container, _element) => throw new ParseException(container.toString + " is not a valid groups query") } + ).orElse({ case notAList => throw new ParseException("invalid groups query") }) + + val extractor = asGroup.Extractor + + getOption[Any](options, 'groups) match { + case Some(extractor(value)) => + Some(value) + case None => + None + } + } + + private[clouseau] def getOption[T](options: Map[Symbol, Any], field: Symbol): Option[T] = { + // Unfortunately CouchDB sends 'nil over the wire + options.get(field) match { + case Some('nil) => None + case Some(value) => Some(value.asInstanceOf[T]) + case None => None + } + } } diff --git a/src/main/scala/com/cloudant/clouseau/Main.scala b/src/main/scala/com/cloudant/clouseau/Main.scala index 3d317cab..c8b80157 100644 --- a/src/main/scala/com/cloudant/clouseau/Main.scala +++ b/src/main/scala/com/cloudant/clouseau/Main.scala @@ -25,7 +25,7 @@ object Main extends App { Thread.setDefaultUncaughtExceptionHandler( new Thread.UncaughtExceptionHandler { - def uncaughtException(t: Thread, e: Throwable) { + def uncaughtException(t: Thread, e: Throwable) = { logger.error("Uncaught exception: " + e.getMessage) System.exit(1) } diff --git a/src/main/scala/com/cloudant/clouseau/QueryExplainer.scala b/src/main/scala/com/cloudant/clouseau/QueryExplainer.scala index 7f4eec9b..b84c51c2 100644 --- a/src/main/scala/com/cloudant/clouseau/QueryExplainer.scala +++ b/src/main/scala/com/cloudant/clouseau/QueryExplainer.scala @@ -23,7 +23,7 @@ object QueryExplainer { builder.toString } - private def planBooleanQuery(builder: StringBuilder, query: BooleanQuery) { + private def planBooleanQuery(builder: StringBuilder, query: BooleanQuery): Unit = { for (clause <- query.getClauses) { builder.append(clause.getOccur) explain(builder, clause.getQuery) @@ -32,7 +32,7 @@ object QueryExplainer { builder.setLength(builder.length - 1) } - private def planFuzzyQuery(builder: StringBuilder, query: FuzzyQuery) { + private def planFuzzyQuery(builder: StringBuilder, query: FuzzyQuery) = { builder.append(query.getTerm) builder.append(",prefixLength=") builder.append(query.getPrefixLength) @@ -40,7 +40,7 @@ object QueryExplainer { builder.append(query.getMaxEdits) } - private def planNumericRangeQuery(builder: StringBuilder, query: NumericRangeQuery[_]) { + private def planNumericRangeQuery(builder: StringBuilder, query: NumericRangeQuery[_]) = { builder.append(query.getMin) builder.append(" TO ") builder.append(query.getMax) @@ -48,25 +48,25 @@ object QueryExplainer { builder.append(query.getMin.getClass.getSimpleName) } - private def planPrefixQuery(builder: StringBuilder, query: PrefixQuery) { + private def planPrefixQuery(builder: StringBuilder, query: PrefixQuery) = { builder.append(query.getPrefix) } - private def planTermQuery(builder: StringBuilder, query: TermQuery) { + private def planTermQuery(builder: StringBuilder, query: TermQuery) = { builder.append(query.getTerm) } - private def planTermRangeQuery(builder: StringBuilder, query: TermRangeQuery) { + private def planTermRangeQuery(builder: StringBuilder, query: TermRangeQuery) = { builder.append(query.getLowerTerm.utf8ToString) builder.append(" TO ") builder.append(query.getUpperTerm.utf8ToString) } - private def planWildcardQuery(builder: StringBuilder, query: WildcardQuery) { + private def planWildcardQuery(builder: StringBuilder, query: WildcardQuery) = { builder.append(query.getTerm) } - private def explain(builder: StringBuilder, query: Query) { + private def explain(builder: StringBuilder, query: Query) = { builder.append(query.getClass.getSimpleName) builder.append("(") query match { diff --git a/src/main/scala/com/cloudant/clouseau/Utils.scala b/src/main/scala/com/cloudant/clouseau/Utils.scala index 7b07f1ec..b84f5651 100644 --- a/src/main/scala/com/cloudant/clouseau/Utils.scala +++ b/src/main/scala/com/cloudant/clouseau/Utils.scala @@ -29,4 +29,55 @@ object Utils { new BytesRef(string) } + /** + * The function `ensureElementsType` maps over elements of a list and applies a partial function `pf` + * to each element until it encounters an element for which the `pf` is not defined. At that point, + * it stops iteration and calls the `orElse` partial function passing the original `list` + * and the current element. + * + * This function is intended to be used to refine a generic list to a list containing specific type of elements. + * For example, you can convert a `List[Any]` to `List[Inner]`, where `Inner` is a generic parameter + * of the function. + * + * @param pf - `PartialFunction[Any, Inner]` - Partial function to apply to each element of the given list + * @param orElse - `PartialFunction[List[Any], Any), Inner]` - Partial function to apply to the first element for which `pf` is not defined. + * + * Here's an example usage: + * + * ```scala + * val asListOfStrings: PartialFunction[Any, List[String]] = ensureElementsType( + * { case string: String => string }, + * { case (container, element) => throw new ParseException(container.toString + " contains non-string element " + element.toString) } + * ) + * + * val asListOfStringsExtractor = asListofStrings.Extractor + * + * value match { + * case Some(asDrilldownExtractor(value)) => + * Some(value) + * case None => + * None + * } + * ``` + * In this example, the `asListOfStrings` partial function will convert any list to a `List[String]`, + * and throw an exception if it encounters a non-string element. + */ + + def ensureElementsType[Inner](pf: PartialFunction[Any, Inner], orElse: PartialFunction[(List[Any], Any), List[Inner]]): PartialFunction[Any, List[Inner]] = { + case list: List[_] => { + val extractor = pf.lift + val result: Either[Any, List[Inner]] = list.foldLeft(Right(List.empty[Inner]): Either[Any, List[Inner]]) { + case (Right(acc), value) => extractor(value) match { + case Some(v) => Right(acc :+ v) + case None => Left(value) + } + case (err @ Left(_), _) => err + } + + result match { + case Right(list: List[_]) => list.asInstanceOf[List[Inner]] + case Left(err) => orElse.apply((list.asInstanceOf[List[Any]], err)) + } + } + } } diff --git a/src/test/scala/com/cloudant/clouseau/IndexServiceSpec.scala b/src/test/scala/com/cloudant/clouseau/IndexServiceSpec.scala index d3744b8d..3fade7df 100644 --- a/src/test/scala/com/cloudant/clouseau/IndexServiceSpec.scala +++ b/src/test/scala/com/cloudant/clouseau/IndexServiceSpec.scala @@ -25,10 +25,192 @@ import scalang.Pid import scala.Some import java.io.File import scala.collection.JavaConverters._ +import org.apache.lucene.queryparser.classic.ParseException +/* +mvn test -Dtest="com.cloudant.clouseau.IndexServiceSpec" +*/ class IndexServiceSpec extends SpecificationWithJUnit { sequential + "a groups parser" should { + def groups(value: Any): Map[Symbol, Any] = Map('groups -> value) + + "return None if option map doesn't contain 'groups'" in { + IndexService.getGroups(Map().asInstanceOf[Map[Symbol, Any]]) must beEqualTo(None) + } + + "return None if 'groups option contain 'nil" in { + IndexService.getGroups(groups('nil)) must beEqualTo(None) + } + + "return correct result for empty list groups" in { + IndexService.getGroups(groups(List())) must beEqualTo(Some(List())) + } + + "return correct result for groups in the expected format" in { + var input = List( + ("first", List(List())), ("second", List(Seq())), ("third", List("String")), + ('null, List(List())), ('null, List(Seq())), ('null, List("String")) + ) + IndexService.getGroups(groups(input)) must beEqualTo(Some(List( + (Some("first"), List(List())), (Some("second"), List(Seq())), (Some("third"), List("String")), + (None, List(List())), (None, List(Seq())), (None, List("String")) + ))) + } + + "return error when groups is not a list" in { + IndexService.getGroups(groups("not a List")) must throwA[ParseException].like { + case e => e.getMessage must contain("invalid groups query") + } + } + + } + + "an include_fields parser" should { + def includeFields(value: Any): Map[Symbol, Any] = Map('include_fields -> value) + + "return None if option map doesn't contain 'include_fields'" in { + IndexService.getIncludeFields(Map().asInstanceOf[Map[Symbol, Any]]) must beEqualTo(None) + } + + "return None if 'include_fields option contain 'nil" in { + IndexService.getIncludeFields(includeFields('nil)) must beEqualTo(None) + } + + "return correct result for empty list include_fields" in { + IndexService.getIncludeFields(includeFields(List())) must beEqualTo(Some(Set("_id"))) + } + + "return correct result for List[String]" in { + var planets = List("Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune") + IndexService.getIncludeFields(includeFields(planets)) must beEqualTo(Some( + Set("Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune", "_id") + )) + } + + "return error when include_fields is not a list" in { + IndexService.getIncludeFields(includeFields("not a List")) must throwA[ParseException].like { + case e => e.getMessage must contain("not a List is not a valid 'include_fields query") + } + } + + "return error when one of the elements of include_fields is not a String" in { + IndexService.getIncludeFields(includeFields(List("one", 2))) must throwA[ParseException].like { + case e => e.getMessage must contain("List(one, 2) is not a valid 'include_fields query") + } + } + } + + "a ranges parser" should { + def ranges(value: Any): Map[Symbol, Any] = Map('ranges -> value) + + "return None if option map doesn't contain 'ranges'" in { + IndexService.getRanges(Map().asInstanceOf[Map[Symbol, Any]]) must beEqualTo(None) + } + + "return None if 'ranges option contain 'nil" in { + IndexService.getRanges(ranges('nil)) must beEqualTo(None) + } + + "return correct result for empty list ranges" in { + IndexService.getRanges(ranges(List())) must beEqualTo(Some(List())) + } + + "return correct result for empty list of queries" in { + val input = List(("first", List())) + IndexService.getRanges(ranges(input)) must beEqualTo(Some(input)) + } + + "return correct result when ranges are in correct format" in { + val input = List(("first", List(("second", "third")))) + IndexService.getRanges(ranges(input)) must beEqualTo(Some(input)) + } + + "return error when ranges is not a list" in { + IndexService.getRanges(ranges("not a List")) must throwA[ParseException].like { + case e => e.getMessage must contain("invalid ranges query") + } + } + + "return error when range name is not a String" in { + IndexService.getRanges(ranges(List(1, List()))) must throwA[ParseException].like { + case e => e.getMessage must contain("List(1, List()) is not a valid ranges query") + } + } + + "return error when query is not tuple with arity 2" in { + IndexService.getRanges(ranges(List("first", List(("second", "third"), (1, 2, 3))))) must throwA[ParseException].like { + case e => e.getMessage must contain("List(first, List((second,third), (1,2,3))) is not a valid ranges query") + } + } + + "return error when query label is not a String" in { + IndexService.getRanges(ranges(List("first", List((1, "query"))))) must throwA[ParseException].like { + case e => e.getMessage must contain("List(first, List((1,query))) is not a valid ranges query") + } + } + + "return error when query is not a list" in { + IndexService.getRanges(ranges(List("first", List(("second", 1))))) must throwA[ParseException].like { + case e => e.getMessage must contain("List(first, List((second,1))) is not a valid ranges query") + } + } + + "return error when queries is not a list" in { + IndexService.getRanges(ranges(List("first", "not a List"))) must throwA[ParseException].like { + case e => e.getMessage must contain("List(first, not a List) is not a valid ranges query") + } + } + } + + "a drilldown parser" should { + def drilldown(value: Any): Map[Symbol, Any] = Map('drilldown -> value) + + "return None if option map doesn't contain 'drilldown'" in { + IndexService.getDrilldown(Map().asInstanceOf[Map[Symbol, Any]]) must beEqualTo(None) + } + + "return None if 'drilldown option contain 'nil" in { + IndexService.getDrilldown(drilldown('nil)) must beEqualTo(None) + } + + "return correct result for single inner list of List[List[String]]" in { + var planets = List(List("Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune")) + IndexService.getDrilldown(drilldown(planets)) must beEqualTo(Some(planets)) + } + + "return correct result for multiple inner lists of List[List[String]]" in { + var planets = List(List("Mercury", "Venus", "Earth"), List("Mars", "Jupiter", "Saturn"), List("Uranus", "Neptune")) + IndexService.getDrilldown(drilldown(planets)) must beEqualTo(Some(planets)) + } + + "return error on List[NotAList]" in { + IndexService.getDrilldown(drilldown(List(1, 2))) must throwA[ParseException].like { + case e => e.getMessage must contain("invalid drilldown query List(1, 2)") + } + } + + "return error on List[MixOFListsAndNoneLists]" in { + IndexService.getDrilldown(drilldown(List(List("1", "2"), "3", "4"))) must throwA[ParseException].like { + case e => e.getMessage must contain("invalid drilldown query List(List(1, 2), 3, 4)") + } + } + + "return error on List[List[NotAString]]" in { + IndexService.getDrilldown(drilldown(List(List(1, 2)))) must throwA[ParseException].like { + case e => e.getMessage must contain("List(1, 2) contains non-string element 1") + } + } + + "return error on List[List[MixOfStringsAndNoneStrings]]" in { + IndexService.getDrilldown(drilldown(List(List("1", 2, "3")))) must throwA[ParseException].like { + case e => e.getMessage must contain("List(1, 2, 3) contains non-string element 2") + } + } + + } + "an index" should { "not be closed if close_if_idle and idle_check_interval_secs not set" in new index_service { @@ -515,7 +697,7 @@ class IndexServiceSpec extends SpecificationWithJUnit { }) } - private def indexNotClosedAfterTimeout(node: Node, service: Pid) { + private def indexNotClosedAfterTimeout(node: Node, service: Pid) = { val value, query = "foo" val doc = new Document() doc.add(new StringField("_id", value, Field.Store.YES)) @@ -531,7 +713,7 @@ class IndexServiceSpec extends SpecificationWithJUnit { (node.isAlive(service) must beTrue) } - private def indexClosedAfterTimeOut(node: Node, service: Pid) { + private def indexClosedAfterTimeOut(node: Node, service: Pid) = { val value, query = "foo" val doc = new Document() doc.add(new StringField("_id", value, Field.Store.YES)) @@ -548,7 +730,7 @@ class IndexServiceSpec extends SpecificationWithJUnit { } private def indexNotClosedAfterActivityBetweenTwoIdleChecks(node: Node, - service: Pid) { + service: Pid) = { var value, query = "foo" var doc = new Document() doc.add(new StringField("_id", value, Field.Store.YES)) diff --git a/src/test/scala/com/cloudant/clouseau/UtilsSpec.scala b/src/test/scala/com/cloudant/clouseau/UtilsSpec.scala new file mode 100644 index 00000000..2f637c01 --- /dev/null +++ b/src/test/scala/com/cloudant/clouseau/UtilsSpec.scala @@ -0,0 +1,85 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +/** + * mvn test -Dtest="com.cloudant.clouseau.UtilsSpec" + */ + +package com.cloudant.clouseau + +import org.specs2.mutable.SpecificationWithJUnit +import scala.Some +import Utils.ensureElementsType +import conversions._ +import scala.collection.mutable + +class UtilsSpec extends SpecificationWithJUnit { + "an ensureElementsType" should { + val asListOfStrings: PartialFunction[Any, List[String]] = ensureElementsType( + { case string: String => string }, + { case (container, element) => throw new Exception(container.toString + " contains non-string element " + element.toString) } + ) + val extractor = asListOfStrings.Extractor + + "throw exception when encounter non matching case" in { + val input = List("1", "2", 3, "4") + input must beLike { case extractor(input) => ko } must throwA[Exception].like { + case e => e.getMessage must contain("List(1, 2, 3, 4) contains non-string element 3") + } + } + + "skip processing of elements after first element of wrong type" in { + val input = List("1", "2", 3, "4") + val recorder: mutable.ListBuffer[Any] = mutable.ListBuffer.empty + val condition: PartialFunction[Any, List[String]] = ensureElementsType( + { + case string: String => { + recorder += string + string + } + }, + { + case (_container, element) => { + recorder += element + throw new MatchError(element) + } + } + ) + val extractor = condition.Extractor + input must beLike { case extractor(input) => ok } must throwA[MatchError] + recorder must beEqualTo(mutable.ListBuffer("1", "2", 3)) + } + + "usable when orElse doesn't throw exception" in { + /* + This is not supported use case for the function. + Because there is no way to distinguish the success and failure. + + This test is written only to verify that orElse function works correctly. + */ + val condition: PartialFunction[Any, List[String]] = ensureElementsType( + { + case string: String => { string } + }, + { + case (container, element) => { List(element.toString) } + } + ) + val extractor = condition.Extractor + + List("1", "2", "3", "4") must beLike { case extractor(input) => input must beEqualTo(List("1", "2", "3", "4")) } + List("1", "2", 3, "4") must beLike { case extractor(input) => input must beEqualTo(List("3")) } + List("1") must beLike { case extractor(input) => input must beEqualTo(List("1")) } + List(1) must beLike { case extractor(input) => input must beEqualTo(List("1")) } + } + } +}