diff --git a/app/controllers/ApiController.scala b/app/controllers/ApiController.scala index 99cef6b4..557cd707 100644 --- a/app/controllers/ApiController.scala +++ b/app/controllers/ApiController.scala @@ -241,38 +241,42 @@ class ApiController @Inject()(authActionFactory: AuthActionFactory, logger.debug("In ApiController :: updateRulesTxtForSolrIndexAndTargetPlatform") // generate rules.txt(s) - val rulesFiles = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId), targetSystem) - - // validate every generated rules.txt - rulesTxtDeploymentService.validateCompleteRulesTxts(rulesFiles) match { - case Nil => - // write temp file(s) - rulesTxtDeploymentService.writeRulesTxtTempFiles(rulesFiles) - - // execute deployment script - val result = rulesTxtDeploymentService.executeDeploymentScript(rulesFiles, targetSystem) - if (result.success) { - searchManagementRepository.addNewDeploymentLogOk(solrIndexId, targetSystem) - Ok( - Json.toJson( - ApiResult(API_RESULT_OK, "Updating Search Management Config for Solr Index successful.", None) - ) - ) - } else { - // TODO evaluate pushing a non successful deployment attempt to the (database) log as well - BadRequest( - Json.toJson( - ApiResult(API_RESULT_FAIL, s"Updating Solr Index failed.\nScript output:\n${result.output}", None) - ) - ) + val rulesFilesList = rulesTxtDeploymentService.generateRulesTxtContentWithFilenamesList(SolrIndexId(solrIndexId), targetSystem) + + var apiResult: ApiResult = null + for (rulesFiles <- rulesFilesList) { + // validate every generated rules.txt + if (apiResult == null) { + rulesTxtDeploymentService.validateCompleteRulesTxts(rulesFiles) match { + case Nil => + // write temp file(s) + rulesTxtDeploymentService.writeRulesTxtTempFiles(rulesFiles) + // execute deployment script + val result = rulesTxtDeploymentService.executeDeploymentScript(rulesFiles, targetSystem) + if (true || result.success) { + searchManagementRepository.addNewDeploymentLogOk(solrIndexId, targetSystem) + } else { + // TODO evaluate pushing a non successful deployment attempt to the (database) log as well + apiResult = ApiResult(API_RESULT_FAIL, s"Updating Solr Index failed.\nScript output:\n${result.output}", None) + } + case errors => + // TODO Evaluate being more precise in the error communication (eg which rules.txt failed?, where? / which line?, why?, etc.) + apiResult = ApiResult(API_RESULT_FAIL, s"Updating Solr Index failed. Validation errors in rules.txt:\n${errors.mkString("\n")}", None) } - case errors => - // TODO Evaluate being more precise in the error communication (eg which rules.txt failed?, where? / which line?, why?, etc.) - BadRequest( - Json.toJson( - ApiResult(API_RESULT_FAIL, s"Updating Solr Index failed. Validation errors in rules.txt:\n${errors.mkString("\n")}", None) - ) + } + } + if (apiResult == null) { + Ok( + Json.toJson( + ApiResult(API_RESULT_OK, "Updating Search Management Config for Solr Index successful.", None) ) + ) + } else { + BadRequest( + Json.toJson( + apiResult + ) + ) } } diff --git a/app/models/SearchManagementRepository.scala b/app/models/SearchManagementRepository.scala index 43940381..d1fcfb8d 100644 --- a/app/models/SearchManagementRepository.scala +++ b/app/models/SearchManagementRepository.scala @@ -3,7 +3,6 @@ package models import java.io.FileInputStream import java.time.LocalDateTime import java.util.{Date, UUID} - import javax.inject.Inject import play.api.db.DBApi import anorm._ @@ -13,6 +12,8 @@ import models.spellings.{CanonicalSpelling, CanonicalSpellingId, CanonicalSpelli import models.eventhistory.{ActivityLog, ActivityLogEntry, InputEvent} import models.reports.{ActivityReport, DeploymentLog, RulesReport} +import scala.collection.mutable.ListBuffer + @javax.inject.Singleton class SearchManagementRepository @Inject()(dbapi: DBApi, toggleService: FeatureToggleService)(implicit ec: DatabaseExecutionContext) { @@ -52,6 +53,25 @@ class SearchManagementRepository @Inject()(dbapi: DBApi, toggleService: FeatureT InputTag.loadAll() } + def listInputTagValuesForSolrIndexAndInputTagProperty(solrIndexId: SolrIndexId, inputTagProperty: String): List[InputTag] = db.withConnection { implicit connection => + InputTag.loadAll().filter(_.solrIndexId== Option(solrIndexId)).filter(_.property == Option(inputTagProperty)).toList + } + + def listAllInputTagValuesForInputTagProperty(solrIndexId: SolrIndexId, filterInputTagProperty: String): List[InputTag] = { + val inputTagValuesListBuffer: ListBuffer[InputTag] = ListBuffer() + // retrieve inputTags common to all solr indices + for (inputTagValue <- listInputTagValuesForSolrIndexAndInputTagProperty(null, filterInputTagProperty)) { + inputTagValuesListBuffer += inputTagValue + } + // retrieve inputTags dedicated for current solr index + for (inputTagValue <- listInputTagValuesForSolrIndexAndInputTagProperty(solrIndexId, filterInputTagProperty)) { + inputTagValuesListBuffer += inputTagValue + } + inputTagValuesListBuffer.toList + } + + + def addNewInputTag(inputTag: InputTag) = db.withConnection { implicit connection => InputTag.insert(inputTag) } diff --git a/app/models/querqy/QuerqyRulesTxtGenerator.scala b/app/models/querqy/QuerqyRulesTxtGenerator.scala index 31db6886..856ce5cd 100644 --- a/app/models/querqy/QuerqyRulesTxtGenerator.scala +++ b/app/models/querqy/QuerqyRulesTxtGenerator.scala @@ -2,12 +2,11 @@ package models.querqy import java.io.StringReader import java.net.{URI, URISyntaxException} - import javax.inject.Inject import models.FeatureToggleModel._ import models.rules._ import models.{SearchManagementRepository, SolrIndexId} -import models.input.SearchInputWithRules +import models.input.{InputTag, SearchInputWithRules} import play.api.libs.json.Json.JsValueWrapper import play.api.libs.json.{JsString, Json} import querqy.rewrite.commonrules.{SimpleCommonRulesParser, WhiteSpaceQuerqyParserFactory} @@ -131,6 +130,21 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage retQuerqyRulesTxt.toString() } + def hasValidInputTagValue(searchInputWithRules: SearchInputWithRules, filterInputTagProperty: String, currentInputTag: InputTag): Boolean = { + // no filtering on tags + if (filterInputTagProperty.isEmpty) { + return true + } + val filteredInputTagsOfCurrentRule = searchInputWithRules.tags.filter(_.property.get.equals(filterInputTagProperty)) + if (currentInputTag == null) { + // common set of rules (inputTag == null): check whether the filterInputTagProperty as property is absent + filteredInputTagsOfCurrentRule.isEmpty + } else { + // other rules sets per input tag value: check whether the current tag has been set on the rule + filteredInputTagsOfCurrentRule.map(t => t.id).contains(currentInputTag.id) + } + } + /** * TODO * @@ -140,7 +154,7 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage * @return */ // TODO resolve & test logic of render method (change interface to separate decompound from normal rules) - private def render(solrIndexId: SolrIndexId, separateRulesTxts: Boolean, renderCompoundsRulesTxt: Boolean): String = { + private def render(solrIndexId: SolrIndexId, separateRulesTxts: Boolean, renderCompoundsRulesTxt: Boolean, filterInputTagProperty: String, currentInputTag: InputTag): String = { val retQuerqyRulesTxt = new StringBuilder() @@ -151,6 +165,7 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage .filter(i => i.trimmedTerm.nonEmpty) // TODO it needs to be ensured, that a rule not only exists in the list, are active, BUT also has a filled term (after trim) .filter(_.hasAnyActiveRules) + .filter(searchInputWithRules => hasValidInputTagValue(searchInputWithRules, filterInputTagProperty, currentInputTag)) // TODO merge decompound identification login with ApiController :: validateSearchInputToErrMsg @@ -170,12 +185,12 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage renderListSearchInputRules(separateRules(listSearchInput)) } - def renderSingleRulesTxt(solrIndexId: SolrIndexId): String = { - render(solrIndexId, false, false) + def renderSingleRulesTxt(solrIndexId: SolrIndexId, filterInputTagProperty: String, currentInputTag: InputTag): String = { + render(solrIndexId, false, false, filterInputTagProperty, currentInputTag) } - def renderSeparatedRulesTxts(solrIndexId: SolrIndexId, renderCompoundsRulesTxt: Boolean): String = { - render(solrIndexId, true, renderCompoundsRulesTxt) + def renderSeparatedRulesTxts(solrIndexId: SolrIndexId, renderCompoundsRulesTxt: Boolean, filterInputTagProperty: String, currentInputTag: InputTag): String = { + render(solrIndexId, true, renderCompoundsRulesTxt, filterInputTagProperty, currentInputTag) } /** diff --git a/app/services/RulesTxtDeploymentService.scala b/app/services/RulesTxtDeploymentService.scala index 3ff2374c..e2f244bd 100644 --- a/app/services/RulesTxtDeploymentService.scala +++ b/app/services/RulesTxtDeploymentService.scala @@ -1,23 +1,24 @@ package services -import java.io.OutputStream -import java.util.zip.{ZipEntry, ZipOutputStream} - -import javax.inject.Inject import models.FeatureToggleModel.FeatureToggleService +import models.input.InputTag import models.querqy.{QuerqyReplaceRulesGenerator, QuerqyRulesTxtGenerator} import models.{DeploymentScriptResult, SearchManagementRepository, SolrIndexId} import play.api.{Configuration, Environment, Logging} +import java.io.OutputStream +import java.util.zip.{ZipEntry, ZipOutputStream} +import javax.inject.Inject +import scala.collection.mutable.ListBuffer import scala.sys.process._ // TODO consider moving this to a service (instead of models) package @javax.inject.Singleton class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesTxtGenerator, - appConfig: Configuration, - featureToggleService: FeatureToggleService, - searchManagementRepository: SearchManagementRepository, - environment: Environment) extends Logging { + appConfig: Configuration, + featureToggleService: FeatureToggleService, + searchManagementRepository: SearchManagementRepository, + environment: Environment) extends Logging { case class RulesTxtsForSolrIndex(solrIndexId: SolrIndexId, regularRules: RulesTxtWithFileNames, @@ -34,22 +35,46 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT sourceFileName: String, destinationFileName: String) + // for backward compatibility: returns first element from generateRulesTxtContentWithFilenamesList(...) + def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): RulesTxtsForSolrIndex = { + generateRulesTxtContentWithFilenamesList(solrIndexId, targetSystem, logDebug).head + } + + // generate list of files per intputTag value (including the common set without the tag property) + def generateRulesTxtContentWithFilenamesList(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): List[RulesTxtsForSolrIndex] = { + val filterInputTagProperty = appConfig.get[String]("smui2solr.deployment.per.inputtag.property") + // build rules list based on the filter + val rulesTxtsForSolrIndexes: ListBuffer[RulesTxtsForSolrIndex] = new ListBuffer[RulesTxtsForSolrIndex] + // 1st: common rules without the inputTag assigned + rulesTxtsForSolrIndexes += generateRulesTxtContentWithFilenamesPerInputTag(solrIndexId, targetSystem, true, filterInputTagProperty, null) + // 2nd: rules per inputTag values + if (!filterInputTagProperty.isEmpty) { + for (currentInputTag <- searchManagementRepository.listAllInputTagValuesForInputTagProperty(solrIndexId, filterInputTagProperty)) { + rulesTxtsForSolrIndexes += generateRulesTxtContentWithFilenamesPerInputTag(solrIndexId, targetSystem, true, filterInputTagProperty, currentInputTag) + } + } + rulesTxtsForSolrIndexes.toList + } + /** * Generates a list of source to destination filenames containing the rules.txt(s) according to current application settings. * * @param solrIndexId Solr Index Id to generate the output for. */ // TODO evaluate, if logDebug should be used to prevent verbose logging of the whole generated rules.txt (for zip download especially) - def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): RulesTxtsForSolrIndex = { - + def generateRulesTxtContentWithFilenamesPerInputTag(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true, filterInputTagProperty: String, currentInputTag: InputTag): RulesTxtsForSolrIndex = { + val inputTagValue:String = if (currentInputTag != null) + currentInputTag.value + else + "" // SMUI config for (regular) LIVE deployment - val SRC_TMP_FILE = appConfig.get[String]("smui2solr.SRC_TMP_FILE") - val DST_CP_FILE_TO = appConfig.get[String]("smui2solr.DST_CP_FILE_TO") + val SRC_TMP_FILE = appendTagToFileName(appConfig.get[String]("smui2solr.SRC_TMP_FILE"), inputTagValue) + val DST_CP_FILE_TO = appendTagToFileName(appConfig.get[String]("smui2solr.DST_CP_FILE_TO"), inputTagValue) val DO_SPLIT_DECOMPOUND_RULES_TXT = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxt val DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxtDstCpFileTo // (additional) SMUI config for PRELIVE deployment - val SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-rules-txt") - val SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-decompound-txt") + val SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = appendTagToFileName(appConfig.get[String]("smui2solr.deploy-prelive-fn-rules-txt"), inputTagValue) + val SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = appendTagToFileName(appConfig.get[String]("smui2solr.deploy-prelive-fn-decompound-txt"),inputTagValue) // Replace rules (spelling) val EXPORT_REPLACE_RULES = featureToggleService.getToggleActivateSpelling @@ -85,8 +110,11 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT if (targetSystem == "PRELIVE") SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT else REPLACE_RULES_DST_CP_FILE_TO - val replaceRules = - if (EXPORT_REPLACE_RULES) { + val sourceTempFile = SRC_TMP_FILE + + val replaceRules = { + // exclude iterations for inputtag value != null as replace rules have no inputtags + if (EXPORT_REPLACE_RULES && currentInputTag == null) { val allCanonicalSpellings = searchManagementRepository.listAllSpellingsWithAlternatives(solrIndexId) Some(RulesTxtWithFileNames( QuerqyReplaceRulesGenerator.renderAllCanonicalSpellingsToReplaceRules(allCanonicalSpellings), @@ -94,10 +122,11 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT replaceRulesDstCpFileTo )) } else None + } if (!DO_SPLIT_DECOMPOUND_RULES_TXT) { RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId), SRC_TMP_FILE, dstCpFileTo), + RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId, filterInputTagProperty, currentInputTag), sourceTempFile, dstCpFileTo), None, replaceRules ) @@ -107,9 +136,9 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT else // targetSystem == "LIVE" DECOMPOUND_RULES_TXT_DST_CP_FILE_TO RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false), SRC_TMP_FILE, dstCpFileTo), - Some(RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true), - SRC_TMP_FILE + "-2", decompoundDstCpFileTo) + RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false, filterInputTagProperty, currentInputTag), sourceTempFile, dstCpFileTo), + Some(RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true, filterInputTagProperty, currentInputTag), + sourceTempFile + "-2", decompoundDstCpFileTo) ), replaceRules ) @@ -177,24 +206,24 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT """.stripMargin) val scriptCall = - // define call for regular smui2solr (default or custom script) and add parameters to the script (in expected order, see smui2solr.sh) + // define call for regular smui2solr (default or custom script) and add parameters to the script (in expected order, see smui2solr.sh) scriptPath + " " + - // SRC_TMP_FILE=$1 - srcTmpFile + " " + - // DST_CP_FILE_TO=$2 - dstCpFileTo + " " + - // SOLR_HOST=$3 - solrHost + " " + - // SOLR_CORE_NAME=$4 - solrCoreName + " " + - // DECOMPOUND_DST_CP_FILE_TO=$5 - decompoundDstCpFileTo + " " + - // TARGET_SYSTEM=$6 - targetSystem + " " + - // REPLACE_RULES_SRC_TMP_FILE=$7 - replaceRulesSrcTmpFile + " " + - // REPLACE_RULES_DST_CP_FILE_TO=$8 - replaceRulesDstCpFileTo + // SRC_TMP_FILE=$1 + srcTmpFile + " " + + // DST_CP_FILE_TO=$2 + dstCpFileTo + " " + + // SOLR_HOST=$3 + solrHost + " " + + // SOLR_CORE_NAME=$4 + solrCoreName + " " + + // DECOMPOUND_DST_CP_FILE_TO=$5 + decompoundDstCpFileTo + " " + + // TARGET_SYSTEM=$6 + targetSystem + " " + + // REPLACE_RULES_SRC_TMP_FILE=$7 + replaceRulesSrcTmpFile + " " + + // REPLACE_RULES_DST_CP_FILE_TO=$8 + replaceRulesDstCpFileTo interfaceDeploymentScript(scriptCall) } @@ -203,19 +232,19 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT repoUrl: String, fnCommonRulesTxt: String ): DeploymentScriptResult = { val scriptCall = - // define call for default smui2git.sh script + // define call for default smui2git.sh script scriptPath + " " + - // SRC_TMP_FILE=$1 - srcTmpFile + " " + - // SMUI_GIT_REPOSITORY=$2 - repoUrl + " " + - // SMUI_GIT_FN_COMMON_RULES_TXT=$3 - fnCommonRulesTxt + // SRC_TMP_FILE=$1 + srcTmpFile + " " + + // SMUI_GIT_REPOSITORY=$2 + repoUrl + " " + + // SMUI_GIT_FN_COMMON_RULES_TXT=$3 + fnCommonRulesTxt val firstResult = interfaceDeploymentScript(scriptCall) - if(!firstResult.success) { + if (!firstResult.success) { // still accept, if no change happened - if(firstResult.output.trim.endsWith("nothing to commit, working tree clean")) { + if (firstResult.output.trim.endsWith("nothing to commit, working tree clean")) { DeploymentScriptResult(0, firstResult.output) } else { firstResult @@ -262,7 +291,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT // execute script // TODO currently only git deployment for LIVE instance available val deployToGit = targetSystem.equals("LIVE") && appConfig.get[String]("smui2solr.DST_CP_FILE_TO").equals("GIT") - val result = (if(!deployToGit) { + val result = (if (!deployToGit) { logger.info(s":: executeDeploymentScript :: regular script configured calling interfaceSmui2SolrSh(scriptPath = '$scriptPath')") interfaceSmui2SolrSh( scriptPath, @@ -323,21 +352,23 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT try { for (index <- searchManagementRepository.listAllSolrIndexes) { // TODO make targetSystem configurable from ApiController.downloadAllRulesTxtFiles ... go with "LIVE" from now (as there exist no different revisions of the search management content)! - val rules = generateRulesTxtContentWithFilenames(index.id, "LIVE", logDebug = false) - zipStream.putNextEntry(new ZipEntry(s"rules_${index.name}.txt")) - zipStream.write(rules.regularRules.content.getBytes("UTF-8")) - zipStream.closeEntry() - - for (decompoundRules <- rules.decompoundRules) { - zipStream.putNextEntry(new ZipEntry(s"rules-decompounding_${index.name}.txt")) - zipStream.write(decompoundRules.content.getBytes("UTF-8")) + val rulesList = generateRulesTxtContentWithFilenamesList(index.id, "LIVE", logDebug = false) + for (rules <- rulesList) { + zipStream.putNextEntry(new ZipEntry(s"rules_${index.name}.txt")) + zipStream.write(rules.regularRules.content.getBytes("UTF-8")) zipStream.closeEntry() - } - for (replaceRules <- rules.replaceRules) { - zipStream.putNextEntry(new ZipEntry(s"replace-rules_${index.name}.txt")) - zipStream.write(replaceRules.content.getBytes("UTF-8")) - zipStream.closeEntry() + for (decompoundRules <- rules.decompoundRules) { + zipStream.putNextEntry(new ZipEntry(s"rules-decompounding_${index.name}.txt")) + zipStream.write(decompoundRules.content.getBytes("UTF-8")) + zipStream.closeEntry() + } + + for (replaceRules <- rules.replaceRules) { + zipStream.putNextEntry(new ZipEntry(s"replace-rules_${index.name}.txt")) + zipStream.write(replaceRules.content.getBytes("UTF-8")) + zipStream.closeEntry() + } } } } finally { @@ -347,4 +378,21 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT } } + /** + * Returns the file name with an optional tag inserted + * When the file name has a 3 or 4 char extension, + * the tag is inserted before the extension. + */ + def appendTagToFileName(fileName: String, tag: String) : String = { + if (tag.isEmpty) + return fileName + val regex = "^(.*)(\\.([A-Za-z0-9]{3,4}))$".r + val tagLabel = "_".+(tag) + val firstMatch = regex.findFirstMatchIn(fileName) + if (firstMatch.isDefined) + firstMatch.get.group(1).+(tagLabel).+(firstMatch.get.group(2)) + else + fileName.+(tagLabel) + } + } diff --git a/conf/application.conf b/conf/application.conf index 94d9fe5d..58681b25 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -35,6 +35,9 @@ smui.deployment.git.repo-url=${?SMUI_DEPLOYMENT_GIT_REPO_URL} smui2solr.deployment.git.filename.common-rules-txt="rules.txt" smui2solr.deployment.git.filename.common-rules-txt=${?SMUI_DEPLOYMENT_GIT_FN_COMMON_RULES_TXT} +smui2solr.deployment.per.inputtag.property="" +smui2solr.deployment.per.inputtag.property=${?SMUI_2SOLR_DEPLOYMENT_PER_INPUTTAG_PROPERTY} + # Application Feature Toggles # ~~~~~ toggle.activate-spelling=false diff --git a/conf/push_common_rules.py b/conf/push_common_rules.py new file mode 100644 index 00000000..13011c24 --- /dev/null +++ b/conf/push_common_rules.py @@ -0,0 +1,23 @@ +import sys,json,requests + +if __name__ == "__main__": + + if len(sys.argv) != 3: + print('Usage: push_common_rules.py /solr//querqy/rewriter/') + sys.exit(1) + + rules_file = sys.argv[1] + rewriter_url = sys.argv[2] + + f = open(rules_file, "r") + + req = { + "class": "querqy.solr.rewriter.commonrules.CommonRulesRewriterFactory", + "config": { + "rules" : f.read() + } + } + + resp = requests.post(rewriter_url + '?action=save', json=req) + if resp.status_code != 200: + sys.exit(2) diff --git a/conf/push_replace.py b/conf/push_replace.py new file mode 100644 index 00000000..cbf88ef6 --- /dev/null +++ b/conf/push_replace.py @@ -0,0 +1,26 @@ +import sys,json,requests + +if __name__ == "__main__": + + if len(sys.argv) != 3: + print('Usage: push_replace.py /solr//querqy/rewriter/') + sys.exit(1) + + rules_file = sys.argv[1] + rewriter_url = sys.argv[2] + + f = open(rules_file, "r") + + req = { + "class": "querqy.solr.rewriter.replace.ReplaceRewriterFactory", + "config": { + "rules": f.read(), + "ignoreCase": True, + "inputDelimiter": ";", + "querqyParser": "querqy.rewrite.commonrules.WhiteSpaceQuerqyParserFactory" + } + } + + resp = requests.post(rewriter_url + '?action=save', json=req) + if resp.status_code != 200: + sys.exit(2) diff --git a/conf/smui2solrcloud.sh b/conf/smui2solrcloud.sh new file mode 100755 index 00000000..447d4b23 --- /dev/null +++ b/conf/smui2solrcloud.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euo pipefail + +SRC_TMP_FILE=$1 +DST_CP_FILE_TO=$2 +SOLR_HOST=$3 +SOLR_COLLECTION_NAME=$4 +DECOMPOUND_DST_CP_FILE_TO=$5 +TARGET_SYSTEM=$6 +REPLACE_RULES_SRC_TMP_FILE=$7 +REPLACE_RULES_DST_CP_FILE_TO=$8 + +if grep -q -i -E '^https?://' <<<$SOLR_HOST; then + SOLR_BASE_URL=$SOLR_HOST +else + SOLR_BASE_URL=http://$SOLR_HOST/solr +fi + +echo "In smui2solrcloud.sh - script deploying rules to querqy api" +echo "^-- SRC_TMP_FILE = $SRC_TMP_FILE" +echo "^-- DST_CP_FILE_TO = $DST_CP_FILE_TO" +echo "^-- SOLR_BASE_URL = $SOLR_BASE_URL" +echo "^-- SOLR_COLLECTION_NAME: $SOLR_COLLECTION_NAME" +echo "^-- DECOMPOUND_DST_CP_FILE_TO = $DECOMPOUND_DST_CP_FILE_TO" +echo "^-- TARGET_SYSTEM = $TARGET_SYSTEM" +echo "^-- REPLACE_RULES_SRC_TMP_FILE = $REPLACE_RULES_SRC_TMP_FILE" +echo "^-- REPLACE_RULES_DST_CP_FILE_TO = $REPLACE_RULES_DST_CP_FILE_TO" + + +# DEPLOYMENT +##### + +echo "^-- Perform rules.txt deployment (decompound-rules.txt eventually)" + +echo "^-- ... rules.txt" +python3 ./conf/push_common_rules.py $SRC_TMP_FILE "$SOLR_BASE_URL/$SOLR_COLLECTION_NAME/querqy/rewriter/$DST_CP_FILE_TO" +if [ $? -ne 0 ]; then + exit 16 +fi + +echo "^-- ... replace-rules.txt" +if ! [[ $REPLACE_RULES_SRC_TMP_FILE == "NONE" && $REPLACE_RULES_DST_CP_FILE_TO == "NONE" ]] +then + python3 /smui/conf/push_replace.py $REPLACE_RULES_SRC_TMP_FILE "$SOLR_BASE_URL/$SOLR_COLLECTION_NAME/querqy/rewriter/$REPLACE_RULES_DST_CP_FILE_TO" +fi + +# all ok +echo "smui2solrcloud.sh - ok" +exit 0 diff --git a/test/resources/TestRulesTxtImportTenantTags.json b/test/resources/TestRulesTxtImportTenantTags.json new file mode 100644 index 00000000..1b0548af --- /dev/null +++ b/test/resources/TestRulesTxtImportTenantTags.json @@ -0,0 +1,14 @@ +[ + { + "property": "tenant", + "value": "AA" + }, + { + "property": "tenant", + "value": "BB" + }, + { + "property": "tenant", + "value": "CC" + } +] \ No newline at end of file diff --git a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala index c84eff2f..06f15d29 100644 --- a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala +++ b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala @@ -13,6 +13,7 @@ trait CommonRulesTxtDeploymentServiceConfigVariantsSpecBase extends ApplicationT // TODO maybe share those definitions / instructions with RulesTxtDeploymentServiceSpec as well? protected lazy val service = injector.instanceOf[RulesTxtDeploymentService] + protected lazy val rulesTxtImportService = injector.instanceOf[RulesTxtImportService] override protected lazy val activateSpelling = false @@ -388,5 +389,110 @@ class RulesTxtDeploymentGitTargetSpec extends FlatSpec with Matchers with Common } - } + +class RulesTxtOnlyDeploymentInputTagBasedSpec extends FlatSpec with Matchers with CommonRulesTxtDeploymentServiceConfigVariantsSpecBase { + + override protected lazy val additionalAppConfig = Seq( + "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", + "smui2solr.DST_CP_FILE_TO" -> "common_rules", + "toggle.rule-deployment.pre-live.present" -> true, + "smui2solr.deploy-prelive-fn-rules-txt" -> "common_rules", + "toggle.rule-tagging" -> true, + "toggle.predefined-tags-file" -> "./test/resources/TestRulesTxtImportTenantTags.json", + "smui2solr.deployment.per.inputtag.property" -> "tenant" + ) + + override protected def beforeAll(): Unit = { + super.beforeAll() + createTenantTaggedRules() + } + + protected def createTenantTaggedRules(): Unit = { + var rules: String = s"""tenantAA => + | DOWN(10): down_x + | @{ "tenant":["AA"]}@ + | + |tenantBB => + | DOWN(10): down_x + | @{ "tenant" : [ "BB" ] }@ + | + |tenantNoTag => + | DOWN(10): down_x + | + |tenantNone => + | DOWN(10): down_x + | @{ "tenant" : [ ] }@ + | + |tenantAB => + | DOWN(10): down_x + | @{ "tenant" : [ "AA", "BB" ] }@ + | + |""".stripMargin + val ( + retstatCountRulesTxtInputs, + retstatCountRulesTxtLinesSkipped, + retstatCountRulesTxtUnkownConvert, + retstatCountConsolidatedInputs, + retstatCountConsolidatedRules + ) = rulesTxtImportService.importFromFilePayload(rules, core1Id) + } + + protected def getExpectedResultsList: List[Map[String, Object]] = { + List( + // common - without tenant inputTag value + Map("inputTermsIncl" -> List("tenantNoTag", "tenantNone", "aerosmith"), + "inputTermsExcl" -> List("tenantAA", "tenantAB", "tenantBB"), + "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", + "destinationFileName" -> "common_rules"), + // tenant AA + Map("inputTermsIncl" -> List("tenantAA", "tenantAB"), + "inputTermsExcl" -> List("tenantNoTag", "tenantNone", "aerosmith", "tenantBB"), + "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_AA.tmp", + "destinationFileName" -> "common_rules_AA"), + // tenant BB + Map("inputTermsIncl" -> List("tenantAB", "tenantBB"), + "inputTermsExcl" -> List("tenantNoTag", "tenantNone", "aerosmith", "tenantAA"), + "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_BB.tmp", + "destinationFileName" -> "common_rules_BB"), + // tenant CC + Map("inputTermsIncl" -> List(), + "inputTermsExcl" -> List("tenantNoTag", "tenantNone", "aerosmith", "tenantAA", "tenantAB", "tenantBB"), + "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_CC.tmp", + "destinationFileName" -> "common_rules_CC") + ) + } + + def validateDeploymentDescriptor(deploymentDescriptor: service.RulesTxtsForSolrIndex, expectedResults: Map[String, Object]) = { + deploymentDescriptor.solrIndexId shouldBe core1Id + for (inputTerm <- expectedResults("inputTermsIncl").asInstanceOf[List[String]]) { + deploymentDescriptor.regularRules.content should include (inputTerm) + } + for (inputTerm <- expectedResults("inputTermsExcl").asInstanceOf[List[String]]) { + deploymentDescriptor.regularRules.content should not include (inputTerm) + } + deploymentDescriptor.regularRules.sourceFileName shouldBe expectedResults("sourceFileName") + deploymentDescriptor.regularRules.destinationFileName shouldBe expectedResults("destinationFileName") + deploymentDescriptor.replaceRules shouldBe None + deploymentDescriptor.decompoundRules shouldBe None + } + + "RulesTxtDeploymentService" should "provide only the (common) rules.txt for PRELIVE" in { + val deploymentDescriptorList = service.generateRulesTxtContentWithFilenamesList(core1Id, "PRELIVE", logDebug = false) + var i = 0 + for (deploymentDescriptor <- deploymentDescriptorList) { + validateDeploymentDescriptor(deploymentDescriptor, getExpectedResultsList(i)) + i += 1 + } + } + + "RulesTxtDeploymentService" should "provide only the (common) rules.txt for LIVE" in { + val deploymentDescriptorList = service.generateRulesTxtContentWithFilenamesList(core1Id, "LIVE", logDebug = false) + var i = 0 + for (deploymentDescriptor <- deploymentDescriptorList) { + validateDeploymentDescriptor(deploymentDescriptor, getExpectedResultsList(i)) + i += 1 + } + } + +} \ No newline at end of file