settings in the mapping

jprante · Nov 20, 2014 · 1368fbe · 1368fbe
1 parent 91e1b4b
commit 1368fbe
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -77,28 +77,29 @@ zh-cn
 zh-tw
 
 
-
 ## Versions
 
 ![Travis](https://travis-ci.org/jprante/elasticsearch-langdetect.png)
 
 | Elasticsearch  | Plugin         | Release date |
 | -------------- | -------------- | ------------ |
+| 1.4.0          | 1.4.0.1        | Nov 20, 2014 |
 | 1.4.0          | 1.4.0.0        | Nov 14, 2014 |
 | 1.3.1          | 1.3.0.0        | Jul 30, 2014 |
 | 1.2.1          | 1.2.1.1        | Jun 18, 2014 |
 
 
 ## Installation
 
-    ./bin/plugin -install langdetect -url http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-langdetect/1.4.0.0/elasticsearch-langdetect-1.4.0.0-plugin.zip
+    ./bin/plugin -install langdetect -url http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-langdetect/1.4.0.1/elasticsearch-langdetect-1.4.0.1-plugin.zip
 
 Do not forget to restart the node after installing.
 
 ## Checksum
 
 | File                                          | SHA1                                     |
 | --------------------------------------------- | -----------------------------------------|
+| elasticsearch-langdetect-1.4.0.1-plugin.zip   | a7ab2401ae68789bf1e5841427ce40440c903da4 |
 | elasticsearch-langdetect-1.4.0.0-plugin.zip   | f95361fa1a81b2681e2e9002b03ca6aad57f3012 |
 | elasticsearch-langdetect-1.3.0.0-plugin.zip   | e2dd56c72f19cec861141becd8beb18d7bb26ee6 |
 | elasticsearch-langdetect-1.2.1.1-plugin.zip   | cc3a0d5ccecf1210b96771dcb5c9935176e1cc35 |

diff --git a/src/main/java/org/xbib/elasticsearch/index/mapper/langdetect/LangdetectMapper.java b/src/main/java/org/xbib/elasticsearch/index/mapper/langdetect/LangdetectMapper.java
@@ -55,6 +55,11 @@ public Builder lang(StringFieldMapper.Builder lang) {
             return this;
         }
 
+        public Builder profile(String profile) {
+            settingsBuilder.put("profile", profile);
+            return this;
+        }
+
         public Builder ntrials(int trials) {
             settingsBuilder.put("number_of_trials", trials);
             return this;
@@ -203,6 +208,10 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                         builder.languages((List<String>) fieldNode);
                         break;
                     }
+                    case "profile": {
+                        builder.profile((String) fieldNode);
+                        break;
+                    }
                 }
             }
             return builder;

diff --git a/src/main/java/org/xbib/elasticsearch/module/langdetect/LangdetectService.java b/src/main/java/org/xbib/elasticsearch/module/langdetect/LangdetectService.java
@@ -26,7 +26,7 @@ public class LangdetectService extends AbstractLifecycleComponent<LangdetectServ
     private final static Pattern word = Pattern.compile("[\\P{IsWord}]", Pattern.UNICODE_CHARACTER_CLASS);
 
     private final static String[] DEFAULT_LANGUAGES = new String[] {
-            "af",
+           // "af",
             "ar",
             "bg",
             "bn",
@@ -48,27 +48,27 @@ public class LangdetectService extends AbstractLifecycleComponent<LangdetectServ
             "id",
             "it",
             "ja",
-            "kn",
+           // "kn",
             "ko",
             "lt",
             "lv",
             "mk",
             "ml",
-            "mr",
-            "ne",
+           // "mr",
+           // "ne",
             "nl",
             "no",
             "pa",
             "pl",
             "pt",
             "ro",
             "ru",
-            "sk",
-            "sl",
-            "so",
+           // "sk",
+            //"sl",
+           // "so",
             "sq",
             "sv",
-            "sw",
+           // "sw",
             "ta",
             "te",
             "th",

diff --git a/src/test/java/org/xbib/elasticsearch/index/mapper/langdetect/LangdetectMappingTest.java b/src/test/java/org/xbib/elasticsearch/index/mapper/langdetect/LangdetectMappingTest.java
@@ -80,6 +80,27 @@ public void testBinary() throws Exception {
         assertEquals(doc.getFields("someField.lang")[0].stringValue(), "en");
     }
 
+    @Test
+    public void testShortTextProfile() throws Exception {
+        String mapping = copyToStringFromClasspath("short-text-mapping.json");
+        DocumentMapper docMapper = createMapperParser().parse(mapping);
+        String sampleText = copyToStringFromClasspath("english.txt");
+        BytesReference json = jsonBuilder().startObject().field("_id", 1).field("someField", sampleText).endObject().bytes();
+        ParseContext.Document doc = docMapper.parse(json).rootDoc();
+        assertEquals(doc.get(docMapper.mappers().smartName("someField").mapper().names().indexName()), sampleText);
+        assertEquals(doc.getFields("someField.lang").length, 1);
+        assertEquals(doc.getFields("someField.lang")[0].stringValue(), "en");
+
+        // re-parse it
+        String builtMapping = docMapper.mappingSource().string();
+        docMapper = createMapperParser().parse(builtMapping);
+        json = jsonBuilder().startObject().field("_id", 1).field("someField", sampleText).endObject().bytes();
+        doc = docMapper.parse(json).rootDoc();
+        assertEquals(doc.get(docMapper.mappers().smartName("someField").mapper().names().indexName()), sampleText);
+        assertEquals(doc.getFields("someField.lang").length, 1);
+        assertEquals(doc.getFields("someField.lang")[0].stringValue(), "en");
+    }
+
     private DocumentMapperParser createMapperParser() throws IOException {
         return createMapperParser(ImmutableSettings.EMPTY);
     }

diff --git a/src/test/resources/org/xbib/elasticsearch/index/mapper/langdetect/short-text-mapping.json b/src/test/resources/org/xbib/elasticsearch/index/mapper/langdetect/short-text-mapping.json
@@ -0,0 +1,10 @@
+{
+    "someType" : {
+        "properties" : {
+            "someField" : {
+                 "type" : "langdetect",
+                 "profile" : "/langdetect/short-text/"
+            }
+        }
+    }
+}