Support elastic search7 travis fix - support platforms.

.travis.yml

@@ -1,6 +1,7 @@
 ---
 language: java
-script: 
+dist: trusty
+script:
   - ./gradlew clean check --info
 jdk:
   - oraclejdk8

README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://travis-ci.org/allegro/embedded-elasticsearch.svg?branch=master)](https://travis-ci.org/allegro/embedded-elasticsearch)
 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/pl.allegro.tech/embedded-elasticsearch/badge.svg)](http://central.maven.org/maven2/pl/allegro/tech/embedded-elasticsearch)
 
-Small utility for creating integration tests that use Elasticsearch. Instead of using `Node` it downloads Elasticsearch in specified version and starts it in a separate process. It also allows you to install required plugins which is not possible when using `NodeBuilder`. Utility was tested with 1.x, 2.x, 5.x and 6.x versions of Elasticsearch.
+Small utility for creating integration tests that use Elasticsearch. Instead of using `Node` it downloads Elasticsearch in specified version and starts it in a separate process. It also allows you to install required plugins which is not possible when using `NodeBuilder`. Utility was tested with 1.x, 2.x, 5.x, 6.x and 7.x versions of Elasticsearch.
 
 ## Introduction
 
@@ -63,6 +63,7 @@ Available `IndexSettings.Builder` options
 | Method | Description |
 | ------------- | ------------- |
 | `withType(String type, String mapping)` | specify type and it's mappings |
+| `withMapping(String mapping)` | starting from Elasticseatch 7, there is no more types, so when using an ES version 7.0 and above this method should be used insted of withType method |
 | `withSettings(String settings)` | specify index settings |
 
 

core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java

@@ -164,10 +164,13 @@ private String indexMetadataJson(String indexName, String indexType, String id,
             joiner.add("\"_id\": \"" + id + "\"");
         }
 
-        if(routing != null) {
-            joiner.add("\"_routing\": \"" + routing + "\"");
+        if (routing != null) {
+            if (newESVersion()) {
+                joiner.add("\"routing\": \"" + routing + "\"");
+            } else {
+                joiner.add("\"_routing\": \"" + routing + "\"");
+            }
         }
-
         return "{ \"index\": {" + joiner.toString() + "} }";
     }
 
@@ -180,6 +183,20 @@ void refresh() {
         }
     }
 
+    private boolean newESVersion() {
+        HttpGet request = new HttpGet(url("/"));
+        return httpClient.execute(request, response -> {
+            JsonNode jsonNode;
+            try {
+                jsonNode = OBJECT_MAPPER.readTree(readBodySafely(response));
+            } catch (IOException e) {
+                return false;
+            }
+            String esV = jsonNode.get("version").get("number").asText();
+            return Integer.parseInt(esV.substring(0,1)) >= 7; //if version is 7 and above
+        });
+    }
+
     private void performBulkRequest(String requestUrl, String bulkRequestBody) {
         HttpPost request = new HttpPost(requestUrl);
         request.setHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"));

core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java

@@ -7,16 +7,21 @@
 import org.slf4j.LoggerFactory;
 import pl.allegro.tech.embeddedelasticsearch.InstallationDescription.Plugin;
 
+import java.io.BufferedInputStream;
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.lang.ProcessBuilder.Redirect;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.zip.GZIPInputStream;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.commons.io.FileUtils.forceMkdir;
@@ -64,16 +69,37 @@ private void installElastic(Path downloadedTo) throws IOException {
         logger.info("Installing Elasticsearch" + " into " + destination + "...");
         try {
             unzip(downloadedTo, destination);
+            makeExecutable("bin", "elasticsearch");
+            makeExecutable("bin", "plugin");
+            makeExecutable("bin", "elasticsearch-plugin");
+            makeExecutable("modules", "x-pack", "x-pack-ml", "platform", "linux-x86_64", "bin", "controller");
             logger.info("Done");
         } catch (IOException e) {
             logger.info("Failure : " + e);
             throw new EmbeddedElasticsearchStartupException(e);
         }
     }
 
+    private void makeExecutable(String... names) {
+        File executable = getFile(getInstallationDirectory(), names);
+        if (!executable.canExecute()) {
+            executable.setExecutable(true);
+        }
+    }
+
     private void unzip(Path downloadedTo, File destination) throws IOException {
-        Archiver archiver = ArchiverFactory.createArchiver("zip");
-        archiver.extract(downloadedTo.toFile(), destination);
+        Archiver archiver = ArchiverFactory.createArchiver(downloadedTo.endsWith("zip") ? "zip" : "tar");
+        try (InputStream is = toStream(downloadedTo)) {
+            archiver.extract(is, destination);
+        }
+    }
+
+    private InputStream toStream(Path downloadedTo) throws IOException {
+        InputStream result = new FileInputStream(downloadedTo.toFile());
+        if (downloadedTo.toFile().getName().endsWith(".gz")) {
+            result = new GZIPInputStream(result);
+        }
+        return new BufferedInputStream(result);
     }
 
     private void configureElastic() throws IOException {

core/src/main/java/pl/allegro/tech/embeddedelasticsearch/IndexSettings.java

@@ -17,31 +17,65 @@ public class IndexSettings {
 
     private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
+    private final Optional<JsonNode> mappings;
     private final List<TypeWithMapping> types;
     private final Optional<JsonNode> settings;
     private final Optional<JsonNode> aliases;
-
+    private boolean includeTypeName=false;
     public static Builder builder() {
         return new Builder();
     }
 
-    public IndexSettings(List<TypeWithMapping> types, Optional<String> settings) {
+    private IndexSettings(List<TypeWithMapping> types, Optional<String> settings, Optional<String> aliases) {
+        this.mappings = rawToJson(Optional.of("{}"));
         this.types = types;
         this.settings = rawToJson(settings);
         this.aliases = Optional.empty();
     }
 
-    private IndexSettings(List<TypeWithMapping> types, Optional<String> settings, Optional<String> aliases) {
-        this.types = types;
+    public IndexSettings(Optional<String>  mapping, Optional<String> settings) {
+        includeTypeName = true;
+        this.mappings = rawToJson(mapping);
+        this.settings = rawToJson(settings);
+        this.aliases = Optional.empty();
+        this.types = new ArrayList<>();
+    }
+
+    private IndexSettings(Optional<String> mapping, Optional<String> settings, Optional<String> aliases) {
+        this.mappings = rawToJson(mapping);
+        includeTypeName = true;
         this.settings = rawToJson(settings);
         this.aliases = rawToJson(aliases);
+        this.types = new ArrayList<>();
     }
 
     public static class Builder {
 
-        private final List<TypeWithMapping> types = new ArrayList<>();
+        private Optional<String> mapping  = Optional.empty();
         private Optional<String> settings = Optional.empty();
         private Optional<String> aliases = Optional.empty();
+        private final List<TypeWithMapping> types = new ArrayList<>();
+
+        /**
+         * Type with mappings to create with index
+         *
+         * @param mapping mappings for created type
+         */
+        public Builder withMapping(Object mapping) throws IOException {
+            String mappingString;
+            if (mapping == null) {
+                return this;
+            }
+            else if (mapping instanceof InputStream) {
+                InputStream mappingStream = (InputStream) mapping;
+                mappingString = IOUtils.toString(mappingStream, UTF_8);
+            }
+            else {
+                mappingString = (String) mapping;
+            }
+            this.mapping = Optional.of(mappingString);
+            return this;
+        }
 
         /**
          * Specify type inside created index
@@ -53,6 +87,7 @@ public Builder withType(String type, InputStream mapping) throws IOException {
             return withType(type, IOUtils.toString(mapping, UTF_8));
         }
 
+
         /**
          * Type with mappings to create with index
          *
@@ -106,16 +141,23 @@ public Builder withAliases(String aliases) {
          * @return IndexSettings with specified parameters
          */
         public IndexSettings build() {
-            return new IndexSettings(types, settings, aliases);
+            return new IndexSettings(mapping, settings, aliases);
         }
     }
 
     public ObjectNode toJson() {
         ObjectNode objectNode = new ObjectMapper().createObjectNode();
         objectNode.set("settings", settings.orElse(OBJECT_MAPPER.createObjectNode()));
         objectNode.set("aliases", aliases.orElse(OBJECT_MAPPER.createObjectNode()));
-        ObjectNode mappingsObject = prepareMappingsObject();
-        objectNode.set("mappings", mappingsObject);
+
+        if (includeTypeName){
+            objectNode.set("mappings",mappings.orElse(OBJECT_MAPPER.createObjectNode()));
+        }
+        else {
+            ObjectNode mappingsObject = prepareMappingsObject();
+            objectNode.set("mappings", mappingsObject);
+        }
+
         return objectNode;
     }
 

core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java

@@ -1,6 +1,7 @@
 package pl.allegro.tech.embeddedelasticsearch;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.SystemUtils;
 
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -29,7 +30,7 @@ public URL resolveDownloadUrl() {
     private URL urlFromVersion(String version) {
         ElsDownloadUrl elsDownloadUrl = ElsDownloadUrl.getByVersion(version);
         try {
-            return new URL(StringUtils.replace(elsDownloadUrl.downloadUrl, "{VERSION}", version));
+            return new URL(StringUtils.replace(elsDownloadUrl.getDownloadUrl(), "{VERSION}", version));
         } catch (MalformedURLException e) {
             throw new RuntimeException(e);
         }
@@ -39,7 +40,26 @@ private enum ElsDownloadUrl {
         ELS_1x("1.", "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-{VERSION}.zip"),
         ELS_2x("2.", "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/zip/elasticsearch/{VERSION}/elasticsearch-{VERSION}.zip"),
         ELS_5x("5.", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{VERSION}.zip"),
-        ELS_6x("6.", ELS_5x.downloadUrl);
+        ELS_6x("6.", ELS_5x.downloadUrl),
+        ELS_7x("7.", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{VERSION}-{PLATFORM}-x86_64.{EXTENSION}") {
+            @Override
+            String getDownloadUrl() {
+                if (SystemUtils.IS_OS_LINUX) {
+                    String result = StringUtils.replace(downloadUrl, PLATFORM, "linux");
+                    return StringUtils.replace(result, EXTENSION, "tar.gz");
+                } else if (SystemUtils.IS_OS_MAC) {
+                    String result = StringUtils.replace(downloadUrl, PLATFORM, "darwin");
+                    return StringUtils.replace(result, EXTENSION, "tar.gz");
+                } else if (SystemUtils.IS_OS_WINDOWS) {
+                    String result = StringUtils.replace(downloadUrl, PLATFORM, "windows");
+                    return StringUtils.replace(result, EXTENSION, "zip");
+                }
+                throw new IllegalArgumentException(("Unsupported OS version " + SystemUtils.OS_NAME));
+            }
+        };
+
+        private static final String PLATFORM = "{PLATFORM}";
+        private static final String EXTENSION = "{EXTENSION}";
 
         String versionPrefix;
         String downloadUrl;
@@ -53,6 +73,10 @@ boolean versionMatch(String elasticVersion) {
             return elasticVersion.startsWith(versionPrefix);
         }
 
+        String getDownloadUrl() {
+            return downloadUrl;
+        }
+
         static ElsDownloadUrl getByVersion(String elasticVersion) {
             return Arrays.stream(ElsDownloadUrl.values())
                     .filter(u -> u.versionMatch(elasticVersion))

core/src/main/java/pl/allegro/tech/embeddedelasticsearch/TypeWithMapping.java

@@ -30,4 +30,4 @@ public String getType() {
     public JsonNode getMapping() {
         return mapping;
     }
-}
+}

core/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/InstallationSourceSpec.groovy

@@ -4,6 +4,10 @@ import spock.lang.Specification
 import spock.lang.Unroll
 
 class InstallationSourceSpec extends Specification {
+    static {
+        //Need to do this here since we read the value statically
+        System.setProperty("os.name", "Linux")
+    }
 
     def "should construct valid url for version"() {
         given:
@@ -14,6 +18,16 @@ class InstallationSourceSpec extends Specification {
             resolvedUrl != null
     }
 
+    def "should construct valid url for platform specific version"() {
+        given:
+            final installationSource = new InstallFromVersion("7.6.2")
+        when:
+            final resolvedUrl = installationSource.resolveDownloadUrl()
+        then:
+            resolvedUrl != null
+            resolvedUrl.toExternalForm() == "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.6.2-linux-x86_64.tar.gz"
+    }
+
     def "should extract properly version from normal url"() {
         given:
             final expectedVersion = "2.3.4"

es73-test/build.gradle

@@ -0,0 +1,10 @@
+dependencies {
+    testCompile project(':test-base')
+
+    testCompile group: 'org.elasticsearch', name: 'elasticsearch', version: '7.3.2'
+    testCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.3.2'
+    testCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-client', version: '7.3.2'
+    testCompile group: 'org.locationtech.spatial4j', name: 'spatial4j', version: '0.6'
+    testCompile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.6.2'
+    testCompile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.6.2'
+}