Adjustment to UI Deployment (#516)

.github/workflows/integration.yml

@@ -30,10 +30,10 @@ jobs:
                 run: ./gradlew printJavaVersion
             -   name: Assemble
                 timeout-minutes: 10
-                run: ./gradlew assemble
+                run: ./gradlew assemble -PwithoutpullingUi=true
             -   name: Build Plugins
                 timeout-minutes: 5
-                run: ./gradlew assemblePlugins
+                run: ./gradlew assemblePlugins -PwithoutpullingUi=true
             -   name: Execute integration tests for ${{ matrix.adapter }}
                 timeout-minutes: 30
-                run: ./gradlew integrationTests -Dstore.default=${{ matrix.adapter }}
+                run: ./gradlew integrationTests -Dstore.default=${{ matrix.adapter }} -PwithoutpullingUi=true

.github/workflows/matrix.yml

@@ -31,10 +31,10 @@ jobs:
                 run: ./gradlew printJavaVersion
             -   name: Assemble
                 timeout-minutes: 60
-                run: ./gradlew assemble
+                run: ./gradlew assemble -PwithoutpullingUi=true
             -   name: Build Plugins
                 timeout-minutes: 60
-                run: ./gradlew assemblePlugins
+                run: ./gradlew assemblePlugins -PwithoutpullingUi=true
             -   name: Execute tests
                 timeout-minutes: 30
-                run: ./gradlew check
+                run: ./gradlew check -PwithoutpullingUi=true

.github/workflows/plugins.yml

@@ -30,10 +30,10 @@ jobs:
                 run: ./gradlew printJavaVersion
             -   name: Assemble
                 timeout-minutes: 60
-                run: ./gradlew assemble
+                run: ./gradlew assemble -PwithoutpullingUi=true
             -   name: Build Plugins
                 timeout-minutes: 60
-                run: ./gradlew assemblePlugins
+                run: ./gradlew assemblePlugins -PwithoutpullingUi=true
             -   name: Execute tests
                 timeout-minutes: 30
-                run: ./gradlew -p plugins test
+                run: ./gradlew -p plugins test -PwithoutpullingUi=true

.gitignore

@@ -2,6 +2,8 @@
 # Generated files
 gen/
 
+# webapp
+webui/src/main/resources/webapp/*
 
 
 # Created by .ignore support plugin (hsz.mobi)

config/build.gradle

@@ -4,6 +4,9 @@ group "org.polypheny"
 dependencies {
     implementation group: "io.javalin", name: "javalin", version: javalin_version // Apache 2.0
 
+    implementation group: 'net.lingala.zip4j', name: 'zip4j', version: zip4j_version // Apache 2.0
+
+
     implementation group: "com.google.code.gson", name: "gson", version: gson_version  // Apache 2.0
 
     implementation group: "com.fasterxml.jackson.core", name: "jackson-databind", version: jackson_databind_version  // Apache 2.0

config/src/main/java/org/polypheny/db/util/PolyphenyHomeDirManager.java

@@ -28,6 +28,8 @@
 import java.util.Optional;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import net.lingala.zip4j.ZipFile;
+import net.lingala.zip4j.exception.ZipException;
 import org.jetbrains.annotations.NotNull;
 
 
@@ -269,8 +271,17 @@ public File registerNewFolder( String folder ) {
     }
 
 
-    public File registerNewGlobalFolder( String testBackup ) {
-        return registerNewFolder( this.root, testBackup );
+    public File registerNewGlobalFolder( String folder ) {
+        return registerNewFolder( this.root, folder );
+    }
+
+
+    public void unzipInto( File zip, File target ) {
+        try {
+            new ZipFile( zip ).extractAll( target.getAbsolutePath() );
+        } catch ( ZipException e ) {
+            throw new RuntimeException( "Could not unzip: {}", e );
+        }
     }
 
 
@@ -279,8 +290,13 @@ public boolean isAccessible( File file ) {
     }
 
 
-    public File getRootPath() {
+    public File getHomePath() {
         return home;
     }
 
+
+    public File getRootPath() {
+        return root;
+    }
+
 }

dbms/src/main/java/org/polypheny/db/PolyphenyDb.java

@@ -244,7 +244,7 @@ public void runPolyphenyDb() {
                 throw new GenericRuntimeException( "Unable to backup the Polypheny folder since there is already a backup folder." );
             }
             File backupFolder = dirManager.registerNewFolder( "_test_backup" );
-            for ( File item : dirManager.getRootPath().listFiles() ) {
+            for ( File item : dirManager.getHomePath().listFiles() ) {
                 if ( item.getName().equals( "_test_backup" ) ) {
                     continue;
                 }
@@ -458,7 +458,7 @@ private static void restoreHomeFolderIfNecessary( PolyphenyHomeDirManager dirMan
         if ( dirManager.getHomeFile( "_test_backup" ).isPresent() && dirManager.getHomeFile( "_test_backup" ).get().isDirectory() ) {
             File backupFolder = dirManager.getHomeFile( "_test_backup" ).get();
             // Cleanup Polypheny folder
-            for ( File item : dirManager.getRootPath().listFiles() ) {
+            for ( File item : dirManager.getHomePath().listFiles() ) {
                 if ( item.getName().equals( "_test_backup" ) ) {
                     continue;
                 }
@@ -471,7 +471,7 @@ private static void restoreHomeFolderIfNecessary( PolyphenyHomeDirManager dirMan
             // Restore contents from backup
             for ( File item : backupFolder.listFiles() ) {
                 if ( dirManager.getHomeFile( "_test_backup/" + item.getName() ).isPresent() ) {
-                    if ( !item.renameTo( new File( dirManager.getRootPath(), item.getName() ) ) ) {
+                    if ( !item.renameTo( new File( dirManager.getHomePath(), item.getName() ) ) ) {
                         throw new GenericRuntimeException( "Unable to restore the Polypheny folder." );
                     }
                 }

gradle.properties

@@ -36,6 +36,10 @@ commons_io_version = 2.15.1
 commons_lang3_version = 3.16.0
 cottontaildb_version = 0.13.3
 cottontaildb_driver_version = 0.13.0
+cottontaildb_eclipse_version = 7.1.2
+cottontaildb_guave_version = 15.0
+cottontaildb_kotlin_version = 1.2.41
+cottontaildb_mapdb_version = 3.0.8
 cottontaildb_grpc_version = 1.36.0
 esri_geometry_api_version = 2.2.0
 google_api_client_version = 1.33.0
@@ -92,3 +96,4 @@ slf4j_api_version = 2.0.16
 typesafe_config_version = 1.2.1
 unirest_version = 3.14.5
 web3j_version = 5.0.0
+zip4j_version = 2.11.5

plugins/cottontail-adapter/build.gradle

@@ -1,20 +1,34 @@
 plugins {
     id "org.jetbrains.kotlin.jvm" version "1.7.22"
+    id("de.undercouch.download") version "5.6.0"
+}
+
+configurations {
+    preDependencyConfig
 }
 
 group "org.polypheny"
 
 apply plugin: "kotlin"
 
+
 dependencies {
     compileOnly project(":core")
     compileOnly project(":plugins:sql-language")
 
     implementation group: "io.grpc", name: "grpc-all", version: cottontaildb_grpc_version
-    // Cottontail DB
-    implementation(group: "org.vitrivr", name: "cottontaildb", version: cottontaildb_version)
-    implementation group: "org.vitrivr", name: "cottontaildb-proto", version: cottontaildb_driver_version
 
+    // Cottontail DB Dependencies (necessary due to dbisnexus shutdown)
+    implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: cottontaildb_kotlin_version
+    implementation group: 'org.eclipse.collections', name: 'eclipse-collections-api', version: cottontaildb_eclipse_version
+    implementation group: 'org.eclipse.collections', name: 'eclipse-collections', version: cottontaildb_eclipse_version
+    implementation group: 'org.eclipse.collections', name: 'eclipse-collections-forkjoin', version: cottontaildb_eclipse_version
+    implementation group: 'com.google.guava', name: 'guava', version: cottontaildb_guave_version
+    implementation group: 'org.mapdb', name: 'mapdb', version: cottontaildb_mapdb_version
+
+    // Cottontail DB
+    implementation group: 'org.vitrivr', name: 'cottontaildb', version: cottontaildb_version
+    implementation group: 'org.vitrivr', name: 'cottontaildb-proto', version: cottontaildb_driver_version
 
     // --- Test Compile ---
     testImplementation project(path: ":dbms", configuration: "test")
@@ -23,6 +37,7 @@ dependencies {
     testImplementation project(path: ":core")
 }
 
+
 compileKotlin {
     dependsOn(":plugins:sql-language:processResources")
 }
@@ -36,14 +51,6 @@ delombok {
     dependsOn(":plugins:sql-language:processResources")
 }
 
-/*inspectClassesForKotlinIC {
-    dependsOn(":plugins:cottontail-adapter:compileTestKotlin")
-}
-
-plugin {
-    dependsOn(":plugins:cottontail-adapter:compileTestKotlin")
-}*/
-
 test.dependsOn(":dbms:shadowJar")
 
 
@@ -79,16 +86,12 @@ jar {
         attributes "Copyright": "The Polypheny Project (polypheny.org)"
         attributes "Version": "$project.version"
     }
-    //dependsOn(":plugins:cottontail-adapter:compileTestKotlin")
 }
 java {
     withJavadocJar()
     withSourcesJar()
 }
 
-javadoc {
-    //dependsOn(":plugins:cottontail-adapter:compileTestKotlin")
-}
 
 licensee {
     allow('Apache-2.0')
@@ -107,4 +110,6 @@ licensee {
     allowDependency('org.vitrivr', 'cottontaildb-proto', '0.13.0') { because 'MIT license' }
 
     allowDependency('javax.annotation', 'javax.annotation-api', '1.3.2') { because 'CDDL license' }
+
+    allowDependency('net.jcip', 'jcip-annotations', '1.0') { because 'transitive, Apache 2. according to github' }
 }

plugins/http-interface/src/main/java/org/polypheny/db/http/HttpInterfacePlugin.java

@@ -147,7 +147,6 @@ public void run() {
                     }
                 } ) );
                 config.enableCorsForAllOrigins();
-                config.addStaticFiles( staticFileConfig -> staticFileConfig.directory = "webapp/" );
             } ).start( port );
             server.exception( Exception.class, ( e, ctx ) -> {
                 log.warn( "Caught exception in the HTTP interface", e );
@@ -287,4 +286,4 @@ public void remove() {
 
     }
 
-}
+}

webui/build.gradle

@@ -1,3 +1,7 @@
+plugins {
+    id("de.undercouch.download") version "5.6.0"
+}
+
 group "org.polypheny"
 
 
@@ -16,7 +20,7 @@ dependencies {
     implementation project(":core")
     implementation project(":monitoring")
 
-    uiFiles group: "org.polypheny", name: "polypheny-ui", version: polypheny_ui_version
+    //uiFiles group: "org.polypheny", name: "polypheny-ui", version: polypheny_ui_version
 
     implementation group: "commons-io", name: "commons-io", version: commons_io_version  // Apache 2.0
     implementation group: "io.javalin", name: "javalin", version: javalin_version // Apache 2.0
@@ -39,7 +43,7 @@ sourceSets {
             srcDirs = ["src/main/java"]
         }
         resources {
-            srcDirs = ["src/main/resources", "$buildDir/webapp"]
+            srcDirs = ["src/main/resources", "build/resources/main"]
         }
         output.resourcesDir = file(project.buildDir.absolutePath + "/classes")
     }
@@ -59,29 +63,56 @@ compileJava {
     dependsOn(":monitoring:processResources")
 }
 
+
 processResources {
-    dependsOn(":webui:unzipUiFiles")
+    setDuplicatesStrategy(DuplicatesStrategy.WARN)
+    if (!project.hasProperty('withoutpullingUi')) {
+        dependsOn(":webui:downloadUiZip")
+    }
+}
+
+
+gradle.taskGraph.whenReady { taskGraph ->
+    if (!project.hasProperty('withoutpullingUi')
+        && !layout.buildDirectory.get().file("polypheny-ui.zip").asFile.exists()) {
+        println("Pulling UI files")
+        handleUi.dependsOn downloadUiZip
+    }
 }
 
+task downloadUiZip(type: Download) {
+    src 'https://artifacts.polypheny.com/ui/polypheny-ui.zip'
+    dest "${layout.buildDirectory.asFile.get().absolutePath}/classes/polypheny-ui.zip"
+    overwrite true
+    onlyIfModified true
+}
+
+private Object getPolyphenyHome() {
+    def homeDir = System.getProperty('user.home');
+    def polyphenyDir = System.getenv('POLYPHENY_HOME') ?: "${homeDir}/.polypheny"
+    polyphenyDir
+}
 
-// unzip ui files
-task unzipUiFiles(type: Copy) {
-    from zipTree(configurations.uiFiles.singleFile)
-    into "$buildDir/webapp"
-    doLast {
-        file("$buildDir/webapp/index").mkdirs()
-        ant.move file: "$buildDir/webapp/webapp/index.html",
-            toDir: "$buildDir/webapp/index/"
+task deleteUiOverwrite() {
+    def home = getPolyphenyHome()
+    def ui = file("${home}/ui")
+    if (ui.exists() && ui.canWrite()) {
+        delete ui
     }
 }
-tasks.register('deleteUiFiles') {
-    project.delete(
-        fileTree("$buildDir/webapp")
-    )
+
+clean.dependsOn(deleteUiOverwrite)
+
+build.dependsOn(":webui:handleUi")
+
+
+task handleUi() {
+    // empty on purpose due to gradle weirdness
 }
-unzipUiFiles.dependsOn(deleteUiFiles)
-classes.dependsOn(unzipUiFiles)
 
+if (!project.hasProperty('withoutpullingUi')) {
+    handleUi.finalizedBy downloadUiZip
+}
 
 /**
  * JARs
@@ -92,7 +123,6 @@ jar {
         attributes "Copyright": "The Polypheny Project (polypheny.org)"
         attributes "Version": "$project.version"
     }
-    from("$buildDir/webapp") // include webapp files
     duplicatesStrategy = 'include'
 }
 java {