From d2d9d310462d3c0e2bcbb4b4485d16e8cb979efd Mon Sep 17 00:00:00 2001 From: Drew Date: Thu, 14 May 2020 18:05:06 -0400 Subject: [PATCH 1/2] Update for new dataformat --- .../com/openlattice/launchpad/Launchpad.java | 6 +- .../com/openlattice/launchpad/LaunchpadCli.kt | 62 +++++++++++++++---- .../configuration/IntegrationConfiguration.kt | 18 ++++++ 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/openlattice/launchpad/Launchpad.java b/src/main/java/com/openlattice/launchpad/Launchpad.java index 69dfad6..d39c2f8 100644 --- a/src/main/java/com/openlattice/launchpad/Launchpad.java +++ b/src/main/java/com/openlattice/launchpad/Launchpad.java @@ -26,11 +26,9 @@ */ @SuppressFBWarnings(value = "SECPTI", justification = "User input for file is considered trusted.") public class Launchpad { - private static final ObjectMapper mapper = JacksonSerializationConfiguration.yamlMapper; private static final Logger logger = LoggerFactory.getLogger( Launchpad.class ); - public static void main( String[] args ) throws ParseException, IOException { CommandLine cl = LaunchpadCli.parseCommandLine( args ); @@ -45,6 +43,8 @@ public static void main( String[] args ) throws ParseException, IOException { Preconditions.checkState( StringUtils.isNotBlank( integrationFilePath ) ); File integrationFile = new File( integrationFilePath ); + ObjectMapper mapper = JacksonSerializationConfiguration.yamlMapper; + IntegrationConfiguration config = mapper.readValue( integrationFile, IntegrationConfiguration.class ); @@ -59,6 +59,8 @@ public static void main( String[] args ) throws ParseException, IOException { System.exit( -1 ); } + config = LaunchpadCli.readUsernamesPasswordsAndUpdateConfiguration( config, cl ); + IntegrationRunner.runIntegrations( config ); } diff --git a/src/main/kotlin/com/openlattice/launchpad/LaunchpadCli.kt b/src/main/kotlin/com/openlattice/launchpad/LaunchpadCli.kt index fd0fe20..46b549f 100644 --- a/src/main/kotlin/com/openlattice/launchpad/LaunchpadCli.kt +++ b/src/main/kotlin/com/openlattice/launchpad/LaunchpadCli.kt @@ -1,9 +1,11 @@ package com.openlattice.launchpad -import org.apache.commons.cli.CommandLine -import org.apache.commons.cli.DefaultParser -import org.apache.commons.cli.HelpFormatter -import org.apache.commons.cli.Options +import com.fasterxml.jackson.module.kotlin.readValue +import com.openlattice.launchpad.configuration.DataLake +import com.openlattice.launchpad.configuration.IntegrationConfiguration +import com.openlattice.launchpad.serialization.JacksonSerializationConfiguration +import org.apache.commons.cli.* +import java.util.* /** * @author Drew Bailey <drew@openlattice.com> @@ -13,26 +15,64 @@ class LaunchpadCli { companion object { const val HELP = "help" const val FILE = "file" + const val USERNAMES = "usernames" + const val PASSWORDS = "passwords" private val options = Options() - private val clp = DefaultParser() - private val hf = HelpFormatter() init { options.addOption(HELP, "Print help message." ); - options.addOption(FILE, - true, - "File in which the final model will be saved. Also used as prefix for intermediate saves of the model." ); + options.addOption( Option.builder(FILE) + .hasArg() + .required() + .desc("File in which the final model will be saved. Also used as prefix for intermediate saves of the model." ) + .build()); + options.addOption( Option.builder(USERNAMES) + .hasArg() + .desc("Usernames for connection to the client database. Formatted as a json map {:,...}") + .build()); + options.addOption( Option.builder(PASSWORDS) + .hasArg() + .desc("Passwords for connection to the client database. Formatted as a json map {:,...}") + .build()); } @JvmStatic fun parseCommandLine(args: Array ) : CommandLine { - return clp.parse(options, args ) + return DefaultParser().parse(options, args ) } @JvmStatic fun printHelp(): Unit { - hf.printHelp( "launchpad", options); + HelpFormatter().printHelp( "launchpad", options); + } + + @JvmStatic + fun readUsernamesPasswordsAndUpdateConfiguration(config: IntegrationConfiguration, cl: CommandLine ): IntegrationConfiguration { + val mapper = JacksonSerializationConfiguration.jsonMapper + var lakeToUsername: Map = mapOf() + if (cl.hasOption(USERNAMES) && cl.getOptionValue(USERNAMES).isNotBlank() ) { + lakeToUsername = mapper.readValue(cl.getOptionValue(USERNAMES)) + } + + var lakeToPassword: Map = mapOf() + if (cl.hasOption(PASSWORDS) && cl.getOptionValue(PASSWORDS).isNotBlank() ) { + lakeToPassword = mapper.readValue(cl.getOptionValue(PASSWORDS)) + } + val newLakes = config.datalakes.get().map { + val username = lakeToUsername.get( it.name ) + val password = lakeToPassword.get( it.name ) + DataLake.withUsernameAndPassword( it, username, password ) + } + return IntegrationConfiguration( + config.name, + config.description, + config.awsConfig, + config.datasources, + config.destinations, + Optional.of(newLakes), + config.integrations + ) } } } \ No newline at end of file diff --git a/src/main/kotlin/com/openlattice/launchpad/configuration/IntegrationConfiguration.kt b/src/main/kotlin/com/openlattice/launchpad/configuration/IntegrationConfiguration.kt index 7fdaa5c..736696c 100644 --- a/src/main/kotlin/com/openlattice/launchpad/configuration/IntegrationConfiguration.kt +++ b/src/main/kotlin/com/openlattice/launchpad/configuration/IntegrationConfiguration.kt @@ -125,6 +125,24 @@ data class DataLake( ) { companion object { private val logger = LoggerFactory.getLogger(DataLake::class.java) + + @JvmStatic + fun withUsernameAndPassword( dataLake: DataLake, username: String?, password: String? ): DataLake { + return DataLake( + dataLake.name, + dataLake.url, + dataLake.driver, + dataLake.dataFormat, + username?:dataLake.username, + password?:dataLake.password, + dataLake.header, + dataLake.fetchSize, + dataLake.batchSize, + dataLake.writeMode, + dataLake.latticeLogger, + dataLake.properties + ) + } } init { From 5ef5a3016e5e14e7a2219c0c611ee0fd03b95c6a Mon Sep 17 00:00:00 2001 From: Drew Date: Fri, 15 May 2020 17:56:16 -0400 Subject: [PATCH 2/2] remove some unused --- src/main/java/com/openlattice/launchpad/Launchpad.java | 6 ------ .../launchpad/configuration/LaunchpadDatasource.java | 4 ++-- .../launchpad/configuration/LaunchpadDestination.java | 4 ++-- .../openlattice/launchpad/postgres/StatementHolder.java | 4 ++-- src/main/kotlin/com/openlattice/launchpad/LaunchpadCli.kt | 8 +++++++- .../com/openlattice/launchpad/configuration/Constants.kt | 1 + 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/openlattice/launchpad/Launchpad.java b/src/main/java/com/openlattice/launchpad/Launchpad.java index d39c2f8..78ac1c0 100644 --- a/src/main/java/com/openlattice/launchpad/Launchpad.java +++ b/src/main/java/com/openlattice/launchpad/Launchpad.java @@ -13,8 +13,6 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.ParseException; import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -27,8 +25,6 @@ @SuppressFBWarnings(value = "SECPTI", justification = "User input for file is considered trusted.") public class Launchpad { - private static final Logger logger = LoggerFactory.getLogger( Launchpad.class ); - public static void main( String[] args ) throws ParseException, IOException { CommandLine cl = LaunchpadCli.parseCommandLine( args ); @@ -37,8 +33,6 @@ public static void main( String[] args ) throws ParseException, IOException { System.exit(0); } - Preconditions.checkArgument( cl.hasOption( LaunchpadCli.FILE ), "Integration file must be specified!" ); - final String integrationFilePath = cl.getOptionValue( LaunchpadCli.FILE ); Preconditions.checkState( StringUtils.isNotBlank( integrationFilePath ) ); File integrationFile = new File( integrationFilePath ); diff --git a/src/main/java/com/openlattice/launchpad/configuration/LaunchpadDatasource.java b/src/main/java/com/openlattice/launchpad/configuration/LaunchpadDatasource.java index aa3b8b9..f0d849d 100644 --- a/src/main/java/com/openlattice/launchpad/configuration/LaunchpadDatasource.java +++ b/src/main/java/com/openlattice/launchpad/configuration/LaunchpadDatasource.java @@ -79,8 +79,8 @@ public LaunchpadDatasource( } public DataLake asDataLake() { - String lakeDataFormat = ""; - String lakeDriver = ""; + String lakeDataFormat; + String lakeDriver; switch ( driver ){ case S3_DRIVER: lakeDriver = S3_DRIVER; diff --git a/src/main/java/com/openlattice/launchpad/configuration/LaunchpadDestination.java b/src/main/java/com/openlattice/launchpad/configuration/LaunchpadDestination.java index 46a244b..dbd33b3 100644 --- a/src/main/java/com/openlattice/launchpad/configuration/LaunchpadDestination.java +++ b/src/main/java/com/openlattice/launchpad/configuration/LaunchpadDestination.java @@ -93,8 +93,8 @@ public LaunchpadDestination( } public DataLake asDataLake() { - String lakeDataFormat = ""; - String lakeDriver = ""; + String lakeDataFormat; + String lakeDriver; switch ( writeDriver ){ case S3_DRIVER: lakeDriver = S3_DRIVER; diff --git a/src/main/java/com/openlattice/launchpad/postgres/StatementHolder.java b/src/main/java/com/openlattice/launchpad/postgres/StatementHolder.java index 24ef167..bdf8c0c 100644 --- a/src/main/java/com/openlattice/launchpad/postgres/StatementHolder.java +++ b/src/main/java/com/openlattice/launchpad/postgres/StatementHolder.java @@ -100,10 +100,10 @@ public synchronized void close() { otherStatements.forEach( this::safeTryClose ); final long elapsed = sw.elapsed( TimeUnit.MILLISECONDS ); - if ( elapsed > this.longRunningQueryLimit ) { + if ( elapsed > longRunningQueryLimit ) { logger.warn( "The following statement was involved in a long lived connection that took {} ms: {}", elapsed, - statement.toString() ); + statement ); } sw.stop(); diff --git a/src/main/kotlin/com/openlattice/launchpad/LaunchpadCli.kt b/src/main/kotlin/com/openlattice/launchpad/LaunchpadCli.kt index 46b549f..0a665cc 100644 --- a/src/main/kotlin/com/openlattice/launchpad/LaunchpadCli.kt +++ b/src/main/kotlin/com/openlattice/launchpad/LaunchpadCli.kt @@ -39,7 +39,13 @@ class LaunchpadCli { @JvmStatic fun parseCommandLine(args: Array ) : CommandLine { - return DefaultParser().parse(options, args ) + try { + return DefaultParser().parse(options, args ) + } catch ( ex: MissingOptionException ) { + println("Integration file must be specified!") + System.exit(-1) + } + return CommandLine.Builder().build() } @JvmStatic diff --git a/src/main/kotlin/com/openlattice/launchpad/configuration/Constants.kt b/src/main/kotlin/com/openlattice/launchpad/configuration/Constants.kt index 925a4ce..02f111e 100644 --- a/src/main/kotlin/com/openlattice/launchpad/configuration/Constants.kt +++ b/src/main/kotlin/com/openlattice/launchpad/configuration/Constants.kt @@ -30,6 +30,7 @@ object Constants { const val LEGACY_CSV_FORMAT = "com.openlattice.launchpad.csv" const val CSV_FORMAT = "csv" const val ORC_FORMAT = "orc" + const val FILESYSTEM_DRIVER = "filesystem" const val S3_DRIVER = "s3" const val UNKNOWN = "unknown"