diff --git a/examples/example-github-java-connector/README.md b/examples/example-github-java-connector/README.md index 660abe7..7d9f2ad 100644 --- a/examples/example-github-java-connector/README.md +++ b/examples/example-github-java-connector/README.md @@ -58,7 +58,6 @@ Let's take a look at the structure of this connector. │ ├── main │ └── test ├── integration-test -├── java-module ├── manifest.yml ├── scripts ├── setup.sql diff --git a/examples/example-github-java-connector/java-module/build.gradle b/examples/example-github-java-connector/java-module/build.gradle deleted file mode 100644 index 9044e57..0000000 --- a/examples/example-github-java-connector/java-module/build.gradle +++ /dev/null @@ -1,50 +0,0 @@ -buildscript { - repositories { - mavenCentral() - gradlePluginPortal() - } - dependencies { - classpath "gradle.plugin.com.github.johnrengelman:shadow:8.0.0" - } -} - -plugins { - id 'java' -} - - -repositories { - mavenCentral() - mavenLocal() -} - -compileJava { - sourceCompatibility = '11' - targetCompatibility = '11' -} - -group 'com.snowflake' -version '1.0-SNAPSHOT' - -apply plugin: 'com.github.johnrengelman.shadow' - -dependencies { - implementation 'com.snowflake:snowpark:1.8.0' - implementation 'com.snowflake:connectors-java-sdk:1.0-SNAPSHOT' - - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' -} - -jar { - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - from { - configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } - } -} - -test { - useJUnitPlatform() -} -sourceCompatibility = JavaVersion.VERSION_11 -targetCompatibility = JavaVersion.VERSION_11 diff --git a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/api/ConnectorManagementProcedures.java b/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/api/ConnectorManagementProcedures.java deleted file mode 100644 index d5d7898..0000000 --- a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/api/ConnectorManagementProcedures.java +++ /dev/null @@ -1,70 +0,0 @@ -package com.snowflake.connectors.sdk.example_github_connector.api; - -import com.snowflake.connectors.sdk.common.DefaultKeyValueTable; -import com.snowflake.snowpark_java.Session; -import com.snowflake.snowpark_java.types.Variant; - -import java.util.Map; - -import static com.snowflake.connectors.sdk.example_github_connector.application.Infrastructure.APP_ROLE; -import static java.lang.String.format; - -public class ConnectorManagementProcedures { - public String provision_connector( - Session session, - Variant configuration - ) { - Map map = configuration.asMap(); - String integrationName = map.get("external_access_integration_name").asString(); - String tokenName = map.get("secret_name").asString(); - String destinationDatabaseName = map.get("destination_database").asString(); - - Map connectionConfiguration = Map.of( - "external_access_integration_name", integrationName, - "secret_name", map.get("secret_name").asString(), - "destination_database", map.get("destination_database").asString(), - "warehouse", map.get("warehouse").asString() - ); - setConfigValue(session, "config", new Variant(connectionConfiguration)); - alterConnectivityUdf(session, tokenName, integrationName); - createDestinationDatabase(session, destinationDatabaseName); - - return "Connector provisioned"; - } - - private void createDestinationDatabase(Session session, String databaseName) { - session.sql(format("CREATE DATABASE %s", databaseName)).collect(); - session.sql(format("GRANT USAGE ON DATABASE %s TO APPLICATION ROLE %s", databaseName, APP_ROLE)).collect(); - session.sql(format("CREATE SCHEMA %s.DEST_SCHEMA", databaseName)).collect(); - session.sql(format("GRANT USAGE ON SCHEMA %s.DEST_SCHEMA TO APPLICATION ROLE %s", databaseName, APP_ROLE)).collect(); - } - - public String configure_connector(Session session, String secretName, String apiIntegrationName, String databaseName) { - setConfigValue(session, "gh_secret_name", new Variant(secretName)); - alterConnectivityUdf(session, secretName, apiIntegrationName); - - createDestinationDatabase(session, databaseName); - - var table = new DefaultKeyValueTable(session, "STATE.APP_CONFIGURATION"); - table.update("dest_database_name", new Variant(databaseName)); - return "Connector configured."; - } - - private void alterConnectivityUdf(Session session, String secretName, String integrationName) { - String connectivityUdfName = "PUBLIC.INGEST_DATA(STRING)"; - String query = format("ALTER PROCEDURE %s \n" + - " SET SECRETS=('token' = %s)\n" + - " EXTERNAL_ACCESS_INTEGRATIONS=(%s)", - connectivityUdfName, - secretName, - integrationName - ); - session.sql(query).collect(); - } - - public String setConfigValue(Session session, String configName, Variant configValue) { - var table = new DefaultKeyValueTable(session, "STATE.APP_CONFIGURATION"); - table.update(configName, configValue); - return "Set config " + configName + " to value " + configValue.asJsonString(); - } -} diff --git a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/api/ResourcesManagementApi.java b/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/api/ResourcesManagementApi.java deleted file mode 100644 index a061399..0000000 --- a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/api/ResourcesManagementApi.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.snowflake.connectors.sdk.example_github_connector.api; - -import com.snowflake.connectors.sdk.common.DefaultKeyValueTable; -import com.snowflake.snowpark_java.Session; -import com.snowflake.snowpark_java.types.Variant; - -import java.util.Map; - -import static com.snowflake.connectors.sdk.example_github_connector.application.Infrastructure.TASKS_SCHEMA; -import static java.lang.String.format; - -public class ResourcesManagementApi { - - public String enable_resource(Session session, String resourceId) { - var table = new DefaultKeyValueTable(session, "STATE.RESOURCE_CONFIGURATION"); - Map resourceConfiguration = Map.of("enabled", true); - table.update(resourceId, new Variant(resourceConfiguration)); - - createTask(session, resourceId); - - return format("Resource %s enabled.", resourceId); - } - - private void createTask(Session session, String resourceId) { - var table = new DefaultKeyValueTable(session, "STATE.APP_CONFIGURATION"); - Map configuration = table.fetch("config").asMap(); - - String warehouse = configuration.get("warehouse").toString(); - - String taskName = format("%s.INGEST2_%s", TASKS_SCHEMA, resourceId.replaceAll("/", "_").toUpperCase()); - String createTaskDdl = format("CREATE OR REPLACE TASK IDENTIFIER('%s') WAREHOUSE = %s SCHEDULE = '15 minutes' AS CALL PUBLIC.INGEST_DATA('%s')", taskName, warehouse, resourceId); - session.sql(createTaskDdl).collect(); - - session.sql(format("EXECUTE TASK IDENTIFIER('%s')", taskName)).collect(); - session.sql(format("ALTER TASK IDENTIFIER('%s') RESUME", taskName)).collect(); - } - -} diff --git a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/application/Infrastructure.java b/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/application/Infrastructure.java deleted file mode 100644 index 0091bcf..0000000 --- a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/application/Infrastructure.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.snowflake.connectors.sdk.example_github_connector.application; - -public class Infrastructure { - public static final String SCHEMA = "PUBLIC"; - public static final String APP_ROLE = "APP_ROLE"; - public static final String TASKS_SCHEMA = "TASKS"; -} diff --git a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/GithubApiHttpClient.java b/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/GithubApiHttpClient.java deleted file mode 100644 index 6b38a8d..0000000 --- a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/GithubApiHttpClient.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.snowflake.connectors.sdk.example_github_connector.ingestion; - -import com.snowflake.snowpark_java.types.SnowflakeSecrets; - -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; - -public class GithubApiHttpClient { - private final HttpClient client; - private final String secret; - - public GithubApiHttpClient() { - this.client = HttpClient.newHttpClient(); - this.secret = SnowflakeSecrets.newInstance().getGenericSecretString("token"); - } - - public HttpResponse get(String url) { - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .GET() - .header("Authorization", String.format("Bearer %s", secret)) - .header("Content-Type", "application/json") - .timeout(Duration.ofSeconds(60)) - .build(); - try { - HttpResponse response = - client.send(request, HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() != 200) { - throw new RuntimeException("HttpRequest request failed with response code: " + response.statusCode() + " and body: " + response.statusCode()); - } - return response; - } catch(IOException | InterruptedException ex) { - throw new RuntimeException("HttpRequest failed", ex); - } - } -} diff --git a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/Ingestion.java b/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/Ingestion.java deleted file mode 100644 index 9c83f48..0000000 --- a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/Ingestion.java +++ /dev/null @@ -1,110 +0,0 @@ -package com.snowflake.connectors.sdk.example_github_connector.ingestion; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.snowflake.connectors.sdk.common.DefaultKeyValueTable; -import com.snowflake.connectors.sdk.example_github_connector.model.GithubIssue; -import com.snowflake.snowpark_java.Row; -import com.snowflake.snowpark_java.SaveMode; -import com.snowflake.snowpark_java.Session; -import com.snowflake.snowpark_java.types.DataTypes; -import com.snowflake.snowpark_java.types.StructField; -import com.snowflake.snowpark_java.types.StructType; -import com.snowflake.snowpark_java.types.Variant; - -import java.net.http.HttpResponse; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.snowflake.connectors.sdk.example_github_connector.application.Infrastructure.APP_ROLE; - -public class Ingestion { - private static final ObjectMapper objectMapper = new ObjectMapper(); - - public String ingest_resource(Session session, String resourceName) { - var table = new DefaultKeyValueTable(session, "STATE.APP_CONFIGURATION"); - String destDatabaseName = table.fetch("dest_database_name").asString(); - String destTableName = destDatabaseName + "." + "DEST_SCHEMA" + "." + resourceName; - - String url = String.format("https://api.github.com/repos/apache/%s/issues", resourceName); - String query = String.format("SELECT rs.value AS result FROM LATERAL FLATTEN(INPUT => parse_json(PUBLIC.FETCH_DATA('%s'))) rs", url); - session.sql(query).write().mode(SaveMode.Append).saveAsTable(destTableName); - - session.sql(String.format("GRANT SELECT ON TABLE %s TO APPLICATION ROLE %s", destTableName, APP_ROLE)).collect(); - return "Stored data to table " + destTableName; - } - - public Variant ingest_data(Session session, String resourceId) { - var table = new DefaultKeyValueTable(session, "STATE.APP_CONFIGURATION"); - Map configuration = table.fetch("config").asMap(); - String destDatabaseName = configuration.get("destination_database").toString(); - - String destTableName = destDatabaseName + "." + "DEST_SCHEMA" + "." + resourceId.replaceAll("/","_"); - - var split = resourceId.split("/"); - String organization = split[0]; - String repository = split[1]; - - String url = String.format("https://api.github.com/repos/%s/%s/issues", organization, repository); - - while (true) { - FetchResult result = fetch_single_batch(session, url, destTableName); - session.sql(String.format("GRANT SELECT ON TABLE %s TO APPLICATION ROLE %s", destTableName, APP_ROLE)).collect(); - if (result.getNextPageUrl() == null) { - break; - } - url = result.getNextPageUrl(); - } - - return new Variant("Stored data to table " + destTableName); - } - - class FetchResult { - private final String nextPageUrl; - - FetchResult(String nextPageUrl) { - this.nextPageUrl = nextPageUrl; - } - - public String getNextPageUrl() { - return nextPageUrl; - } - } - - public FetchResult fetch_single_batch(Session session, String url, String destTableName) { - - HttpResponse response = new GithubApiHttpClient().get(url); - - String body = response.body(); - GithubIssue[] issues; - try { - issues = objectMapper.readValue(body, GithubIssue[].class); - } catch (JsonProcessingException e) { - throw new RuntimeException("Cannot parse json", e); - } - - StructType schema = - StructType.create( - new StructField("id", DataTypes.IntegerType), - new StructField("number", DataTypes.IntegerType), - new StructField("title", DataTypes.StringType), - new StructField("state", DataTypes.StringType), - new StructField("node_id", DataTypes.StringType), - new StructField("url", DataTypes.StringType) - ); - - List list = Arrays.stream(issues).map((i) -> Row.create(i.getId(), i.getNumber(), i.getTitle(), i.getState(), i.getNode_id(), i.getUrl())).collect(Collectors.toList()); - session.createDataFrame( list.toArray(new Row[]{}), schema).write().mode(SaveMode.Append).saveAsTable(destTableName);; - return parseLinks(response); - } - - private FetchResult parseLinks(HttpResponse response) { - return response.headers() - .firstValue("Link") - .map((it) -> new LinkHeaderParser().parseLink(it)) - .map((it) -> new FetchResult(it.get("next"))) - .orElse(new FetchResult(null)); - } -} diff --git a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/LinkHeaderParser.java b/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/LinkHeaderParser.java deleted file mode 100644 index eb8f6df..0000000 --- a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/LinkHeaderParser.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.snowflake.connectors.sdk.example_github_connector.ingestion; - -import java.util.AbstractMap; -import java.util.Arrays; -import java.util.Map; -import java.util.stream.Collectors; - -import static java.util.Arrays.stream; - -/** - * Simple parser for Link header. - * This is NOT a complete implementation of https://www.rfc-editor.org/rfc/rfc5988.html - */ -public class LinkHeaderParser { - Map parseLink(String linkHeader) { - return stream(linkHeader.split(",")).map(String::trim).map( it -> { - var split = it.split(";"); - String url = split[0].substring(1, split[0].length()-1); - String rawRel = split[1].trim().split("=")[1]; - String rel = rawRel.trim().substring(1, rawRel.length()-1); - return new AbstractMap.SimpleImmutableEntry<>(rel, url); - }).collect(Collectors.toMap(AbstractMap.SimpleImmutableEntry::getKey, AbstractMap.SimpleImmutableEntry::getValue)); - } -} diff --git a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/model/GithubIssue.java b/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/model/GithubIssue.java deleted file mode 100644 index 5cb857b..0000000 --- a/examples/example-github-java-connector/java-module/src/main/java/com/snowflake/connectors/sdk/example_github_connector/model/GithubIssue.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.snowflake.connectors.sdk.example_github_connector.model; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class GithubIssue { - private int id; - private int number; - private String title; - private String state; - private String url; - - - public void setId(int id) { - this.id = id; - } - - public void setNode_id(String node_id) { - this.node_id = node_id; - } - - public int getId() { - return id; - } - - public String getNode_id() { - return node_id; - } - - private String node_id; - - public void setState(String state) { - this.state = state; - } - - public void setUrl(String url) { - this.url = url; - } - - public void setLocked(boolean locked) { - this.locked = locked; - } - - - public String getState() { - return state; - } - - public String getUrl() { - return url; - } - - public boolean isLocked() { - return locked; - } - - private boolean locked; - - - - public GithubIssue() { - } - - public int getNumber() { - return number; - } - - public String getTitle() { - return title; - } - - public void setNumber(int number) { - this.number = number; - } - - public void setTitle(String title) { - this.title = title; - } -} diff --git a/examples/example-github-java-connector/java-module/src/test/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/LinkHeaderParserTest.java b/examples/example-github-java-connector/java-module/src/test/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/LinkHeaderParserTest.java deleted file mode 100644 index 13f0588..0000000 --- a/examples/example-github-java-connector/java-module/src/test/java/com/snowflake/connectors/sdk/example_github_connector/ingestion/LinkHeaderParserTest.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.snowflake.connectors.sdk.example_github_connector.ingestion; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class LinkHeaderParserTest { - - @Test - void testParsingLinkHeader() { - // given - String linkHeader = "; rel=\"prev\", ; rel=\"first\""; - - // when - var result = new LinkHeaderParser().parseLink(linkHeader); - - // then - Map expected = Map.of( - "prev", "https://api.github.com/repositories/33884891/issues?page=27", - "first", "https://api.github.com/repositories/33884891/issues?page=1" - ); - assertTrue(result.equals(expected)); - } -}