diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad473c28..99794fc9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,35 +1,46 @@ name: Release -on: - workflow_dispatch +permissions: + contents: write +on: workflow_dispatch jobs: release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Release - uses: qcastel/github-actions-maven-release@v1.11.1 - with: - release-branch-name: "master" - maven-args: "-P sonatype" - git-release-bot-name: "release-bot" - git-release-bot-email: "release-bot@fleetpin.co.nz" - - gpg-enabled: "true" - gpg-key-id: ${{ secrets.GPG_KEY_ID }} - gpg-key: ${{ secrets.GPG_KEY }} - - maven-repo-server-id: sonatype - maven-repo-server-username: ${{ secrets.MVN_REPO_PRIVATE_REPO_USER }} - maven-repo-server-password: ${{ secrets.MVN_REPO_PRIVATE_REPO_PASSWORD }} - - access-token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: zulu + - uses: whelk-io/maven-settings-xml-action@v22 + with: + servers: > + [ + { "id": "sonatype", "username": "${{ secrets.MVN_REPO_PRIVATE_REPO_USER }}", "password": "${{ secrets.MVN_REPO_PRIVATE_REPO_PASSWORD }}" } + ] + - name: set name + run: | + git config --global user.name "release-bot"; + git config --global user.email "release-bot@fleetpin.co.nz"; + + - name: add key + run: | + echo "${{ secrets.GPG_KEY }}" | base64 -d > private.key + gpg --batch --import ./private.key + rm ./private.key + gpg --list-secret-keys --keyid-format LONG + + - name: prepare + run: | + mvn release:prepare -Dusername=${{ secrets.GITHUB_TOKEN }} -P sonatype + + - name: release + run: | + mvn release:perform -Dusername=${{ secrets.GITHUB_TOKEN }} -P sonatype diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bee3c8fe..75e8a838 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,25 +1,26 @@ name: Test -on: +on: push: jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Build with Maven - run: mvn -B test --file pom.xml - - uses: ashley-taylor/junit-report-annotations-action@master - if: always() - with: - access-token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: zulu + - name: Build with Maven + run: mvn -B test --file pom.xml + - uses: ashley-taylor/junit-report-annotations-action@master + if: always() + with: + access-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/graphql-database-dynmodb-history-lambda/pom.xml b/graphql-database-dynmodb-history-lambda/pom.xml index 2e172789..d3b4a691 100644 --- a/graphql-database-dynmodb-history-lambda/pom.xml +++ b/graphql-database-dynmodb-history-lambda/pom.xml @@ -5,7 +5,7 @@ com.fleetpin graphql-database-manager - 0.2.30-SNAPSHOT + 3.0.4-SNAPSHOT graphql-database-dynmodb-history-lambda @@ -37,9 +37,9 @@ - https://github.com/fleetpin/graphql-dynamodb-manager - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git + https://github.com/ashley-taylor/graphql-dynamodb-manager + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git HEAD @@ -72,24 +72,20 @@ graphql-database-manager-dynamo ${project.parent.version} - com.amazonaws - aws-java-sdk-dynamodb - 1.11.724 + aws-lambda-java-events-sdk-transformer + 3.1.0 com.amazonaws - aws-lambda-java-core - 1.1.0 + aws-lambda-java-events + 3.14.0 - com.amazonaws - aws-lambda-java-events - 2.2.7 + aws-lambda-java-core + 1.2.3 @@ -98,7 +94,7 @@ org.apache.maven.plugins maven-dependency-plugin - 2.10 + 3.8.0 copy @@ -129,7 +125,7 @@ attach-sources - jar + jar-no-fork @@ -137,7 +133,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.1.1 + 3.10.0 attach-javadocs @@ -150,7 +146,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + 3.2.6 sign-artifacts @@ -164,7 +160,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.7.0 true sonatype diff --git a/graphql-database-dynmodb-history-lambda/src/main/java/com/fleetpin/graphql/database/dynamo/history/lambda/DynamoUtil.java b/graphql-database-dynmodb-history-lambda/src/main/java/com/fleetpin/graphql/database/dynamo/history/lambda/DynamoUtil.java deleted file mode 100644 index 6fad3e55..00000000 --- a/graphql-database-dynmodb-history-lambda/src/main/java/com/fleetpin/graphql/database/dynamo/history/lambda/DynamoUtil.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.fleetpin.graphql.database.dynamo.history.lambda; - -import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; -import software.amazon.awssdk.core.SdkBytes; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.Identity; -import software.amazon.awssdk.services.dynamodb.model.Record; -import software.amazon.awssdk.services.dynamodb.model.StreamRecord; - -public class DynamoUtil { - - public static Record toV2(DynamodbStreamRecord record) { - var builder = Record.builder(); - builder.awsRegion(record.getAwsRegion()); - builder.dynamodb(toV2(record.getDynamodb())); - builder.eventID(record.getEventID()); - builder.eventName(record.getEventName()); - builder.eventSource(record.getEventSource()); - builder.eventVersion(record.getEventVersion()); - builder.userIdentity(toV2(record.getUserIdentity())); - return builder.build(); - } - - public static Identity toV2(com.amazonaws.services.dynamodbv2.model.Identity userIdentity) { - if (userIdentity == null) { - return null; - } - var builder = Identity.builder(); - builder.principalId(userIdentity.getPrincipalId()); - builder.type(userIdentity.getType()); - return builder.build(); - } - - public static StreamRecord toV2(com.amazonaws.services.dynamodbv2.model.StreamRecord dynamodb) { - var builder = StreamRecord.builder(); - - builder.approximateCreationDateTime(dynamodb.getApproximateCreationDateTime().toInstant()); - builder.keys(toV2(dynamodb.getKeys())); - builder.newImage(toV2(dynamodb.getNewImage())); - builder.oldImage(toV2(dynamodb.getOldImage())); - builder.sequenceNumber(dynamodb.getSequenceNumber()); - builder.sizeBytes(dynamodb.getSizeBytes()); - builder.streamViewType(dynamodb.getStreamViewType()); - - return builder.build(); - } - - public static Map toV2(Map attribute) { - if (attribute == null) { - return null; - } - Map toReturn = new HashMap<>(); - attribute.forEach((key, value) -> toReturn.put(key, toV2(value))); - return toReturn; - } - - public static AttributeValue toV2(com.amazonaws.services.dynamodbv2.model.AttributeValue value) { - if (value.getB() != null) { - return AttributeValue.builder().b(SdkBytes.fromByteBuffer(value.getB())).build(); - } - if (value.getBOOL() != null) { - return AttributeValue.builder().bool(value.getBOOL()).build(); - } - if (value.getNULL() != null) { - return AttributeValue.builder().nul(value.getNULL()).build(); - } - if (value.getBS() != null) { - return AttributeValue.builder().bs(value.getBS().stream().map(SdkBytes::fromByteBuffer).collect(Collectors.toList())).build(); - } - if (value.getL() != null) { - return AttributeValue.builder().l(value.getL().stream().map(DynamoUtil::toV2).collect(Collectors.toList())).build(); - } - if (value.getM() != null) { - return AttributeValue.builder().m(toV2(value.getM())).build(); - } - if (value.getN() != null) { - return AttributeValue.builder().n(value.getN()).build(); - } - if (value.getNS() != null) { - return AttributeValue.builder().ns(value.getNS()).build(); - } - if (value.getS() != null) { - return AttributeValue.builder().s(value.getS()).build(); - } - if (value.getSS() != null) { - return AttributeValue.builder().ss(value.getSS()).build(); - } - throw new RuntimeException("Unknown type " + value); - } -} diff --git a/graphql-database-dynmodb-history-lambda/src/main/java/com/fleetpin/graphql/database/dynamo/history/lambda/HistoryLambda.java b/graphql-database-dynmodb-history-lambda/src/main/java/com/fleetpin/graphql/database/dynamo/history/lambda/HistoryLambda.java index fc162e9c..654ca24f 100644 --- a/graphql-database-dynmodb-history-lambda/src/main/java/com/fleetpin/graphql/database/dynamo/history/lambda/HistoryLambda.java +++ b/graphql-database-dynmodb-history-lambda/src/main/java/com/fleetpin/graphql/database/dynamo/history/lambda/HistoryLambda.java @@ -1,3 +1,15 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + package com.fleetpin.graphql.database.dynamo.history.lambda; import static java.util.stream.Collectors.groupingBy; @@ -5,14 +17,13 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.transformers.v2.DynamodbEventTransformer; import com.fleetpin.graphql.database.manager.dynamo.HistoryUtil; import java.util.HashMap; import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.Record; import software.amazon.awssdk.services.dynamodb.model.WriteRequest; @@ -22,39 +33,42 @@ public HistoryLambda() {} public abstract String getTableName(); - public abstract DynamoDbAsyncClient getClient(); + public abstract DynamoDbClient getClient(); @Override public Void handleRequest(DynamodbEvent input, Context context) { - var records = input.getRecords().stream().map(DynamoUtil::toV2); + var records = DynamodbEventTransformer.toRecordsV2(input); process(records); return null; } - public void process(Stream records) { + public void process(List records) { int chunkSize = 25; AtomicInteger counter = new AtomicInteger(); var chunks = HistoryUtil - .toHistoryValue(records) + .toHistoryValue(records.stream()) .map(item -> WriteRequest.builder().putRequest(builder -> builder.item(item)).build()) .collect(groupingBy(x -> counter.getAndIncrement() / chunkSize)) .values(); - var futures = chunks - .stream() + chunks + .parallelStream() .filter(chunk -> !chunk.isEmpty()) - .map(chunk -> { + .forEach(chunk -> { var items = new HashMap>(); items.put(getTableName(), chunk); - return getClient().batchWriteItem(builder -> builder.requestItems(items)); - }) - .toArray(CompletableFuture[]::new); - - try { - CompletableFuture.allOf(futures).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); + writeItems(items); + }); + } + + private void writeItems(Map> items) { + var response = getClient().batchWriteItem(builder -> builder.requestItems(items)); + + var unprocessed = response.unprocessedItems(); + + if (!unprocessed.isEmpty()) { + writeItems(unprocessed); } } } diff --git a/graphql-database-manager-core/pom.xml b/graphql-database-manager-core/pom.xml index 8c3e059d..454a8a79 100644 --- a/graphql-database-manager-core/pom.xml +++ b/graphql-database-manager-core/pom.xml @@ -5,13 +5,13 @@ com.fleetpin graphql-database-manager - 0.2.30-SNAPSHOT + 3.0.4-SNAPSHOT graphql-database-manager-core - 5.6.0 + 5.11.0 UTF-8 @@ -29,9 +29,9 @@ - https://github.com/fleetpin/graphql-dynamodb-manager - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git + https://github.com/ashley-taylor/graphql-dynamodb-manager + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git HEAD @@ -62,10 +62,16 @@ com.fleetpin graphql-builder - 0.1.1 + 3.0.2 provided + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + org.junit.jupiter junit-jupiter @@ -78,7 +84,7 @@ org.apache.maven.plugins maven-dependency-plugin - 2.10 + 3.8.0 copy diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DataWriter.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DataWriter.java index 4992f017..98517f8d 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DataWriter.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DataWriter.java @@ -1,16 +1,20 @@ package com.fleetpin.graphql.database.manager; -import java.util.*; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; import java.util.function.Function; public class DataWriter { private final Function, CompletableFuture> bulkWriter; private final List toPut = new ArrayList<>(); + private final Consumer> handleFuture; - public DataWriter(Function, CompletableFuture> bulkWriter) { + public DataWriter(Function, CompletableFuture> bulkWriter, Consumer> handleFuture) { this.bulkWriter = bulkWriter; + this.handleFuture = handleFuture; } public int dispatchSize() { @@ -40,6 +44,7 @@ public CompletableFuture put(String organisationId, T entit synchronized (toPut) { toPut.add(putValue); } + handleFuture.accept(future); return future; } } diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Database.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Database.java index 86c56529..eac2f4fa 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Database.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Database.java @@ -15,12 +15,20 @@ import com.fleetpin.graphql.database.manager.access.ForbiddenWriteException; import com.fleetpin.graphql.database.manager.access.ModificationPermission; import com.fleetpin.graphql.database.manager.util.BackupItem; +import com.fleetpin.graphql.database.manager.util.HistoryBackupItem; import com.fleetpin.graphql.database.manager.util.TableCoreUtil; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -30,6 +38,8 @@ @SuppressWarnings("unchecked") public class Database { + public static ExecutorService VIRTUAL_THREAD_POOL = Executors.newVirtualThreadPerTaskExecutor(); + private String organisationId; private final DatabaseDriver driver; @@ -40,10 +50,13 @@ public class Database { private final Function> putAllow; + private final AtomicInteger submitted; + Database(String organisationId, DatabaseDriver driver, ModificationPermission putAllow) { this.organisationId = organisationId; this.driver = driver; this.putAllow = putAllow; + this.submitted = new AtomicInteger(); items = new TableDataLoader<>( @@ -52,7 +65,8 @@ public class Database { return driver.get(keys); }, DataLoaderOptions.newOptions().setMaxBatchSize(driver.maxBatchSize()) - ) + ), + this::handleFuture ); // will auto call global queries = @@ -62,7 +76,8 @@ public class Database { return merge(keys.stream().map(driver::query)); }, DataLoaderOptions.newOptions().setBatchingEnabled(false) - ) + ), + this::handleFuture ); // will auto call global queryHistories = @@ -72,10 +87,11 @@ public class Database { return merge(keys.stream().map(driver::queryHistory)); }, DataLoaderOptions.newOptions().setBatchingEnabled(false) - ) + ), + this::handleFuture ); // will auto call global - put = new DataWriter(driver::bulkPut); + put = new DataWriter(driver::bulkPut, this::handleFuture); } public CompletableFuture> query(Class type, Function, QueryBuilder> func) { @@ -119,10 +135,18 @@ public CompletableFuture> takeBackup(String organisationId) { return driver.takeBackup(organisationId); } + public CompletableFuture> takeHistoryBackup(String organisationId) { + return driver.takeHistoryBackup(organisationId); + } + public CompletableFuture restoreBackup(List entities) { return driver.restoreBackup(entities); } + public CompletableFuture restoreHistoryBackup(List entities) { + return driver.restoreHistoryBackup(entities); + } + public CompletableFuture> delete(String organisationId, Class clazz) { return driver.delete(organisationId, clazz); } @@ -219,7 +243,7 @@ public CompletableFuture deleteLinks(T entity) { if (!allow) { throw new ForbiddenWriteException("Delete links not allowed for " + TableCoreUtil.table(entity.getClass()) + " with id " + entity.getId()); } - //impact of clearing links to tricky + // impact of clearing links to tricky items.clearAll(); queries.clearAll(); return driver.deleteLinks(organisationId, entity); @@ -236,18 +260,18 @@ public CompletableFuture destroyOrganisation(final String organisationI * @param database entity type to update * @param entity revision must match database or request will fail * @return updated entity with the revision incremented by one - * CompletableFuture will fail with a RevisionMismatchException + * CompletableFuture will fail with a RevisionMismatchException */ public CompletableFuture put(T entity) { return put(entity, true); } /** - * @param database entity type to update + * @param database entity type to update * @param entity revision must match database or request will fail - * @param check Will only pass if the entity revision matches what is currently in the database + * @param check Will only pass if the entity revision matches what is currently in the database * @return updated entity with the revision incremented by one - * CompletableFuture will fail with a RevisionMismatchException + * CompletableFuture will fail with a RevisionMismatchException */ public CompletableFuture put(T entity, boolean check) { return putAllow @@ -296,24 +320,18 @@ private CompletableFuture> merge(Stream> stream }); } - private static final Executor DELAYER = CompletableFuture.delayedExecutor(10, TimeUnit.MILLISECONDS); - - @SuppressWarnings("rawtypes") - public void start(CompletableFuture toReturn) { - if (toReturn.isDone()) { - return; + private void start() { + if (items.dispatchDepth() > 0) { + items.dispatch(); } - - if (items.dispatchDepth() > 0 || queries.dispatchDepth() > 0 || queryHistories.dispatchDepth() > 0 || put.dispatchSize() > 0) { - CompletableFuture[] all = new CompletableFuture[] { items.dispatch(), queries.dispatch(), queryHistories.dispatch(), put.dispatch() }; - CompletableFuture - .allOf(all) - .whenComplete((response, error) -> { - //go around again - start(toReturn); - }); - } else { - CompletableFuture.supplyAsync(() -> null, DELAYER).acceptEither(toReturn, __ -> start(toReturn)); + if (queries.dispatchDepth() > 0) { + queries.dispatch(); + } + if (queryHistories.dispatchDepth() > 0) { + queryHistories.dispatch(); + } + if (put.dispatchSize() > 0) { + put.dispatch(); } } @@ -395,6 +413,41 @@ public String newId() { } public Set getLinkIds(Table entity, Class type) { - return Collections.unmodifiableSet(TableAccess.getTableLinks(entity).get(TableCoreUtil.table(type))); + var links = TableAccess.getTableLinks(entity).get(TableCoreUtil.table(type)); + if (links == null) { + return Collections.emptySet(); + } + return Collections.unmodifiableSet(links); + } + + private CompletableFuture handleFuture(CompletableFuture future) { + if (future.isDone()) { + return future; + } + while (true) { + var current = submitted.get(); + if (current == 0) { + if (submitted.compareAndSet(0, 1)) { + run(); + } else { + continue; + } + } else { + if (submitted.compareAndSet(current, current + 1)) { + return future.thenApplyAsync(t -> t, VIRTUAL_THREAD_POOL); + } + } + } + } + + private void run() { + VIRTUAL_THREAD_POOL.submit(() -> { + var start = submitted.get(); + start(); + if (submitted.compareAndSet(start, 0)) { + return; + } + run(); + }); } } diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseDriver.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseDriver.java index fadeefa6..b8bb75aa 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseDriver.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseDriver.java @@ -13,10 +13,12 @@ package com.fleetpin.graphql.database.manager; import com.fleetpin.graphql.database.manager.util.BackupItem; -import com.google.common.collect.HashMultimap; +import com.fleetpin.graphql.database.manager.util.HistoryBackupItem; import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; public abstract class DatabaseDriver { @@ -42,8 +44,12 @@ public abstract CompletableFuture> getViaLinks( public abstract CompletableFuture restoreBackup(List entities); + public abstract CompletableFuture restoreHistoryBackup(List entities); + public abstract CompletableFuture> takeBackup(String organisationId); + public abstract CompletableFuture> takeHistoryBackup(String organisationId); + public abstract CompletableFuture> queryHistory(DatabaseQueryHistoryKey key); public abstract CompletableFuture> queryGlobal(Class type, String value); @@ -78,7 +84,7 @@ protected void setLinks(final T entity, final String type, fin entity.setLinks(type, groupIds); } - protected HashMultimap getLinks(final T entity) { + protected Map> getLinks(final T entity) { return entity.getLinks(); } @@ -93,7 +99,7 @@ protected void setUpdatedAt(final T entity, final Instant upda protected void setSource( final T entity, final String sourceTable, - final HashMultimap links, + final Map> links, final String sourceOrganisationId ) { entity.setSource(sourceTable, links, sourceOrganisationId); @@ -106,4 +112,6 @@ protected String getSourceTable(final T entity) { protected DatabaseKey createDatabaseKey(final String organisationId, final Class type, final String id) { return new DatabaseKey<>(organisationId, type, id); } + + protected abstract ScanResult startTableScan(TableScanQuery tableScanQuery, int segment, Object from); } diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseKey.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseKey.java index 9aff2433..98d2e3fe 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseKey.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseKey.java @@ -12,6 +12,7 @@ package com.fleetpin.graphql.database.manager; +import com.fleetpin.graphql.database.manager.util.TableCoreUtil; import java.util.Objects; public class DatabaseKey { @@ -22,7 +23,7 @@ public class DatabaseKey { DatabaseKey(String organisationId, Class type, String id) { this.organisationId = organisationId; - this.type = type; + this.type = TableCoreUtil.baseClass(type); this.id = id; } diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseManager.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseManager.java index 74edaee5..e814f90a 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseManager.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/DatabaseManager.java @@ -2,6 +2,7 @@ import com.fleetpin.graphql.database.manager.access.ModificationPermission; import java.util.concurrent.CompletableFuture; +import java.util.function.Function; public abstract class DatabaseManager { @@ -18,4 +19,16 @@ public Database getDatabase(String organisationId) { public Database getDatabase(String organisationId, ModificationPermission putAllow) { return new Database(organisationId, dynamoDb, putAllow); } + + public VirtualDatabase getVirtualDatabase(String organisationId) { + return new VirtualDatabase(getDatabase(organisationId)); + } + + public VirtualDatabase getVirtualDatabase(String organisationId, ModificationPermission putAllow) { + return new VirtualDatabase(getDatabase(organisationId, putAllow)); + } + + public TableScanner startTableScan(Function builder) { + return new TableScanner(builder.apply(new TableScanQueryBuilder()).build(), dynamoDb, this); + } } diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Query.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Query.java index 508e66d3..fff053c4 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Query.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Query.java @@ -8,15 +8,28 @@ public class Query { private final String startsWith; private final String after; private final Integer limit; + private final Integer threadCount; + private final Integer threadIndex; - Query(Class type, String startsWith, String after, Integer limit) { + Query(Class type, String startsWith, String after, Integer limit, Integer threadCount, Integer threadIndex) { if (type == null) { throw new RuntimeException("type can not be null, did you forget to call .on(Table::class)?"); } + + if (threadCount != null && !isPowerOfTwo(threadCount)) { + throw new RuntimeException("Thread count must be a power of two"); + } + + if ((threadCount != null && threadIndex == null) || (threadCount == null && threadIndex != null)) { + throw new RuntimeException("Thread count and thread index must both be defined if you are doing a parallel request"); + } + this.type = type; this.startsWith = startsWith; this.after = after; this.limit = limit; + this.threadCount = threadCount; + this.threadIndex = threadIndex; } public Class getType() { @@ -35,13 +48,21 @@ public Integer getLimit() { return limit; } + public Integer getThreadCount() { + return threadCount; + } + + public Integer getThreadIndex() { + return threadIndex; + } + public boolean hasLimit() { return getLimit() != null; } @Override public int hashCode() { - return Objects.hash(after, limit, startsWith, type); + return Objects.hash(after, limit, startsWith, type, threadIndex, threadCount); } @Override @@ -54,7 +75,15 @@ public boolean equals(Object obj) { Objects.equals(after, other.after) && Objects.equals(limit, other.limit) && Objects.equals(startsWith, other.startsWith) && - Objects.equals(type, other.type) + Objects.equals(type, other.type) && + Objects.equals(threadCount, other.threadCount) && + Objects.equals(threadIndex, other.threadIndex) ); } + + static boolean isPowerOfTwo(int n) { + if (n == 0) return false; + + return (int) (Math.ceil((Math.log(n) / Math.log(2)))) == (int) (Math.floor(((Math.log(n) / Math.log(2))))); + } } diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/QueryBuilder.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/QueryBuilder.java index 701a5701..5828e441 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/QueryBuilder.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/QueryBuilder.java @@ -8,6 +8,8 @@ public class QueryBuilder { private String startsWith; private String after; private Integer limit; + private Integer threadIndex; + private Integer threadCount; private QueryBuilder(Class type) { this.type = type; @@ -28,13 +30,23 @@ public QueryBuilder after(String from) { return this; } + public QueryBuilder threadCount(Integer threadCount) { + this.threadCount = threadCount; + return this; + } + + public QueryBuilder threadIndex(Integer threadIndex) { + this.threadIndex = threadIndex; + return this; + } + public QueryBuilder applyMutation(Consumer> mutator) { mutator.accept((QueryBuilder) this); return (QueryBuilder) this; } public Query build() { - return new Query(type, startsWith, after, limit); + return new Query(type, startsWith, after, limit, threadCount, threadIndex); } public static QueryBuilder create(Class type) { diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/QueryHistoryBuilder.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/QueryHistoryBuilder.java index 034f8520..f8f221f5 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/QueryHistoryBuilder.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/QueryHistoryBuilder.java @@ -1,7 +1,6 @@ package com.fleetpin.graphql.database.manager; import com.fleetpin.graphql.database.manager.util.HistoryCoreUtil; -import com.google.common.base.Preconditions; import java.time.Instant; import java.util.function.Consumer; @@ -55,16 +54,22 @@ public QueryHistoryBuilder applyMutation(Consumer> mut } public QueryHistory build() { - Preconditions.checkArgument(HistoryCoreUtil.hasHistory(type), "Can only do history when history annotation is present."); - Preconditions.checkArgument(!(id != null && startsWith != null), "ID and StartsWith cannot both be set."); - Preconditions.checkArgument(!(id == null && startsWith == null), "ID or StartsWith must be set."); + checkArgument(HistoryCoreUtil.hasHistory(type), "Can only do history when history annotation is present."); + checkArgument(!(id != null && startsWith != null), "ID and StartsWith cannot both be set."); + checkArgument(!(id == null && startsWith == null), "ID or StartsWith must be set."); if (fromRevision != null || toRevision != null) { - Preconditions.checkArgument(fromUpdatedAt == null && toUpdatedAt == null, "Revision and CreatedAt cannot both be set."); - Preconditions.checkArgument(startsWith == null, "StartsWith can only be used with updatedAt."); + checkArgument(fromUpdatedAt == null && toUpdatedAt == null, "Revision and CreatedAt cannot both be set."); + checkArgument(startsWith == null, "StartsWith can only be used with updatedAt."); } return new QueryHistory(type, startsWith, id, fromRevision, toRevision, fromUpdatedAt, toUpdatedAt); } + private void checkArgument(boolean pass, String msg) { + if (!pass) { + throw new IllegalArgumentException(msg); + } + } + public static QueryHistoryBuilder create(Class type) { return new QueryHistoryBuilder(type); } diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/ScanResult.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/ScanResult.java new file mode 100644 index 00000000..6b679292 --- /dev/null +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/ScanResult.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.fleetpin.graphql.database.manager; + +import java.util.ArrayList; +import java.util.function.Consumer; + +public record ScanResult(ArrayList> items, Object next) { + public record Item(String organisationId, T entity, Consumer replace, Consumer delete) {} +} diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/ScanUpdater.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/ScanUpdater.java new file mode 100644 index 00000000..5064843e --- /dev/null +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/ScanUpdater.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.fleetpin.graphql.database.manager; + +import com.fleetpin.graphql.database.manager.ScanResult.Item; +import java.util.function.BiConsumer; + +public record ScanUpdater(Class type, BiConsumer, T> updater) { + public static class ScanContext { + + private VirtualDatabase virtualDatabase; + private Item item; + + protected ScanContext(VirtualDatabase virtualDatabase, Item item) { + this.virtualDatabase = virtualDatabase; + this.item = item; + } + + public VirtualDatabase getVirtualDatabase() { + return virtualDatabase; + } + + public void delete() { + item.delete().accept(item.entity()); + } + + public void replace(T entity) { + item.replace().accept(entity); + } + } +} diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Table.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Table.java index f2d6bfff..35e48514 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Table.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/Table.java @@ -15,10 +15,13 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fleetpin.graphql.builder.annotations.GraphQLIgnore; import com.fleetpin.graphql.builder.annotations.Id; -import com.google.common.collect.HashMultimap; import java.time.Instant; import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; import java.util.Objects; +import java.util.Set; public abstract class Table { @@ -34,7 +37,7 @@ public abstract class Table { private String sourceOrganistaionId; @JsonIgnore - private HashMultimap links = HashMultimap.create(); + private Map> links = new HashMap<>(); @Id public String getId() { @@ -91,7 +94,7 @@ String getSourceTable() { return sourceTable; } - void setSource(String sourceTable, HashMultimap links, String sourceOrganisationId) { + void setSource(String sourceTable, Map> links, String sourceOrganisationId) { //so bad data does not cause error if (createdAt == null) { createdAt = Instant.MIN; @@ -111,13 +114,12 @@ String getSourceOrganisationId() { } void setLinks(String type, Collection groupIds) { - this.links.removeAll(type); - this.links.putAll(type, groupIds); + this.links.put(type, new HashSet<>(groupIds)); } @JsonIgnore @GraphQLIgnore - HashMultimap getLinks() { + Map> getLinks() { return links; } } diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableAccess.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableAccess.java index f8b54141..5f2c6adf 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableAccess.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableAccess.java @@ -1,12 +1,13 @@ package com.fleetpin.graphql.database.manager; -import com.google.common.collect.HashMultimap; +import java.util.Map; +import java.util.Set; public interface TableAccess { public static void setTableSource( final T table, final String sourceTable, - final HashMultimap links, + final Map> links, final String sourceOrganisationId ) { table.setSource(sourceTable, links, sourceOrganisationId); @@ -16,7 +17,7 @@ public static String getTableSourceOrganisation(final T table) return table.getSourceOrganisationId(); } - public static HashMultimap getTableLinks(final T table) { + public static Map> getTableLinks(final T table) { return table.getLinks(); } } diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableDataLoader.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableDataLoader.java index 0287607b..29e39ff3 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableDataLoader.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableDataLoader.java @@ -2,24 +2,29 @@ import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.function.Function; import java.util.stream.Collectors; import org.dataloader.DataLoader; public class TableDataLoader { private final DataLoader loader; + private final Function, CompletableFuture> handleFuture; - TableDataLoader(DataLoader loader) { + TableDataLoader(DataLoader loader, Function, CompletableFuture> handleFuture) { this.loader = loader; + this.handleFuture = handleFuture; } public CompletableFuture load(K key) { - return (CompletableFuture) loader.load(key); + var future = loader.load(key); + return (CompletableFuture) this.handleFuture.apply(future); } public CompletableFuture> loadMany(List keys) { - //annoying waste of memory/cpu to get around cast :( - return loader.loadMany(keys).thenApply(r -> r.stream().map(t -> (T) t).collect(Collectors.toList())); + // annoying waste of memory/cpu to get around cast :( + var future = loader.loadMany(keys).thenApply(r -> r.stream().map(t -> (T) t).collect(Collectors.toList())); + return (CompletableFuture>) this.handleFuture.apply(future); } public void clear(K key) { diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableScanQuery.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableScanQuery.java new file mode 100644 index 00000000..5078ee76 --- /dev/null +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableScanQuery.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.fleetpin.graphql.database.manager; + +import java.util.List; + +public record TableScanQuery(TableScanMonitor monitor, Integer parallelism, List> updaters) { + interface TableScanMonitor { + public void onScanSegmentStart(int segment, int itemCount, Object from); + + public void onScanSegmentComplete(int segment); + } +} diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableScanQueryBuilder.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableScanQueryBuilder.java new file mode 100644 index 00000000..9b8aac56 --- /dev/null +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableScanQueryBuilder.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.fleetpin.graphql.database.manager; + +import com.fleetpin.graphql.database.manager.ScanUpdater.ScanContext; +import com.fleetpin.graphql.database.manager.TableScanQuery.TableScanMonitor; +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; + +public class TableScanQueryBuilder { + + private int parallelism = Runtime.getRuntime().availableProcessors() * 5; + private List> updaters = new ArrayList<>(); + private TableScanMonitor monitor; + + public TableScanQueryBuilder parallelism(int parallelism) { + this.parallelism = parallelism; + return this; + } + + public TableScanQueryBuilder updater(ScanUpdater updater) { + updaters.add(updater); + return this; + } + + public TableScanQueryBuilder updater(Class type, BiConsumer, T> updater) { + updaters.add(new ScanUpdater(type, updater)); + return this; + } + + public TableScanQuery build() { + return new TableScanQuery(monitor, parallelism, updaters); + } +} diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableScanner.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableScanner.java new file mode 100644 index 00000000..441d1365 --- /dev/null +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/TableScanner.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.fleetpin.graphql.database.manager; + +import com.fleetpin.graphql.database.manager.ScanResult.Item; +import com.fleetpin.graphql.database.manager.ScanUpdater.ScanContext; +import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; + +public class TableScanner { + + private final TableScanQuery query; + private final DatabaseDriver driver; + private final DatabaseManager databaseManager; + + public TableScanner(TableScanQuery query, DatabaseDriver driver, DatabaseManager databaseManager) { + this.query = query; + this.driver = driver; + this.databaseManager = databaseManager; + } + + public CompletableFuture start() { + var workers = new ArrayList>(); + for (int i = 0; i < query.parallelism(); i++) { + var segment = i; + workers.add(CompletableFuture.supplyAsync(new ScannerWorker(segment), Database.VIRTUAL_THREAD_POOL)); + } + return CompletableFuture.allOf(workers.toArray(CompletableFuture[]::new)); + } + + private CompletableFuture process(Item item) { + for (var updater : query.updaters()) { + if (updater.type().isAssignableFrom(item.entity().getClass())) { + @SuppressWarnings("unchecked") + ScanUpdater update = (ScanUpdater) updater; + return CompletableFuture.runAsync( + () -> { + var virtualDatabase = databaseManager.getVirtualDatabase(item.organisationId()); + update.updater().accept(new ScanContext(virtualDatabase, item), item.entity()); + }, + Database.VIRTUAL_THREAD_POOL + ); + } + } + return CompletableFuture.completedFuture(null); + } + + private class ScannerWorker implements Supplier { + + private final int segment; + + public ScannerWorker(int segment) { + this.segment = segment; + } + + @Override + public ScannerWorker get() { + Object from = null; + do { + var scan = driver.startTableScan(query, segment, from); + if (query.monitor() != null) { + query.monitor().onScanSegmentStart(segment, scan.items().size(), from); + } + from = scan.next(); + + var workers = new ArrayList>(); + for (var item : scan.items()) { + workers.add(process(item)); + } + + CompletableFuture.allOf(workers.toArray(CompletableFuture[]::new)).join(); + } while (from != null); + if (query.monitor() != null) { + query.monitor().onScanSegmentComplete(segment); + } + + return this; + } + } +} diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/VirtualDatabase.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/VirtualDatabase.java new file mode 100644 index 00000000..8071adef --- /dev/null +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/VirtualDatabase.java @@ -0,0 +1,153 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.fleetpin.graphql.database.manager; + +import com.fleetpin.graphql.database.manager.util.BackupItem; +import com.fleetpin.graphql.database.manager.util.HistoryBackupItem; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +public class VirtualDatabase { + + private final Database database; + + public VirtualDatabase(Database database) { + this.database = database; + } + + public List delete(String organisationId, Class type) { + return database.delete(organisationId, type).join(); + } + + public T delete(T entity, boolean deleteLinks) { + return database.delete(entity, deleteLinks).join(); + } + + public List getLinks(final Table entry, Class target) { + return database.getLinks(entry, target).join(); + } + + public Boolean destroyOrganisation(final String organisationId) { + return database.destroyOrganisation(organisationId).join(); + } + + public T get(Class type, String id) { + return database.get(type, id).join(); + } + + public List get(Class type, List ids) { + return database.get(type, ids).join(); + } + + public T getLink(final Table entry, Class target) { + return database.getLink(entry, target).join(); + } + + public Set getLinkIds(Table entity, Class type) { + return database.getLinkIds(entity, type); + } + + public Optional getLinkOptional(final Table entry, Class target) { + return database.getLinkOptional(entry, target).join(); + } + + public Optional getOptional(Class type, String id) { + return database.getOptional(type, id).join(); + } + + public String getSourceOrganisationId(Table entity) { + return database.getSourceOrganisationId(entity); + } + + public T link(T entity, Class class1, String targetId) { + return database.link(entity, class1, targetId).join(); + } + + public T links(T entity, Class class1, List targetIds) { + return database.links(entity, class1, targetIds).join(); + } + + public String newId() { + return database.newId(); + } + + public T put(T entity) { + return database.put(entity).join(); + } + + public T put(T entity, boolean check) { + return database.put(entity, check).join(); + } + + public T putGlobal(T entity) { + return database.putGlobal(entity).join(); + } + + public List query(Class type) { + return database.query(type).join(); + } + + public List query(Query query) { + return database.query(query).join(); + } + + public List query(Class type, Function, QueryBuilder> func) { + return database.query(type, func).join(); + } + + public List queryGlobal(Class type, String id) { + return database.queryGlobal(type, id).join(); + } + + public T queryGlobalUnique(Class type, String id) { + return database.queryGlobalUnique(type, id).join(); + } + + public List queryHistory(QueryHistory query) { + return database.queryHistory(query).join(); + } + + public List querySecondary(Class type, String id) { + return database.querySecondary(type, id).join(); + } + + public T querySecondaryUnique(Class type, String id) { + return database.querySecondaryUnique(type, id).join(); + } + + public Void restoreBackup(List entities) { + return database.restoreBackup(entities).join(); + } + + public Void restoreHistoryBackup(List entities) { + return database.restoreHistoryBackup(entities).join(); + } + + public void setOrganisationId(String organisationId) { + database.setOrganisationId(organisationId); + } + + public List takeBackup(String organisationId) { + return database.takeBackup(organisationId).join(); + } + + public List takeHistoryBackup(String organisationId) { + return database.takeHistoryBackup(organisationId).join(); + } + + public T takeHistoryBackup(final T entity, final Class type, final String targetId) { + return database.unlink(entity, type, targetId).join(); + } +} diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/BackupItem.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/BackupItem.java index 257f00d6..8f4d33dd 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/BackupItem.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/BackupItem.java @@ -12,16 +12,15 @@ package com.fleetpin.graphql.database.manager.util; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.HashMultimap; import java.util.Map; +import java.util.Set; public interface BackupItem { String getTable(); - Map getItem(); + Map getItem(); - HashMultimap getLinks(); + Map> getLinks(); String getId(); diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/HistoryBackupItem.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/HistoryBackupItem.java new file mode 100644 index 00000000..c61401ec --- /dev/null +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/HistoryBackupItem.java @@ -0,0 +1,15 @@ +package com.fleetpin.graphql.database.manager.util; + +import java.nio.ByteBuffer; + +public interface HistoryBackupItem extends BackupItem { + String getOrganisationIdType(); + + byte[] getIdRevision(); + + byte[] getIdDate(); + + byte[] getStartsWithUpdatedAt(); + + Long getUpdatedAt(); +} diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/TableCoreUtil.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/TableCoreUtil.java index 27bdcbe3..cde76597 100644 --- a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/TableCoreUtil.java +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/util/TableCoreUtil.java @@ -10,17 +10,26 @@ public final class TableCoreUtil { public static String table(Class type) { + var tmp = baseClass(type); + var name = tmp.getDeclaredAnnotation(TableName.class); + if (name == null) { + return type.getSimpleName().toLowerCase() + "s"; + } else { + return name.value(); + } + } + + public static Class baseClass(Class type) { Class tmp = type; TableName name = null; while (name == null && tmp != null) { name = tmp.getDeclaredAnnotation(TableName.class); + if (name != null) { + return (Class) tmp; + } tmp = tmp.getSuperclass(); } - if (name == null) { - return type.getSimpleName().toLowerCase() + "s"; - } else { - return name.value(); - } + return type; } public static CompletableFuture> all(List> collect) { diff --git a/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/virtual/VirtualDataRunner.java b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/virtual/VirtualDataRunner.java new file mode 100644 index 00000000..29cdc20e --- /dev/null +++ b/graphql-database-manager-core/src/main/java/com/fleetpin/graphql/database/manager/virtual/VirtualDataRunner.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.fleetpin.graphql.database.manager.virtual; + +import com.fleetpin.graphql.builder.DataFetcherRunner; +import com.fleetpin.graphql.builder.annotations.Context; +import com.fleetpin.graphql.database.manager.Database; +import com.fleetpin.graphql.database.manager.VirtualDatabase; +import graphql.schema.DataFetcher; +import java.lang.reflect.Method; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +public class VirtualDataRunner implements DataFetcherRunner { + + @Override + public DataFetcher manage(Method method, DataFetcher fetcher) { + for (var parameter : method.getParameterTypes()) { + if (parameter.isAssignableFrom(VirtualDatabase.class) || hasContext(parameter)) { + var isCompletableFuture = CompletionStage.class.isAssignableFrom(method.getReturnType()); + return env -> { + var result = CompletableFuture.supplyAsync( + () -> { + try { + return fetcher.get(env); + } catch (Exception e) { + if (e instanceof RuntimeException runtime) { + throw runtime; + } + throw new RuntimeException(e); + } + }, + Database.VIRTUAL_THREAD_POOL + ); + + if (isCompletableFuture) { + return result.thenCompose(r -> (CompletionStage) r); + } + + return result; + }; + } + } + + return fetcher; + } + + private boolean hasContext(Class parameter) { + if (parameter.isAnnotationPresent(Context.class)) { + return true; + } + + for (var inter : parameter.getInterfaces()) { + if (hasContext(inter)) { + return true; + } + } + var parent = parameter.getSuperclass(); + if (parent != null && hasContext(parent)) { + return true; + } + return false; + } +} diff --git a/graphql-database-manager-core/src/test/java/com/fleetpin/graphql/database/manager/virtual/VirtualDataRunnerTest.java b/graphql-database-manager-core/src/test/java/com/fleetpin/graphql/database/manager/virtual/VirtualDataRunnerTest.java new file mode 100644 index 00000000..6f1f6293 --- /dev/null +++ b/graphql-database-manager-core/src/test/java/com/fleetpin/graphql/database/manager/virtual/VirtualDataRunnerTest.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.fleetpin.graphql.database.manager.virtual; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fleetpin.graphql.builder.annotations.Context; +import com.fleetpin.graphql.database.manager.VirtualDatabase; +import graphql.schema.DataFetcher; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("unchecked") +public class VirtualDataRunnerTest { + + @Test + public void testNormalMethod() throws Exception { + var runner = new VirtualDataRunner(); + + DataFetcher fetcher = g -> "test"; + + var replacement = runner.manage(String.class.getMethod("toLowerCase"), fetcher); + + // no context or database so returns itself + assertEquals(fetcher, replacement); + + assertEquals("test", replacement.get(null)); + } + + @Test + public void testContext() throws Exception { + var runner = new VirtualDataRunner(); + + DataFetcher fetcher = g -> "test"; + + var replacement = runner.manage(VirtualDataRunnerTest.class.getMethod("needsContext", TestContext.class), fetcher); + + assertNotEquals(fetcher, replacement); + + CompletableFuture future = (CompletableFuture) replacement.get(null); + + assertEquals("test", future.join()); + } + + @Test + public void testDatabase() throws Exception { + var runner = new VirtualDataRunner(); + + DataFetcher fetcher = g -> "test"; + + var replacement = runner.manage(VirtualDataRunnerTest.class.getMethod("needsDb", VirtualDatabase.class), fetcher); + + assertNotEquals(fetcher, replacement); + + CompletableFuture future = (CompletableFuture) replacement.get(null); + + assertEquals("test", future.join()); + } + + @Test + public void testException() throws Exception { + var runner = new VirtualDataRunner(); + + var exception = new RuntimeException("failed"); + DataFetcher fetcher = g -> { + throw exception; + }; + + var replacement = runner.manage(VirtualDataRunnerTest.class.getMethod("needsDb", VirtualDatabase.class), fetcher); + + assertNotEquals(fetcher, replacement); + + CompletableFuture future = (CompletableFuture) replacement.get(null); + + var got = assertThrows(CompletionException.class, future::join); + assertEquals(exception, got.getCause()); + } + + @Context + record TestContext() {} + + public static String needsContext(TestContext context) { + return ""; + } + + public static String needsDb(VirtualDatabase db) { + return ""; + } +} diff --git a/graphql-database-manager-dynamo/pom.xml b/graphql-database-manager-dynamo/pom.xml index a4f7a6f3..aca71a3b 100644 --- a/graphql-database-manager-dynamo/pom.xml +++ b/graphql-database-manager-dynamo/pom.xml @@ -5,14 +5,14 @@ com.fleetpin graphql-database-manager - 0.2.30-SNAPSHOT + 3.0.4-SNAPSHOT graphql-database-manager-dynamo - 5.6.0 + 5.11.0 UTF-8 @@ -30,9 +30,9 @@ - https://github.com/fleetpin/graphql-dynamodb-manager - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git + https://github.com/ashley-taylor/graphql-dynamodb-manager + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git HEAD @@ -60,6 +60,16 @@ + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.google.guava + guava + 33.3.1-jre + software.amazon.awssdk dynamodb @@ -68,7 +78,7 @@ com.fleetpin graphql-builder - 0.1.5 + 3.0.2 provided @@ -90,7 +100,7 @@ org.apache.maven.plugins maven-dependency-plugin - 2.10 + 3.8.0 copy @@ -121,7 +131,7 @@ attach-sources - jar + jar-no-fork @@ -129,7 +139,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.1.1 + 3.10.0 attach-javadocs @@ -142,7 +152,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + 3.2.6 sign-artifacts @@ -156,7 +166,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.7.0 true sonatype diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoBackupItem.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoBackupItem.java index 12e6e6ac..787cdc7d 100644 --- a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoBackupItem.java +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoBackupItem.java @@ -12,24 +12,23 @@ package com.fleetpin.graphql.database.manager.dynamo; -import static com.fleetpin.graphql.database.manager.util.TableCoreUtil.table; - import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.databind.ObjectMapper; import com.fleetpin.graphql.database.manager.util.BackupItem; -import com.google.common.collect.HashMultimap; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; public class DynamoBackupItem implements Comparable, BackupItem { private String table; - private Map item; + private Map item; private String id; - private HashMultimap links; + private Map> links; private String organisationId; private boolean hashed; private String parallelHash; @@ -37,19 +36,17 @@ public class DynamoBackupItem implements Comparable, BackupIte public DynamoBackupItem() {} public DynamoBackupItem(String table, Map item, ObjectMapper objectMapper) { - objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - this.table = table; - this.item = (Map) TableUtil.convertTo(objectMapper, item, Map.class); + this.item = (Map) TableUtil.convertTo(objectMapper, item, Map.class); - this.links = HashMultimap.create(); + this.links = new HashMap<>(); var links = item.get("links"); if (links != null) { links .m() .forEach((t, value) -> { - this.links.putAll(t, value.ss()); + this.links.put(t, new HashSet<>(value.ss())); }); } this.id = item.get("id").s(); @@ -71,12 +68,12 @@ public String getTable() { return table; } - public Map getItem() { + public Map getItem() { return item; } @JsonIgnore - public HashMultimap getLinks() { + public Map> getLinks() { return links; } diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoDb.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoDb.java index 3b030461..80fe937d 100644 --- a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoDb.java +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoDb.java @@ -24,21 +24,25 @@ import com.fleetpin.graphql.database.manager.Query; import com.fleetpin.graphql.database.manager.QueryBuilder; import com.fleetpin.graphql.database.manager.RevisionMismatchException; +import com.fleetpin.graphql.database.manager.ScanResult; +import com.fleetpin.graphql.database.manager.ScanResult.Item; import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.TableDataLoader; +import com.fleetpin.graphql.database.manager.TableScanQuery; import com.fleetpin.graphql.database.manager.annotations.Hash; import com.fleetpin.graphql.database.manager.annotations.Hash.HashExtractor; import com.fleetpin.graphql.database.manager.annotations.HashLocator; import com.fleetpin.graphql.database.manager.annotations.HashLocator.HashQueryBuilder; import com.fleetpin.graphql.database.manager.util.BackupItem; import com.fleetpin.graphql.database.manager.util.CompletableFutureUtil; +import com.fleetpin.graphql.database.manager.util.HistoryBackupItem; import com.fleetpin.graphql.database.manager.util.HistoryCoreUtil; import com.fleetpin.graphql.database.manager.util.TableCoreUtil; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Throwables; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Lists; import com.google.common.hash.Hashing; +import graphql.VisibleForTesting; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; @@ -54,6 +58,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -70,6 +76,7 @@ import software.amazon.awssdk.services.dynamodb.model.QueryRequest; import software.amazon.awssdk.services.dynamodb.model.QueryRequest.Builder; import software.amazon.awssdk.services.dynamodb.model.ReturnValue; +import software.amazon.awssdk.services.dynamodb.model.ScanRequest; import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest; import software.amazon.awssdk.services.dynamodb.model.WriteRequest; @@ -77,9 +84,9 @@ public class DynamoDb extends DatabaseDriver { private static final AttributeValue REVISION_INCREMENT = AttributeValue.builder().n("1").build(); private static final int BATCH_WRITE_SIZE = 25; - private static final int MAX_RETRY = 10; + private static final int MAX_RETRY = 20; - private final List entityTables; //is in reverse order so easy to over ride as we go through + private final List entityTables; // is in reverse order so easy to override as we go through private final String historyTable; private final String entityTable; private final DynamoDbAsyncClient client; @@ -89,13 +96,33 @@ public class DynamoDb extends DatabaseDriver { private final int maxRetry; private final boolean globalEnabled; private final boolean hash; + private final String classPath; + + private final String parallelHashIndex; private final ConcurrentHashMap, Optional> extractorCache = new ConcurrentHashMap<>(); + private enum BackupTableType { + Entity, + History, + } + private final Map hashKeyExpander; + private final Map> classes; - public DynamoDb(ObjectMapper mapper, List entityTables, DynamoDbAsyncClient client, Supplier idGenerator) { - this(mapper, entityTables, null, client, idGenerator, BATCH_WRITE_SIZE, MAX_RETRY, true, true, null); + public DynamoDb(ObjectMapper mapper, List entityTables, List historyTables, DynamoDbAsyncClient client, Supplier idGenerator) { + this(mapper, entityTables, null, client, idGenerator, BATCH_WRITE_SIZE, MAX_RETRY, true, true, null, null); + } + + public DynamoDb( + ObjectMapper mapper, + List entityTables, + List historyTables, + DynamoDbAsyncClient client, + Supplier idGenerator, + String parallelHashIndex + ) { + this(mapper, entityTables, null, client, idGenerator, BATCH_WRITE_SIZE, MAX_RETRY, true, true, null, parallelHashIndex); } public DynamoDb( @@ -108,7 +135,8 @@ public DynamoDb( int maxRetry, boolean globalEnabled, boolean hash, - String classPath + String classPath, + String parallelHashIndex ) { this.mapper = mapper; this.entityTables = entityTables; @@ -120,13 +148,17 @@ public DynamoDb( this.maxRetry = maxRetry; this.globalEnabled = globalEnabled; this.hash = hash; + this.classPath = classPath; + this.parallelHashIndex = parallelHashIndex; if (classPath != null) { var tableObjects = new Reflections(classPath).getSubTypesOf(Table.class); this.hashKeyExpander = new HashMap<>(); + this.classes = new HashMap<>(); for (var obj : tableObjects) { + classes.put(TableCoreUtil.table(obj), TableCoreUtil.baseClass(obj)); HashLocator hashLocator = null; Class tmp = obj; while (hashLocator == null && tmp != null) { @@ -147,6 +179,7 @@ public DynamoDb( } } else { hashKeyExpander = null; + classes = null; } } @@ -163,7 +196,7 @@ public CompletableFuture delete(String organisationId, T en String sourceOrganisation = getSourceOrganisationId(entity); if (!sourceOrganisation.equals(organisationId)) { - //trying to delete a global or something just return without doing anything + // trying to delete a global or something just return without doing anything return CompletableFuture.completedFuture(entity); } @@ -181,12 +214,12 @@ public CompletableFuture delete(String organisationId, T en if (!sourceOrganisationId.equals(organisationId)) { return; } - if (entity.getRevision() == 0) { //we confirm row does not exist with a revision since entry might predate feature + if (entity.getRevision() == 0) { // we confirm row does not exist with a revision since entry might predate feature mutator.conditionExpression("attribute_not_exists(revision)"); } else { Map variables = new HashMap<>(); variables.put(":revision", AttributeValue.builder().n(Long.toString(entity.getRevision())).build()); - //check exists and matches revision + // check exists and matches revision mutator.expressionAttributeValues(variables); mutator.conditionExpression("revision = :revision"); } @@ -203,7 +236,7 @@ public CompletableFuture delete(String organisationId, T en throw new RuntimeException(failure); }); } else { - //we mark as deleted not actual delete + // we mark as deleted not actual delete Map item = mapWithKeys(organisationId, entity, true); item.put("deleted", AttributeValue.builder().bool(true).build()); @@ -219,7 +252,7 @@ private CompletableFuture conditionalBulkWrite(List items) { var all = items .stream() .map(i -> - put(i.getOrganisationId(), i.getEntity(), i.getCheck()) + put(i.getOrganisationId(), i.getEntity(), i.getCheck(), true) .whenComplete((res, e) -> { if (e == null) { i.resolve(); @@ -231,7 +264,7 @@ private CompletableFuture conditionalBulkWrite(List items) { .toArray(CompletableFuture[]::new); return CompletableFuture.allOf(all); } - + private CompletableFuture nonConditionalBulkPutChunk(List items) { var writeRequests = items.stream().map(i -> buildWriteRequest(i)).collect(Collectors.toList()); var data = Map.of(entityTable, writeRequests); @@ -254,57 +287,55 @@ private CompletableFuture nonConditionalBulkPutChunk(List items) { } private CompletableFuture nonConditionalBulkWrite(List items) { - - - if(items.isEmpty()) { + if (items.isEmpty()) { return CompletableFuture.completedFuture(null); } - if(items.size() > batchWriteSize) { - + if (items.size() > batchWriteSize) { ArrayListMultimap byPartition = ArrayListMultimap.create(); - items.stream().forEach(t -> { - var key = mapWithKeys(t.getOrganisationId(), t.getEntity()); - byPartition.put(key.get("organisationId").s(), t); - }); - - var futures = byPartition.keySet().stream().map(key -> { - var partition = byPartition.get(key); - - var toReturn = new CompletableFuture(); - sendBackToBack(toReturn, Lists.partition(partition, batchWriteSize).iterator()); - return toReturn; - }).toArray(CompletableFuture[]::new); - + items + .stream() + .forEach(t -> { + var key = mapWithKeys(t.getOrganisationId(), t.getEntity()); + byPartition.put(key.get("organisationId").s(), t); + }); + + var futures = byPartition + .keySet() + .stream() + .map(key -> { + var partition = byPartition.get(key); + + var toReturn = new CompletableFuture(); + sendBackToBack(toReturn, Lists.partition(partition, batchWriteSize).iterator()); + return toReturn; + }) + .toArray(CompletableFuture[]::new); + return CompletableFuture.allOf(futures); - }else { + } else { return nonConditionalBulkPutChunk(items); } } private void sendBackToBack(CompletableFuture atEnd, Iterator> it) { - - nonConditionalBulkPutChunk(it.next()).whenComplete((response, error) -> { - - if(error != null) { - while(it.hasNext()) { - var f = it.next(); - for(var e: f) { - e.fail(error); + nonConditionalBulkPutChunk(it.next()) + .whenComplete((response, error) -> { + if (error != null) { + while (it.hasNext()) { + var f = it.next(); + for (var e : f) { + e.fail(error); + } + } + atEnd.completeExceptionally(error); + } else { + if (it.hasNext()) { + sendBackToBack(atEnd, it); + } else { + atEnd.complete(null); } } - atEnd.completeExceptionally(error); - }else { - if(it.hasNext()) { - sendBackToBack(atEnd, it); - }else { - atEnd.complete(null); - } - - - } - - }); - + }); } private CompletableFuture putItems(int count, Map> data) { @@ -337,9 +368,9 @@ public CompletableFuture bulkPut(List values) { var nonConditional = values.stream().filter(v -> !v.getCheck()).collect(Collectors.toList()); var conditionalFuture = CompletableFuture.allOf(conditional.stream().map(part -> conditionalBulkWrite(part)).toArray(CompletableFuture[]::new)); - + var nonConditionalFuture = nonConditionalBulkWrite(nonConditional); - + return CompletableFuture.allOf(conditionalFuture, nonConditionalFuture); } catch (Exception e) { for (var v : values) { @@ -349,21 +380,24 @@ public CompletableFuture bulkPut(List values) { } } - public Map buildPutEntity(String organisationId, T entity) { - if (entity.getId() == null) { - entity.setId(idGenerator.get()); - setCreatedAt(entity, Instant.now()); - } - if (entity.getCreatedAt() == null) { - setCreatedAt(entity, Instant.now()); //if missing for what ever reason + public Map buildPutEntity(String organisationId, T entity, boolean update) { + long revision = entity.getRevision(); + if (update) { + if (entity.getId() == null) { + entity.setId(idGenerator.get()); + setCreatedAt(entity, Instant.now()); + } + if (entity.getCreatedAt() == null) { + setCreatedAt(entity, Instant.now()); // if missing for what ever reason + } + setUpdatedAt(entity, Instant.now()); + revision++; } - final long revision = entity.getRevision(); - setUpdatedAt(entity, Instant.now()); Map item = mapWithKeys(organisationId, entity, true); var entries = TableUtil.toAttributes(mapper, entity); entries.remove("revision"); // needs to be at the top level as a limit on dynamo to be able to perform an atomic addition - item.put("revision", AttributeValue.builder().n(Long.toString(revision + 1)).build()); + item.put("revision", AttributeValue.builder().n(Long.toString(revision)).build()); if (HistoryCoreUtil.hasHistory(entity)) { item.put("history", AttributeValue.builder().bool(true).build()); } @@ -371,7 +405,6 @@ public Map buildPutEntity(String organ Map links = new HashMap<>(); getLinks(entity) - .asMap() .forEach((table, link) -> { if (!link.isEmpty()) { links.put(table, AttributeValue.builder().ss(link).build()); @@ -396,15 +429,15 @@ public Map buildPutEntity(String organ } public WriteRequest buildWriteRequest(PutValue value) { - var item = buildPutEntity(value.getOrganisationId(), value.getEntity()); + var item = buildPutEntity(value.getOrganisationId(), value.getEntity(), true); return WriteRequest.builder().putRequest(builder -> builder.item(item)).build(); } - private CompletableFuture put(String organisationId, T entity, boolean check) { + private CompletableFuture put(String organisationId, T entity, boolean check, boolean updateEntity) { final long revision = entity.getRevision(); String sourceTable = getSourceTable(entity); - var item = buildPutEntity(organisationId, entity); + var item = buildPutEntity(organisationId, entity, updateEntity); return client .putItem(request -> @@ -415,12 +448,14 @@ private CompletableFuture put(String organisationId, T enti if (check) { String sourceOrganisationId = getSourceOrganisationId(entity); - if (sourceTable != null && !sourceTable.equals(entityTable) || !sourceOrganisationId.equals(organisationId) || revision == 0) { //we confirm row does not exist with a revision since entry might predate feature + if (sourceTable != null && !sourceTable.equals(entityTable) || !sourceOrganisationId.equals(organisationId) || revision == 0) { // we confirm row does not exist with a + // revision since entry might predate + // feature mutator.conditionExpression("attribute_not_exists(revision)"); } else { Map variables = new HashMap<>(); variables.put(":revision", AttributeValue.builder().n(Long.toString(revision)).build()); - //check exists and matches revision + // check exists and matches revision mutator.expressionAttributeValues(variables); mutator.conditionExpression("revision = :revision"); } @@ -466,7 +501,7 @@ public CompletableFuture> get(List> key for (String table : this.entityTables) { items.put(table, KeysAndAttributes.builder().keys(entries).consistentRead(true).build()); } - return getItems(0, items, new Flattener(this.entityTables, false)) + return getItems(0, items, Flattener.create(this.entityTables, false)) .thenApply(flattener -> { var toReturn = new ArrayList(); for (var key : keys) { @@ -522,6 +557,9 @@ public CompletableFuture> getViaLinks( String tableTarget = table(type); var links = getLinks(entry).get(tableTarget); + if (links == null) { + links = Collections.emptySet(); + } Class query = (Class
) type; List> keys = links.stream().map(link -> createDatabaseKey(organisationId, query, link)).collect(Collectors.toList()); return items.loadMany(keys); @@ -545,7 +583,7 @@ public CompletableFuture> query(DatabaseQueryKey ke var future = CompletableFutureUtil.sequence(futures); return future.thenApply(results -> { - var flattener = new Flattener(this.entityTables, false); + var flattener = Flattener.create(this.entityTables, false); results.forEach(list -> flattener.addItems(list)); return flattener.results(mapper, key.getQuery().getType(), Optional.ofNullable(key.getQuery().getLimit())); @@ -569,13 +607,22 @@ public CompletableFuture> queryHistory(DatabaseQueryHi builder = queryHistoryWithStarts(key, builder, organisationIdType); } - var toReturn = new ArrayList(); + var targetException = new AtomicReference(); + + List toReturn = new ArrayList(); return client .queryPaginator(builder.build()) .subscribe(response -> { - response.items().forEach(item -> toReturn.add(new DynamoItem(historyTable, item).convertTo(mapper, queryHistory.getType()))); + try { + response.items().forEach(item -> toReturn.add(new DynamoItem(historyTable, item).convertTo(mapper, queryHistory.getType()))); + } catch (Exception e) { + targetException.set(e); + } }) .thenApply(__ -> { + if (targetException.get() != null) { + throw new RuntimeException(targetException.get()); + } return toReturn; }); } @@ -703,7 +750,7 @@ public CompletableFuture> queryGlobal(Class type, S ); } return future.thenApply(results -> { - var flattener = new Flattener(this.entityTables, true); + var flattener = Flattener.create(this.entityTables, true); results.forEach(list -> flattener.addItems(list)); return flattener.results(mapper, type); }); @@ -786,7 +833,7 @@ private CompletableFuture> querySecondary(String table, AttributeVa .stream() .map(item -> item.get("id").s()) .map(itemId -> { - return itemId.substring(itemId.indexOf(':') + 1); //Id contains entity name + return itemId.substring(itemId.indexOf(':') + 1); // Id contains entity name }) .forEach(toReturn::add); }) @@ -801,30 +848,55 @@ private CompletableFuture> query(String organisationId, String var id = keys.get("id"); Map keyConditions = new HashMap<>(); keyConditions.put(":organisationId", organisationIdAttribute); - if (id != null && !id.s().isEmpty()) { - keyConditions.put(":table", id); + + String index = null; + boolean consistentRead = true; + boolean parallelRequest; + + if (query.getThreadIndex() != null && query.getThreadCount() != null) { + consistentRead = false; + parallelRequest = true; + index = this.parallelHashIndex; + keyConditions.put(":hash", AttributeValue.builder().s(toPaddedBinary(query.getThreadIndex(), query.getThreadCount())).build()); + } else { + parallelRequest = false; + if (id != null && !id.s().trim().isEmpty()) { + index = null; + keyConditions.put(":table", id); + } } var s = new DynamoQuerySubscriber(table, query.getLimit()); + Boolean finalConsistentRead = consistentRead; + String finalIndex = index; client .queryPaginator(r -> { r .tableName(table) - .consistentRead(true) + .indexName(finalIndex) + .consistentRead(finalConsistentRead) .expressionAttributeValues(keyConditions) .applyMutation(b -> { - if (id == null || id.s().isEmpty()) { - b.keyConditionExpression("organisationId = :organisationId"); - } else { - b.keyConditionExpression("organisationId = :organisationId AND begins_with(id, :table)"); + var conditionalExpression = "organisationId = :organisationId"; + + if (keyConditions.containsKey(":table")) { + conditionalExpression += " AND begins_with(id, :table)"; + } else if (keyConditions.containsKey(":hash")) { + conditionalExpression += " AND begins_with(parallelHash, :hash)"; } + b.keyConditionExpression(conditionalExpression); + if (query.getLimit() != null) { b.limit(query.getLimit()); } if (query.getAfter() != null) { - b.exclusiveStartKey(mapWithKeys(organisationId, query.getType(), query.getAfter())); + var start = mapWithKeys(organisationId, query.getType(), query.getAfter()); + if (parallelRequest) { + start.put("parallelHash", AttributeValue.builder().s(parallelHash(query.getAfter())).build()); + } + b.exclusiveStartKey(start); } }); }) @@ -835,28 +907,39 @@ private CompletableFuture> query(String organisationId, String @Override public CompletableFuture restoreBackup(List entities) { + return restoreBackup(entities, BackupTableType.Entity, TableUtil::toAttributes); + } + + @Override + public CompletableFuture restoreHistoryBackup(List entities) { + return restoreBackup(entities, BackupTableType.History, HistoryUtil::toAttributes); + } + + private CompletableFuture restoreBackup( + List entities, + BackupTableType backupTableType, + BiFunction> toAttributes + ) { List> completableFutures = Lists .partition( entities .stream() .map(item -> { - return WriteRequest.builder().putRequest(builder -> builder.item(TableUtil.toAttributes(mapper, item)).build()).build(); + return WriteRequest.builder().putRequest(builder -> builder.item(toAttributes.apply(mapper, item)).build()).build(); }) .collect(Collectors.toList()), BATCH_WRITE_SIZE ) .stream() .map(putRequestBatch -> { - final var batchPutRequest = BatchWriteItemRequest.builder().requestItems(Map.of(entityTable, putRequestBatch)).build(); + final var batchPutRequest = BatchWriteItemRequest + .builder() + .requestItems(Map.of(backupTableType == BackupTableType.History ? historyTable : entityTable, putRequestBatch)) + .build(); return client .batchWriteItem(batchPutRequest) - .thenApply(items -> { - do { - client.batchWriteItem(BatchWriteItemRequest.builder().requestItems(items.unprocessedItems()).build()); - } while (items.unprocessedItems().size() > 0); - return items; - }) + .thenCompose(this::batchWriteRetry) .exceptionally(failure -> { if (failure.getCause() instanceof ConditionalCheckFailedException) { throw new RevisionMismatchException(failure.getCause()); @@ -870,6 +953,14 @@ public CompletableFuture restoreBackup(List entities) { return CompletableFuture.allOf(completableFutures.toArray(new CompletableFuture[completableFutures.size()])); } + private CompletableFuture batchWriteRetry(BatchWriteItemResponse items) { + if (items.unprocessedItems().size() > 0) { + return client.batchWriteItem(BatchWriteItemRequest.builder().requestItems(items.unprocessedItems()).build()).thenCompose(this::batchWriteRetry); + } else { + return CompletableFuture.completedFuture(items); + } + } + @Override public CompletableFuture> takeBackup(String organisationId) { CompletableFuture>> future = CompletableFuture.completedFuture(new ArrayList<>()); @@ -890,6 +981,45 @@ public CompletableFuture> takeBackup(String organisationId) { }); } + @Override + public CompletableFuture> takeHistoryBackup(String organisationId) { + if (this.classPath == null) { + throw new RuntimeException("classPath is required to obtain the tables' name to query the history table"); + } + + // Using reflections to loop through tables and get the tables' name + Set> tableObjects = new Reflections(classPath).getSubTypesOf(Table.class); + List toReturn = Collections.synchronizedList(new ArrayList()); + + Set orgIdTypes = tableObjects.stream().map(obj -> organisationId + ":" + TableCoreUtil.table(obj)).collect(Collectors.toSet()); + + var queries = orgIdTypes + .stream() + .map(orgIdType -> { + Map keyConditions = new HashMap<>(); + AttributeValue orgIdTypeAttr = AttributeValue.builder().s(orgIdType).build(); + keyConditions.put(":organisationIdType", orgIdTypeAttr); + return client + .queryPaginator(r -> + r + .tableName(historyTable) + .consistentRead(true) + .keyConditionExpression("organisationIdType = :organisationIdType") + .expressionAttributeValues(keyConditions) + ) + .subscribe(response -> { + response + .items() + .forEach(item -> { + toReturn.add(new DynamoHistoryBackupItem(historyTable, item, mapper)); + }); + }); + }) + .toArray(CompletableFuture[]::new); + + return CompletableFuture.allOf(queries).thenApply(__ -> toReturn); + } + private CompletableFuture> takeBackup(String table, AttributeValue organisationId) { if (hash && hashKeyExpander == null) { throw new UnsupportedOperationException("To perform backups on hashed databases must specify hashLocators"); @@ -1025,7 +1155,7 @@ private CompletableFuture addLinks(String organisationId, Class a) - .handle((r, e) -> { //nasty if attribute now exists use first approach again... + .handle((r, e) -> { // nasty if attribute now exists use first approach again... if (e != null) { if (e.getCause() instanceof ConditionalCheckFailedException) { return client.updateItem(request -> @@ -1075,8 +1205,8 @@ private CompletableFuture updateEntityLinks( String sourceTable = getSourceTable(entity); String sourceOrganisationId = getSourceOrganisationId(entity); - //revision checks don't really work when reading from one env and writing to another, or read from global write to organisation. - //revision would only practically be empty if reading object before revision concept is present + // revision checks don't really work when reading from one env and writing to another, or read from global write to organisation. + // revision would only practically be empty if reading object before revision concept is present if (sourceTable.equals(entityTable) && sourceOrganisationId.equals(organisationId) && entity.getRevision() != 0) { values.put(":revision", AttributeValue.builder().n(Long.toString(entity.getRevision())).build()); extraConditions = " AND revision = :revision"; @@ -1164,6 +1294,9 @@ public CompletableFuture link(String organisationId, T enti var existing = getLinks(entity).get(target); var toAdd = new HashSet<>(groupIds); + if (existing == null) { + existing = Collections.emptySet(); + } toAdd.removeAll(existing); var toRemove = new HashSet<>(existing); @@ -1172,11 +1305,11 @@ public CompletableFuture link(String organisationId, T enti var entityFuture = updateEntityLinks(organisationId, entity, class1, groupIds); return entityFuture.thenCompose(e -> { - //wait until the entity has been updated in-case that fails then update the other targets. + // wait until the entity has been updated in-case that fails then update the other targets. - //remove links that have been removed + // remove links that have been removed CompletableFuture removeFuture = removeLinks(organisationId, class1, toRemove, source, entity.getId()); - //add the new links + // add the new links CompletableFuture addFuture = addLinks(organisationId, class1, toAdd, source, entity.getId()); return CompletableFuture @@ -1214,7 +1347,10 @@ public CompletableFuture unlink( return client.updateItem(updateTargetLinksRequest); }) .thenApply(ignore -> { - getLinks(entity).remove(table(clazz), targetId); + var links = getLinks(entity).get(table(clazz)); + if (links != null) { + links.remove(targetId); + } return entity; }); @@ -1230,7 +1366,6 @@ private UpdateItemRequest createRemoveLinkRequest( .builder() .m( getLinks(entity) - .asMap() .entrySet() .stream() .filter(entry -> !entry.getValue().contains(targetId) && !entry.getKey().equals(table(clazz))) @@ -1260,7 +1395,7 @@ public CompletableFuture deleteLinks(String organisationId, var organisationIdAttribute = AttributeValue.builder().s(organisationId).build(); var id = AttributeValue.builder().s(table(entity.getClass()) + ":" + entity.getId()).build(); - //we first clear out our own object + // we first clear out our own object long revision = entity.getRevision(); Map values = new HashMap<>(); @@ -1280,7 +1415,7 @@ public CompletableFuture deleteLinks(String organisationId, .returnValues(ReturnValue.UPDATED_NEW) .applyMutation(mutator -> { String sourceTable = getSourceTable(entity); - //revision checks don't really work when reading from one env and writing to another. + // revision checks don't really work when reading from one env and writing to another. if (sourceTable != null && !sourceTable.equals(entityTable)) { return; } @@ -1288,7 +1423,7 @@ public CompletableFuture deleteLinks(String organisationId, if (!sourceOrganisationId.equals(organisationId)) { return; } - if (revision == 0) { //we confirm row does not exist with a revision since entry might predate feature + if (revision == 0) { // we confirm row does not exist with a revision since entry might predate feature mutator.conditionExpression("attribute_not_exists(revision)"); } else { values.put(":revision", AttributeValue.builder().n(Long.toString(entity.getRevision())).build()); @@ -1309,35 +1444,37 @@ public CompletableFuture deleteLinks(String organisationId, throw new RuntimeException(failure); }); - //after we successfully clear out our object we clear the remote references + // after we successfully clear out our object we clear the remote references return clearEntity.thenCompose(r -> { - CompletableFuture future = CompletableFuture.completedFuture(null); - var val = AttributeValue.builder().ss(entity.getId()).build(); String source = table(entity.getClass()); - for (var link : getLinks(entity).entries()) { - var targetIdAttribute = AttributeValue.builder().s(link.getKey() + ":" + link.getValue()).build(); - Map targetKey = new HashMap<>(); - targetKey.put("organisationId", organisationIdAttribute); - targetKey.put("id", targetIdAttribute); + CompletableFuture future = getLinks(entity) + .entrySet() + .stream() + .flatMap(s -> s.getValue().stream().map(v -> Map.entry(s.getKey(), v))) + .map(link -> { + var targetIdAttribute = AttributeValue.builder().s(link.getKey() + ":" + link.getValue()).build(); + Map targetKey = new HashMap<>(); + targetKey.put("organisationId", organisationIdAttribute); + targetKey.put("id", targetIdAttribute); - Map v = new HashMap<>(); - v.put(":val", val); - v.put(":revisionIncrement", REVISION_INCREMENT); + Map v = new HashMap<>(); + v.put(":val", val); + v.put(":revisionIncrement", REVISION_INCREMENT); - Map k = new HashMap<>(); - k.put("#table", source); + Map k = new HashMap<>(); + k.put("#table", source); - var destination = client.updateItem(request -> - request - .tableName(entityTable) - .key(targetKey) - .updateExpression("DELETE links.#table :val ADD revision :revisionIncrement") - .expressionAttributeNames(k) - .expressionAttributeValues(v) - ); - future = future.thenCombine(destination, (a, b) -> b); - } + return client.updateItem(request -> + request + .tableName(entityTable) + .key(targetKey) + .updateExpression("DELETE links.#table :val ADD revision :revisionIncrement") + .expressionAttributeNames(k) + .expressionAttributeValues(v) + ); + }) + .reduce(CompletableFuture.completedFuture(null), (a, b) -> a.thenCombine(b, (c, d) -> d)); getLinks(entity).clear(); return future.thenApply(__ -> r); }); @@ -1506,6 +1643,15 @@ private Map mapWithKeys(String organis return item; } + public static String toPaddedBinary(int number, int powerOfTwo) { + var paddingLength = (int) ((Math.log(powerOfTwo) / Math.log(2))); + StringBuilder binaryString = new StringBuilder(Integer.toBinaryString(number)); + while (binaryString.length() < paddingLength) { + binaryString.insert(0, "0"); + } + return binaryString.toString(); + } + @VisibleForTesting protected static String parallelHash(String id) { String empty = "00000000"; @@ -1514,4 +1660,68 @@ protected static String parallelHash(String id) { var toReturn = Integer.toBinaryString(bits); return empty.substring(0, empty.length() - toReturn.length()) + toReturn; } + + @Override + protected ScanResult startTableScan(TableScanQuery tableScanQuery, int segment, Object from) { + return startTableScan(tableScanQuery, segment, from, entityTable); + } + + private ScanResult startTableScan(TableScanQuery tableScanQuery, int segment, Object from, String table) { + var builder = ScanRequest.builder().tableName(table).totalSegments(tableScanQuery.parallelism()).segment(segment); + + if (from != null) { + var startKey = TableUtil.toAttributes(mapper, from); + builder.exclusiveStartKey(startKey); + } + + var scan = client.scan(b -> b.tableName(entityTables.getLast()).totalSegments(tableScanQuery.parallelism()).segment(segment)).join(); + + var items = new ArrayList>(); + for (var item : scan.items()) { + if (!item.containsKey("id") || !item.containsKey("organisationId") || !item.containsKey("item")) { + continue; + } + var id = item.get("id").s(); + var organisationId = item.get("organisationId").s(); + var innerItem = item.get("item").m(); + if (innerItem == null || !innerItem.containsKey("id")) { + continue; + } + var fullId = innerItem.get("id").s(); + + String typeId; + if (id.endsWith(fullId)) { + // not hashed + typeId = id.substring(0, id.indexOf(":")); + } else { + //hashed + var parts = organisationId.split(":"); + typeId = parts[1]; + organisationId = parts[0]; + } + var type = this.classes.get(typeId); + if (type != null) { + var entity = new DynamoItem(table, item).convertTo(mapper, type); + + var orgIdFinal = organisationId; + items.add( + new Item
( + organisationId, + entity, + replacement -> put(orgIdFinal, replacement, true, false).join(), + delete -> { + delete(orgIdFinal, delete); + } + ) + ); + } + } + + Object next = null; + if (!scan.lastEvaluatedKey().isEmpty()) { + next = TableUtil.convertTo(mapper, scan.lastEvaluatedKey(), Object.class); + } + + return new ScanResult(items, next); + } } diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoDbManager.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoDbManager.java index 913c152b..a2fef8bd 100644 --- a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoDbManager.java +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoDbManager.java @@ -12,14 +12,7 @@ package com.fleetpin.graphql.database.manager.dynamo; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; import com.fleetpin.graphql.database.manager.DatabaseDriver; import com.fleetpin.graphql.database.manager.DatabaseManager; import com.google.common.base.Preconditions; @@ -64,6 +57,8 @@ public static class DyanmoDbManagerBuilder { private boolean hash = false; private String classPath = null; + private String parallelIndex = null; + public DyanmoDbManagerBuilder dynamoDbAsyncClient(DynamoDbAsyncClient client) { this.client = client; return this; @@ -129,23 +124,16 @@ public DyanmoDbManagerBuilder classPath(String classPath) { return this; } + public DyanmoDbManagerBuilder parallelIndex(String parallelIndex) { + this.parallelIndex = parallelIndex; + return this; + } + public DynamoDbManager build() { Preconditions.checkNotNull(tables, "Tables must be set"); Preconditions.checkArgument(!tables.isEmpty(), "Empty table array"); + Preconditions.checkNotNull(mapper, "Mapper is null"); - if (mapper == null) { - mapper = - new ObjectMapper() - .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - .registerModule(new ParameterNamesModule()) - .registerModule(new Jdk8Module()) - .registerModule(new JavaTimeModule()) - .disable(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS) - .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) - .disable(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) - .disable(SerializationFeature.WRITE_DURATIONS_AS_TIMESTAMPS) - .setVisibility(PropertyAccessor.FIELD, Visibility.ANY); - } if (client == null) { client = DynamoDbAsyncClient.create(); } @@ -156,7 +144,7 @@ public DynamoDbManager build() { database = Objects.requireNonNullElse( database, - new DynamoDb(mapper, tables, historyTable, client, idGenerator, batchWriteSize, maxRetry, globalEnabled, hash, classPath) + new DynamoDb(mapper, tables, historyTable, client, idGenerator, batchWriteSize, maxRetry, globalEnabled, hash, classPath, parallelIndex) ); return new DynamoDbManager(mapper, idGenerator, client, database); diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoHistoryBackupItem.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoHistoryBackupItem.java new file mode 100644 index 00000000..9c86189d --- /dev/null +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoHistoryBackupItem.java @@ -0,0 +1,59 @@ +package com.fleetpin.graphql.database.manager.dynamo; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fleetpin.graphql.database.manager.util.HistoryBackupItem; +import java.util.Map; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public class DynamoHistoryBackupItem extends DynamoBackupItem implements HistoryBackupItem { + + private String organisationIdType; + + private byte[] idRevision; + + private byte[] idDate; + + private byte[] startsWithUpdatedAt; + + private Long updatedAt; + + public DynamoHistoryBackupItem() {} + + public DynamoHistoryBackupItem(String table, Map item, ObjectMapper objectMapper) { + super(table, item, objectMapper); + this.organisationIdType = item.get("organisationIdType").s(); + + this.idRevision = item.get("idRevision").b().asByteArray(); + + this.idDate = item.get("idDate").b().asByteArray(); + + this.startsWithUpdatedAt = item.get("startsWithUpdatedAt").b().asByteArray(); + + this.updatedAt = Long.parseLong(item.get("updatedAt").n()); + } + + @Override + public String getOrganisationIdType() { + return this.organisationIdType; + } + + @Override + public byte[] getIdRevision() { + return this.idRevision; + } + + @Override + public byte[] getIdDate() { + return this.idDate; + } + + @Override + public byte[] getStartsWithUpdatedAt() { + return this.startsWithUpdatedAt; + } + + @Override + public Long getUpdatedAt() { + return this.updatedAt; + } +} diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoItem.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoItem.java index 8a4fb131..3e365dab 100644 --- a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoItem.java +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/DynamoItem.java @@ -15,9 +15,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.TableAccess; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; public class DynamoItem implements Comparable { @@ -26,21 +27,21 @@ public class DynamoItem implements Comparable { private final Map item; private final String id; - private final HashMultimap links; + private final Map> links; private String organisationId; - DynamoItem(String table, Map item) { + public DynamoItem(String table, Map item) { this.table = table; this.item = item; - this.links = HashMultimap.create(); + this.links = new HashMap<>(); var links = item.get("links"); if (links != null) { links .m() .forEach((t, value) -> { - this.links.putAll(t, value.ss()); + this.links.computeIfAbsent(t, __ -> new HashSet<>()).addAll(value.ss()); }); } var id = item.get("id").s(); @@ -91,7 +92,7 @@ Map getItem() { return item; } - public Multimap getLinks() { + public Map> getLinks() { return links; } diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/Flattener.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/Flattener.java index e1f383c3..565fc4c8 100644 --- a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/Flattener.java +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/Flattener.java @@ -20,83 +20,35 @@ import java.util.stream.Collectors; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -public final class Flattener { +public abstract class Flattener { - private final Map lookup; - private final boolean includeOrganisationId; - private final List tables; - - Flattener(List tables, boolean includeOrganisationId) { - this.tables = tables; - lookup = new HashMap<>(); - this.includeOrganisationId = includeOrganisationId; - } - - private String getId(DynamoItem item) { - if (includeOrganisationId) { - return item.getOrganisationId() + ":" + item.getId(); - } else { - return item.getId(); + public static Flattener create(List entityTables, boolean b) { + if (entityTables.size() > 1) { + return new FlattenerMulti(entityTables, b); } + return new FlattenerSingle(b); } - public DynamoItem get(Optional extractor, Class type, String id) { - String key; - if (extractor.isPresent()) { - key = TableCoreUtil.table(type) + ":" + extractor.get().hashId(id) + "\t" + extractor.get().sortId(id); - } else { - key = TableCoreUtil.table(type) + ":" + id; - } - var got = this.lookup.get(key); - if (got != null && got.isDeleted()) { - return null; - } else { - return got; - } - } + public abstract DynamoItem get(Optional extractor, Class type, String id); - public void addItems(List list) { + protected abstract void addItem(DynamoItem item); + + public final void addItems(List list) { list.forEach(item -> { addItem(item); }); } - public void add(String table, List> list) { + public final void add(String table, List> list) { list.forEach(item -> { var i = new DynamoItem(table, item); addItem(i); }); } - private void addItem(DynamoItem item) { - lookup.merge(getId(item), item, this::merge); - } - - private DynamoItem merge(DynamoItem existing, DynamoItem replace) { - if (tables.indexOf(existing.getTable()) > tables.indexOf(replace.getTable())) { - var tmp = existing; - existing = replace; - replace = tmp; - } - - var item = new HashMap<>(replace.getItem()); - //only links in parent - if (item.get("item") == null) { - item.put("item", existing.getItem().get("item")); - } - var toReturn = new DynamoItem(replace.getTable(), item); - toReturn.getLinks().putAll(existing.getLinks()); - - return toReturn; - } - - public List results(ObjectMapper mapper, Class type) { + public final List results(ObjectMapper mapper, Class type) { return results(mapper, type, Optional.empty()); } - public List results(ObjectMapper mapper, Class type, Optional limit) { - var items = new ArrayList(lookup.values()); - Collections.sort(items); - return items.stream().limit(limit.orElse(Integer.MAX_VALUE)).map(t -> t.convertTo(mapper, type)).collect(Collectors.toList()); - } + public abstract List results(ObjectMapper mapper, Class type, Optional limit); } diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/FlattenerMulti.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/FlattenerMulti.java new file mode 100644 index 00000000..f43ec4f8 --- /dev/null +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/FlattenerMulti.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.fleetpin.graphql.database.manager.dynamo; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fleetpin.graphql.database.manager.Table; +import com.fleetpin.graphql.database.manager.annotations.Hash; +import com.fleetpin.graphql.database.manager.util.TableCoreUtil; +import java.util.*; +import java.util.stream.Collectors; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public final class FlattenerMulti extends Flattener { + + private final Map lookup; + private final boolean includeOrganisationId; + private final List tables; + + FlattenerMulti(List tables, boolean includeOrganisationId) { + this.tables = tables; + lookup = new HashMap<>(); + this.includeOrganisationId = includeOrganisationId; + } + + private String getId(DynamoItem item) { + if (includeOrganisationId) { + return item.getOrganisationId() + ":" + item.getId(); + } else { + return item.getId(); + } + } + + @Override + public DynamoItem get(Optional extractor, Class type, String id) { + String key; + if (extractor.isPresent()) { + key = TableCoreUtil.table(type) + ":" + extractor.get().hashId(id) + "\t" + extractor.get().sortId(id); + } else { + key = TableCoreUtil.table(type) + ":" + id; + } + var got = this.lookup.get(key); + if (got != null && got.isDeleted()) { + return null; + } else { + return got; + } + } + + protected void addItem(DynamoItem item) { + lookup.merge(getId(item), item, this::merge); + } + + private DynamoItem merge(DynamoItem existing, DynamoItem replace) { + if (tables.indexOf(existing.getTable()) > tables.indexOf(replace.getTable())) { + var tmp = existing; + existing = replace; + replace = tmp; + } + + var item = new HashMap<>(replace.getItem()); + //only links in parent + if (item.get("item") == null) { + item.put("item", existing.getItem().get("item")); + } + var toReturn = new DynamoItem(replace.getTable(), item); + for (var entry : existing.getLinks().entrySet()) { + var current = toReturn.getLinks().get(entry.getKey()); + if (current == null) { + toReturn.getLinks().put(entry.getKey(), entry.getValue()); + } else { + current.addAll(entry.getValue()); + } + } + + return toReturn; + } + + public List results(ObjectMapper mapper, Class type, Optional limit) { + var items = new ArrayList(lookup.values()); + Collections.sort(items); + return items.stream().limit(limit.orElse(Integer.MAX_VALUE)).map(t -> t.convertTo(mapper, type)).collect(Collectors.toList()); + } +} diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/FlattenerSingle.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/FlattenerSingle.java new file mode 100644 index 00000000..0a0f0e94 --- /dev/null +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/FlattenerSingle.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.fleetpin.graphql.database.manager.dynamo; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fleetpin.graphql.database.manager.Table; +import com.fleetpin.graphql.database.manager.annotations.Hash; +import com.fleetpin.graphql.database.manager.util.TableCoreUtil; +import java.util.*; +import java.util.stream.Collectors; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public final class FlattenerSingle extends Flattener { + + private final Map lookup; + private final List order; + private final boolean includeOrganisationId; + + FlattenerSingle(boolean includeOrganisationId) { + lookup = new HashMap<>(); + order = new ArrayList<>(); + this.includeOrganisationId = includeOrganisationId; + } + + private String getId(DynamoItem item) { + if (includeOrganisationId) { + return item.getOrganisationId() + ":" + item.getId(); + } else { + return item.getId(); + } + } + + public DynamoItem get(Optional extractor, Class type, String id) { + String key; + if (extractor.isPresent()) { + key = TableCoreUtil.table(type) + ":" + extractor.get().hashId(id) + "\t" + extractor.get().sortId(id); + } else { + key = TableCoreUtil.table(type) + ":" + id; + } + var got = this.lookup.get(key); + if (got != null && got.isDeleted()) { + return null; + } else { + return got; + } + } + + protected void addItem(DynamoItem item) { + lookup.put(getId(item), item); + order.add(item); + } + + public List results(ObjectMapper mapper, Class type, Optional limit) { + return order.stream().limit(limit.orElse(Integer.MAX_VALUE)).map(t -> t.convertTo(mapper, type)).collect(Collectors.toList()); + } +} diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/HistoryUtil.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/HistoryUtil.java index 6cc178f3..505330cb 100644 --- a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/HistoryUtil.java +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/HistoryUtil.java @@ -1,11 +1,11 @@ package com.fleetpin.graphql.database.manager.dynamo; -import com.google.common.primitives.UnsignedBytes; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fleetpin.graphql.database.manager.util.HistoryBackupItem; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.util.HashMap; -import java.util.Objects; +import java.util.*; import java.util.stream.Stream; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; @@ -13,6 +13,15 @@ public class HistoryUtil { + static Map toAttributes(ObjectMapper mapper, HistoryBackupItem entity) { + var attributes = TableUtil.toAttributes(mapper, entity); + attributes.put("idRevision", AttributeValue.builder().b(SdkBytes.fromByteArray(entity.getIdRevision())).build()); + attributes.put("idDate", AttributeValue.builder().b(SdkBytes.fromByteArray(entity.getIdDate())).build()); + attributes.put("startsWithUpdatedAt", AttributeValue.builder().b(SdkBytes.fromByteArray(entity.getStartsWithUpdatedAt())).build()); + + return attributes; + } + public static Stream> toHistoryValue(Stream records) { return records .map(record -> record.dynamodb().newImage()) @@ -92,7 +101,7 @@ public static AttributeValue toUpdatedAtId(String starts, Long updatedAt, Boolea if (from) { updatedAtId.put((byte) 0); } else { - updatedAtId.put(UnsignedBytes.MAX_VALUE); + updatedAtId.put((byte) 0xFF); } updatedAtId.put(date.get()); } diff --git a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/TableUtil.java b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/TableUtil.java index d3ea5c63..33d6c177 100644 --- a/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/TableUtil.java +++ b/graphql-database-manager-dynamo/src/main/java/com/fleetpin/graphql/database/manager/dynamo/TableUtil.java @@ -28,11 +28,8 @@ import com.fleetpin.graphql.database.manager.annotations.GlobalIndex; import com.fleetpin.graphql.database.manager.annotations.SecondaryIndex; import com.fleetpin.graphql.database.manager.util.BackupItem; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.LinkedHashMultimap; import java.io.IOException; import java.io.UncheckedIOException; -import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -223,7 +220,7 @@ public static JsonNode toJson(ObjectMapper mapper, AttributeValue value) { } if (value.n() != null) { double v = Double.parseDouble(value.n()); - if (Math.floor(v) == v && value.n().indexOf('.') == -1 && Long.MAX_VALUE < v && Long.MIN_VALUE > v) { + if (Math.floor(v) == v && value.n().indexOf('.') == -1 && Long.MAX_VALUE > v && Long.MIN_VALUE < v) { return LongNode.valueOf(Long.parseLong(value.n())); } return DoubleNode.valueOf(v); diff --git a/graphql-database-manager-memory/pom.xml b/graphql-database-manager-memory/pom.xml index 97df5020..487dfd5e 100644 --- a/graphql-database-manager-memory/pom.xml +++ b/graphql-database-manager-memory/pom.xml @@ -7,13 +7,13 @@ com.fleetpin graphql-database-manager - 0.2.3 + 3.0.0 graphql-database-manager-memory - 5.6.0 + 5.11.0 UTF-8 @@ -31,9 +31,9 @@ - https://github.com/fleetpin/graphql-dynamodb-manager - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git + https://github.com/ashley-taylor/graphql-dynamodb-manager + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git @@ -63,7 +63,7 @@ com.fleetpin graphql-builder - 0.1.1 + 3.0.2 provided @@ -85,7 +85,7 @@ org.apache.maven.plugins maven-dependency-plugin - 2.10 + 3.8.0 copy @@ -116,7 +116,7 @@ attach-sources - jar + jar-no-fork @@ -124,7 +124,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.1.1 + 3.10.0 attach-javadocs @@ -137,7 +137,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + 3.2.6 sign-artifacts @@ -151,7 +151,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.7.0 true sonatype diff --git a/graphql-database-manager-test/pom.xml b/graphql-database-manager-test/pom.xml index 2fe2a42b..5003fb22 100644 --- a/graphql-database-manager-test/pom.xml +++ b/graphql-database-manager-test/pom.xml @@ -5,13 +5,13 @@ com.fleetpin graphql-database-manager - 0.2.30-SNAPSHOT + 3.0.4-SNAPSHOT graphql-database-manager-test - 5.6.0 + 5.11.0 UTF-8 @@ -28,18 +28,10 @@ - - - dynamodb-local-oregon - DynamoDB Local Release Repository - https://s3-us-west-2.amazonaws.com/dynamodb-local/release - - - - https://github.com/fleetpin/graphql-dynamodb-manager - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git + https://github.com/ashley-taylor/graphql-dynamodb-manager + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git HEAD @@ -70,7 +62,7 @@ com.fleetpin graphql-builder - 0.1.1 + 3.0.2 provided @@ -88,17 +80,48 @@ graphql-database-dynmodb-history-lambda ${project.parent.version} + - com.amazonaws - DynamoDBLocal - 1.11.477 - - + org.mockito + mockito-core + 5.13.0 + + org.junit.jupiter junit-jupiter ${junit.jupiter.version} provided + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.module + jackson-module-parameter-names + ${jackson.version} + test + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + ${jackson.version} + test + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + test + + + + com.amazonaws + DynamoDBLocal + 2.5.2 @@ -107,7 +130,7 @@ org.apache.maven.plugins maven-dependency-plugin - 2.10 + 3.8.0 copy @@ -138,7 +161,7 @@ attach-sources - jar + jar-no-fork @@ -146,7 +169,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.1.1 + 3.10.0 attach-javadocs @@ -159,7 +182,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + 3.2.6 sign-artifacts @@ -173,7 +196,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.7.0 true sonatype diff --git a/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/DynamoDbInitializer.java b/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/DynamoDbInitializer.java index 672961c3..c3da1aa1 100644 --- a/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/DynamoDbInitializer.java +++ b/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/DynamoDbInitializer.java @@ -14,19 +14,26 @@ import com.amazonaws.services.dynamodbv2.local.main.ServerRunner; import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fleetpin.graphql.database.manager.Database; import com.fleetpin.graphql.database.manager.dynamo.DynamoDbManager; import java.io.IOException; import java.net.ServerSocket; import java.net.URI; import java.net.URISyntaxException; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; -import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; +import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndex; +import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; +import software.amazon.awssdk.services.dynamodb.model.KeyType; +import software.amazon.awssdk.services.dynamodb.model.ProjectionType; +import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; +import software.amazon.awssdk.services.dynamodb.model.StreamViewType; import software.amazon.awssdk.services.dynamodb.streams.DynamoDbStreamsAsyncClient; final class DynamoDbInitializer { @@ -46,12 +53,24 @@ static void createTable(final DynamoDbAsyncClient client, final String name) thr KeySchemaElement.builder().attributeName("id").keyType(KeyType.RANGE).build() ) .streamSpecification(streamSpecification -> streamSpecification.streamEnabled(true).streamViewType(StreamViewType.NEW_IMAGE)) - .globalSecondaryIndexes(builder -> - builder + .globalSecondaryIndexes( + GlobalSecondaryIndex + .builder() .indexName("secondaryGlobal") .provisionedThroughput(p -> p.readCapacityUnits(10L).writeCapacityUnits(10L)) .projection(b -> b.projectionType(ProjectionType.ALL)) .keySchema(KeySchemaElement.builder().attributeName("secondaryGlobal").keyType(KeyType.HASH).build()) + .build(), + GlobalSecondaryIndex + .builder() + .indexName("parallelIndex") + .provisionedThroughput(p -> p.readCapacityUnits(10L).writeCapacityUnits(10L)) + .projection(b -> b.projectionType(ProjectionType.ALL)) + .keySchema( + KeySchemaElement.builder().attributeName("organisationId").keyType(KeyType.HASH).build(), + KeySchemaElement.builder().attributeName("parallelHash").keyType(KeyType.RANGE).build() + ) + .build() ) .localSecondaryIndexes(builder -> builder @@ -66,7 +85,8 @@ static void createTable(final DynamoDbAsyncClient client, final String name) thr AttributeDefinition.builder().attributeName("organisationId").attributeType(ScalarAttributeType.S).build(), AttributeDefinition.builder().attributeName("id").attributeType(ScalarAttributeType.S).build(), AttributeDefinition.builder().attributeName("secondaryGlobal").attributeType(ScalarAttributeType.S).build(), - AttributeDefinition.builder().attributeName("secondaryOrganisation").attributeType(ScalarAttributeType.S).build() + AttributeDefinition.builder().attributeName("secondaryOrganisation").attributeType(ScalarAttributeType.S).build(), + AttributeDefinition.builder().attributeName("parallelHash").attributeType(ScalarAttributeType.S).build() ) .provisionedThroughput(p -> p.readCapacityUnits(10L).writeCapacityUnits(10L).build()) ) @@ -115,15 +135,15 @@ static void createHistoryTable(final DynamoDbAsyncClient client, final String na .get(); } - static DynamoDBProxyServer startDynamoServer(final String port) throws Exception { - final String[] localArgs = { "-inMemory", "-port", port }; + static synchronized DynamoDBProxyServer startDynamoServer(final String port) throws Exception { + final String[] localArgs = { "-inMemory", "-disableTelemetry", "-port", port }; final var server = ServerRunner.createServerFromCommandLineArgs(localArgs); server.start(); return server; } - static DynamoDbAsyncClient startDynamoClient(final String port) throws URISyntaxException { + static DynamoDbAsyncClient startDynamoAsyncClient(final String port) throws URISyntaxException { return DynamoDbAsyncClient .builder() .region(Region.AWS_GLOBAL) @@ -132,6 +152,15 @@ static DynamoDbAsyncClient startDynamoClient(final String port) throws URISyntax .build(); } + static DynamoDbClient startDynamoClient(final String port) throws URISyntaxException { + return DynamoDbClient + .builder() + .region(Region.AWS_GLOBAL) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("anything", "anything"))) + .endpointOverride(new URI("http://localhost:" + port)) + .build(); + } + static DynamoDbStreamsAsyncClient startDynamoStreamClient(final String port) throws URISyntaxException { return DynamoDbStreamsAsyncClient .builder() @@ -149,9 +178,8 @@ static String findFreePort() throws IOException { return port; } - static Database getEmbeddedDatabase(final DynamoDbManager dynamoDbManager, final String organisationId, final CompletableFuture future) { + static Database getEmbeddedDatabase(final DynamoDbManager dynamoDbManager, final String organisationId) { final var database = dynamoDbManager.getDatabase(organisationId); - database.start(future); return database; } @@ -162,7 +190,9 @@ static DynamoDbManager getDatabaseManager( String historyTable, boolean globalEnabled, boolean hashed, - String classpath + String classpath, + String parallelIndex, + ObjectMapper objectMapper ) { return DynamoDbManager .builder() @@ -172,35 +202,8 @@ static DynamoDbManager getDatabaseManager( .global(globalEnabled) .hash(hashed) .classPath(classpath) + .parallelIndex(parallelIndex) + .objectMapper(objectMapper) .build(); } - // static Database getInMemoryDatabase( - // final String organisationId, - // final ConcurrentHashMap map, - // final CompletableFuture future - // ) { - // final var objectMapper = new ObjectMapper() - // .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - // .registerModule(new ParameterNamesModule()) - // .registerModule(new Jdk8Module()) - // .registerModule(new JavaTimeModule()) - // .disable(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS) - // .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) - // .disable(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) - // .disable(SerializationFeature.WRITE_DURATIONS_AS_TIMESTAMPS) - // .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); - // - // final var factory = new JsonNodeFactory(false); - // - // final Supplier idGenerator = () -> UUID.randomUUID().toString(); - // - // final var database = DynamoDbManager.builder() - // .tables("local") - // .dynamoDb(new InMemoryDynamoDb(objectMapper, factory, map, idGenerator)) - // .build() - // .getDatabase(organisationId); - // - // database.start(future); - // return database; - // } } diff --git a/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/HistoryProcessor.java b/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/HistoryProcessor.java index eaeaf623..51fcee47 100644 --- a/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/HistoryProcessor.java +++ b/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/HistoryProcessor.java @@ -3,17 +3,17 @@ import com.fleetpin.graphql.database.dynamo.history.lambda.HistoryLambda; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; import java.lang.reflect.Parameter; -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.ShardIteratorType; import software.amazon.awssdk.services.dynamodb.streams.DynamoDbStreamsAsyncClient; public class HistoryProcessor { private String[] tables; - private DynamoDbAsyncClient client; + private DynamoDbClient client; private DynamoDbStreamsAsyncClient streamClient; - public HistoryProcessor(DynamoDbAsyncClient client, DynamoDbStreamsAsyncClient streamClient, Parameter parameter, String organisationId) { + public HistoryProcessor(DynamoDbClient client, DynamoDbStreamsAsyncClient streamClient, Parameter parameter, String organisationId) { final var databaseNames = parameter.getAnnotation(DatabaseNames.class); this.tables = databaseNames != null ? databaseNames.value() : new String[] { "table" }; this.client = client; @@ -22,16 +22,16 @@ public HistoryProcessor(DynamoDbAsyncClient client, DynamoDbStreamsAsyncClient s static class Processor extends HistoryLambda { - private final DynamoDbAsyncClient client; + private final DynamoDbClient client; private final String tableName; - public Processor(DynamoDbAsyncClient client, String tableName) { + public Processor(DynamoDbClient client, String tableName) { this.client = client; this.tableName = tableName; } @Override - public DynamoDbAsyncClient getClient() { + public DynamoDbClient getClient() { return client; } @@ -44,7 +44,7 @@ public String getTableName() { public void process() { try { for (final String table : tables) { - var streamArn = client.describeTable(builder -> builder.tableName(table).build()).get().table().latestStreamArn(); + var streamArn = client.describeTable(builder -> builder.tableName(table).build()).table().latestStreamArn(); var shards = streamClient.describeStream(builder -> builder.streamArn(streamArn).build()).get().streamDescription().shards(); for (final var shard : shards) { @@ -54,7 +54,7 @@ public void process() { .shardIterator(); var response = streamClient.getRecords(builder -> builder.shardIterator(shardIterator)).get(); var processor = new Processor(client, table + "_history"); - processor.process(response.records().stream()); + processor.process(response.records()); } } } catch (Exception e) { diff --git a/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/TestDatabaseProvider.java b/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/TestDatabaseProvider.java index 800f4286..75acaef8 100644 --- a/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/TestDatabaseProvider.java +++ b/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/TestDatabaseProvider.java @@ -12,86 +12,74 @@ package com.fleetpin.graphql.database.manager.test; -import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.*; - -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBStreams; -import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer; +import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.createHistoryTable; +import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.createTable; +import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.findFreePort; +import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.getDatabaseManager; +import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.getEmbeddedDatabase; +import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.startDynamoAsyncClient; +import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.startDynamoClient; +import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.startDynamoServer; +import static com.fleetpin.graphql.database.manager.test.DynamoDbInitializer.startDynamoStreamClient; + +import com.fasterxml.jackson.databind.ObjectMapper; import com.fleetpin.graphql.database.manager.Database; +import com.fleetpin.graphql.database.manager.VirtualDatabase; import com.fleetpin.graphql.database.manager.dynamo.DynamoDbManager; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseOrganisation; import com.fleetpin.graphql.database.manager.test.annotations.GlobalEnabled; import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; +import com.fleetpin.graphql.database.manager.util.CompletableFutureUtil; import java.lang.reflect.AnnotatedElement; import java.util.Arrays; import java.util.List; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -import java.util.stream.Stream; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.streams.DynamoDbStreamsAsyncClient; -public final class TestDatabaseProvider implements ArgumentsProvider { +public final class TestDatabaseProvider implements ParameterResolver, BeforeEachCallback, AfterEachCallback { - private DynamoDBProxyServer server; - private CompletableFuture finished; + private static final DbHolder HOLDER = new DbHolder(); @Override - public Stream provideArguments(final ExtensionContext extensionContext) throws Exception { - closePreviousRun(); - - final String port = findFreePort(); - - server = startDynamoServer(port); - final var client = startDynamoClient(port); - final var streamClient = startDynamoStreamClient(port); - - System.setProperty("sqlite4java.library.path", "native-libs"); - - finished = new CompletableFuture<>(); - - final var testMethod = extensionContext.getRequiredTestMethod(); - final var organisationId = testMethod.getAnnotation(TestDatabase.class).organisationId(); - final var hashed = testMethod.getAnnotation(TestDatabase.class).hashed(); - final var classPath = testMethod.getAnnotation(TestDatabase.class).classPath(); - - final var withHistory = Arrays - .stream(testMethod.getParameters()) - .map(parameter -> parameter.getType().isAssignableFrom(HistoryProcessor.class)) - .filter(p -> p) - .findFirst() - .orElse(false); - - final var argumentsList = Arrays - .stream(testMethod.getParameters()) - .map(parameter -> { - try { - if (parameter.getType().isAssignableFrom(DynamoDbManager.class)) { - return createDynamoDbManager(client, streamClient, parameter, withHistory, hashed, classPath); - } else if (parameter.getType().isAssignableFrom(HistoryProcessor.class)) { - return new HistoryProcessor(client, streamClient, parameter, organisationId); - } else { - return createDatabase(client, streamClient, parameter, organisationId, withHistory, hashed, classPath); - } - } catch (final Exception e) { - e.printStackTrace(); - throw new ExceptionInInitializerError("Could not build parameters"); - } - }) - .collect(Collectors.toList()); - - return Stream.of(gatherArguments(argumentsList)); + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + if (parameterContext.getParameter().getType().isAssignableFrom(DynamoDbManager.class)) { + return true; + } + if (parameterContext.getParameter().getType().isAssignableFrom(HistoryProcessor.class)) { + return true; + } + if (parameterContext.getParameter().getType().isAssignableFrom(Database.class)) { + return true; + } + if (parameterContext.getParameter().getType().isAssignableFrom(VirtualDatabase.class)) { + return true; + } + if (parameterContext.getParameter().getType().isAssignableFrom(DynamoDbAsyncClient.class)) { + return true; + } + if (parameterContext.getParameter().getType().isAssignableFrom(DynamoDbClient.class)) { + return true; + } + return false; } - private void closePreviousRun() throws Exception { - if (server != null) { - finished.complete(null); - server.stop(); - } + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + var store = extensionContext.getStore(Namespace.create(extensionContext.getUniqueId())); + var arguments = store.get("arguments", List.class); + return arguments.get(parameterContext.getIndex()); } private Database createDatabase( @@ -101,14 +89,15 @@ private Database createDatabase( final String organisationId, final boolean withHistory, final boolean hashed, - final String classPath + final String classPath, + final ObjectMapper objectMapper ) throws ExecutionException, InterruptedException { final var databaseOrganisation = parameter.getAnnotation(DatabaseOrganisation.class); final var correctOrganisationId = databaseOrganisation != null ? databaseOrganisation.value() : organisationId; - final var dynamoDbManager = createDynamoDbManager(client, streamClient, parameter, withHistory, hashed, classPath); + final var dynamoDbManager = createDynamoDbManager(client, streamClient, parameter, withHistory, hashed, classPath, objectMapper); - return getEmbeddedDatabase(dynamoDbManager, correctOrganisationId, finished); + return getEmbeddedDatabase(dynamoDbManager, correctOrganisationId); } private DynamoDbManager createDynamoDbManager( @@ -117,7 +106,8 @@ private DynamoDbManager createDynamoDbManager( final AnnotatedElement parameter, final boolean withHistory, final boolean hashed, - final String classPath + final String classPath, + final ObjectMapper objectMapper ) throws ExecutionException, InterruptedException { final var databaseNames = parameter.getAnnotation(DatabaseNames.class); var tables = databaseNames != null ? databaseNames.value() : new String[] { "table" }; @@ -125,9 +115,6 @@ private DynamoDbManager createDynamoDbManager( String historyTable = null; for (final String table : tables) { createTable(client, table); - var streamArn = client.describeTable(builder -> builder.tableName(table).build()).get().table().latestStreamArn(); - var tName = streamClient.describeStream(builder -> builder.streamArn(streamArn).build()).get().streamDescription().tableName(); - //System.out.println("find me: " + tName); if (withHistory) { historyTable = table + "_history"; createHistoryTable(client, historyTable); @@ -141,15 +128,119 @@ private DynamoDbManager createDynamoDbManager( globalEnabled = globalEnabledAnnotation.value(); } - return getDatabaseManager(client, tables, historyTable, globalEnabled, hashed, classPath); + return getDatabaseManager(client, tables, historyTable, globalEnabled, hashed, classPath, "parallelIndex", objectMapper); + } + + @Override + public void beforeEach(ExtensionContext extensionContext) throws Exception { + var store = extensionContext.getStore(Namespace.create(extensionContext.getUniqueId())); + var test = store; + final var testMethod = extensionContext.getRequiredTestMethod(); + var testDatabase = getTestDatabase(testMethod); + + var wrapper = HOLDER.getServer(); + + test.put("table", wrapper); + + final var client = wrapper.clientAsync; + final var streamClient = wrapper.streamClient; + + System.setProperty("sqlite4java.library.path", "native-libs"); + + var organisationId = testDatabase.organisationId(); + var classPath = testDatabase.classPath(); + var hashed = testDatabase.hashed(); + var objectMapper = testDatabase.objectMapper().getConstructor().newInstance().get(); + + final var withHistory = Arrays + .stream(testMethod.getParameters()) + .map(parameter -> parameter.getType().isAssignableFrom(HistoryProcessor.class)) + .filter(p -> p) + .findFirst() + .orElse(false); + + final var argumentsList = Arrays + .stream(testMethod.getParameters()) + .map(parameter -> { + try { + if (parameter.getType().isAssignableFrom(DynamoDbManager.class)) { + return createDynamoDbManager(client, streamClient, parameter, withHistory, hashed, classPath, objectMapper); + } else if (parameter.getType().isAssignableFrom(HistoryProcessor.class)) { + return new HistoryProcessor(wrapper.client, streamClient, parameter, organisationId); + } else if (parameter.getType().isAssignableFrom(DynamoDbAsyncClient.class)) { + return wrapper.clientAsync; + } else if (parameter.getType().isAssignableFrom(DynamoDbClient.class)) { + return wrapper.client; + } else { + var database = createDatabase(client, streamClient, parameter, organisationId, withHistory, hashed, classPath, objectMapper); + if (parameter.getType().isAssignableFrom(VirtualDatabase.class)) { + return new VirtualDatabase(database); + } else { + return database; + } + } + } catch (final Exception e) { + e.printStackTrace(); + throw new ExceptionInInitializerError("Could not build parameters"); + } + }) + .collect(Collectors.toList()); + store.put("arguments", argumentsList); + } + + private TestDatabase getTestDatabase(AnnotatedElement annotatedElement) { + var annotation = annotatedElement.getAnnotation(TestDatabase.class); + if (annotation != null) { + return annotation; + } + for (var a : annotatedElement.getAnnotations()) { + annotation = getTestDatabase(a.annotationType()); + if (annotation != null) { + return annotation; + } + } + return null; + } + + @Override + public void afterEach(ExtensionContext context) throws Exception { + var wrapper = context.getStore(Namespace.create(context.getUniqueId())).get("table", ServerWrapper.class); + if (wrapper != null) { + HOLDER.returnServer(wrapper); + } } - private Arguments gatherArguments(final List argumentsList) { - final var argumentObjects = new Object[argumentsList.size()]; - for (int i = 0; i < argumentObjects.length; i++) { - argumentObjects[i] = argumentsList.get(i); + static class DbHolder { + + private final ConcurrentLinkedQueue servers; + + public DbHolder() { + this.servers = new ConcurrentLinkedQueue<>(); } - return Arguments.of(argumentObjects); + ServerWrapper getServer() throws Exception { + var wrapper = this.servers.poll(); + if (wrapper == null) { + final String port = findFreePort(); + + startDynamoServer(port); + final var clientAsync = startDynamoAsyncClient(port); + final var client = startDynamoClient(port); + + final var streamClient = startDynamoStreamClient(port); + + return new ServerWrapper(client, clientAsync, streamClient); + } else { + var tables = wrapper.client.listTables(); + CompletableFutureUtil.sequence(tables.tableNames().stream().map(table -> wrapper.clientAsync.deleteTable(r -> r.tableName(table)))).get(); + return wrapper; + } + } + + void returnServer(ServerWrapper wrapper) { + this.servers.add(wrapper); + } } + + record ServerWrapper(DynamoDbClient client, DynamoDbAsyncClient clientAsync, DynamoDbStreamsAsyncClient streamClient) {} } diff --git a/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/annotations/TestDatabase.java b/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/annotations/TestDatabase.java index e6225661..a66282e9 100644 --- a/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/annotations/TestDatabase.java +++ b/graphql-database-manager-test/src/main/java/com/fleetpin/graphql/database/manager/test/annotations/TestDatabase.java @@ -12,22 +12,26 @@ package com.fleetpin.graphql.database.manager.test.annotations; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fleetpin.graphql.database.manager.test.TestDatabaseProvider; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ArgumentsSource; +import java.util.function.Supplier; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; -@Target(ElementType.METHOD) +@Target(ElementType.ANNOTATION_TYPE) @Retention(RetentionPolicy.RUNTIME) -@ParameterizedTest -@ArgumentsSource(TestDatabaseProvider.class) +@Test +@ExtendWith(TestDatabaseProvider.class) public @interface TestDatabase { String organisationId() default "organisation"; boolean hashed() default false; String classPath() default ""; + + Class> objectMapper(); } diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbBackupTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbBackupTest.java index da9c5c17..681ede03 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbBackupTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbBackupTest.java @@ -12,25 +12,25 @@ package com.fleetpin.graphql.database.manager.test; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fleetpin.graphql.database.manager.Database; +import com.fleetpin.graphql.database.manager.QueryHistoryBuilder; import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.annotations.GlobalIndex; +import com.fleetpin.graphql.database.manager.annotations.History; import com.fleetpin.graphql.database.manager.annotations.SecondaryIndex; import com.fleetpin.graphql.database.manager.dynamo.DynamoBackupItem; import com.fleetpin.graphql.database.manager.dynamo.DynamoDbManager; -import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; -import com.fleetpin.graphql.database.manager.test.annotations.DatabaseOrganisation; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; +import com.fleetpin.graphql.database.manager.dynamo.DynamoHistoryBackupItem; import com.fleetpin.graphql.database.manager.util.BackupItem; +import com.fleetpin.graphql.database.manager.util.HistoryBackupItem; +import java.sql.Timestamp; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; +import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; final class DynamoDbBackupTest { @@ -41,8 +41,6 @@ final class DynamoDbBackupTest { void testTakeBackup(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); final var db1 = dynamoDbManager.getDatabase("organisation-1"); - db0.start(new CompletableFuture<>()); - db1.start(new CompletableFuture<>()); final var putAvocado = db0.put(new SimpleTable("avocado", "fruit")).get(); final var putBanana = db0.put(new SimpleTable("banana", "fruit")).get(); @@ -64,13 +62,34 @@ void testTakeBackup(final DynamoDbManager dynamoDbManager) throws ExecutionExcep checkResponseNameField(orgQuery2, 0, List.of(putTomato.getName())); } + @TestDatabase + void testTakeHistoryBackup(final DynamoDbManager dynamoDbManager, final HistoryProcessor historyProcessor) throws ExecutionException, InterruptedException { + final var db0 = dynamoDbManager.getDatabase("organisation"); + + final var putAvocado = db0.put(new SimpleHistoryTable("avocado", "fruit")).get(); + final var putBanana = db0.put(new SimpleHistoryTable("banana", "fruit")).get(); + final var putBeer = db0.put(new HistoryDrink("Beer", true)).get(); + + Assertions.assertNotNull(putAvocado); + Assertions.assertNotNull(putBanana); + Assertions.assertNotNull(putBeer); + + historyProcessor.process(); + + final var orgQuery = db0.takeHistoryBackup("organisation").get(); + Assertions.assertEquals(3, orgQuery.size()); + List tablesName = List.of(putBanana.getName(), putAvocado.getName(), putBeer.getName()); + checkResponseNameField(orgQuery, 0, tablesName); + checkResponseNameField(orgQuery, 1, tablesName); + checkResponseNameField(orgQuery, 2, tablesName); + } + @TestDatabase void testRestoreBackup(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final String DRINK_ID = "1234"; final String SIMPLE_ID = "6789"; final var db0 = dynamoDbManager.getDatabase("organisation-0"); - db0.start(new CompletableFuture<>()); Map drinkAttributes = new HashMap<>(); drinkAttributes.put("organisationId", AttributeValue.builder().s("organisation-0").build()); @@ -113,12 +132,71 @@ void testRestoreBackup(final DynamoDbManager dynamoDbManager) throws ExecutionEx Assertions.assertEquals("fruit", simpleTableExists.getGlobalLookup()); } + @TestDatabase + void testRestoreHistoryBackup(final DynamoDbManager dynamoDbManager, HistoryProcessor historyProcessor) throws ExecutionException, InterruptedException { + final String DRINK_ID = "1234"; + final String SIMPLE_ID = "6789"; + + final var db = dynamoDbManager.getDatabase("organisation"); + + Map drinkAttributes = new HashMap<>(); + drinkAttributes.put("organisationId", AttributeValue.builder().s("organisation").build()); + drinkAttributes.put("organisationIdType", AttributeValue.builder().s("organisation:historydrinks").build()); + drinkAttributes.put("idRevision", AttributeValue.builder().b(SdkBytes.fromUtf8String(DRINK_ID + ":")).build()); + drinkAttributes.put("id", AttributeValue.builder().s("historydrinks:" + DRINK_ID).build()); + final var dateString = "2023-11-08 12:00:00"; + final var updatedAt = Long.toString(Timestamp.valueOf(dateString).toInstant().getEpochSecond()); + final var dateBinaryAttribute = AttributeValue.builder().b(SdkBytes.fromUtf8String("08/11/2023")).build(); + drinkAttributes.put("idDate", dateBinaryAttribute); + drinkAttributes.put("startsWithUpdatedAt", dateBinaryAttribute); + drinkAttributes.put("updatedAt", AttributeValue.builder().n(updatedAt).build()); + Map items = new HashMap<>(); + items.put("revision", AttributeValue.builder().s("1").build()); + items.put("name", AttributeValue.builder().s("Beer").build()); + items.put("alcoholic", AttributeValue.builder().bool(true).build()); + items.put("id", AttributeValue.builder().s("historydrinks:" + DRINK_ID).build()); + drinkAttributes.put("item", AttributeValue.builder().m(items).build()); + + Map simpleTableAttributes = new HashMap<>(); + simpleTableAttributes.put("organisationId", AttributeValue.builder().s("organisation").build()); + simpleTableAttributes.put("organisationIdType", AttributeValue.builder().s("organisation:simplehistorytables").build()); + simpleTableAttributes.put("idRevision", AttributeValue.builder().b(SdkBytes.fromUtf8String(SIMPLE_ID + ":")).build()); + simpleTableAttributes.put("id", AttributeValue.builder().s("simplehistorytables:" + SIMPLE_ID).build()); + simpleTableAttributes.put("idDate", dateBinaryAttribute); + simpleTableAttributes.put("startsWithUpdatedAt", dateBinaryAttribute); + simpleTableAttributes.put("updatedAt", AttributeValue.builder().n(updatedAt).build()); + items.clear(); + items.put("revision", AttributeValue.builder().s("1").build()); + items.put("name", AttributeValue.builder().s("avocado").build()); + items.put("globalLookup", AttributeValue.builder().s("fruit").build()); + items.put("id", AttributeValue.builder().s("simplehistorytables:" + SIMPLE_ID).build()); + simpleTableAttributes.put("item", AttributeValue.builder().m(items).build()); + + HistoryBackupItem drinkItem = new DynamoHistoryBackupItem("table_history", drinkAttributes, mapper); + HistoryBackupItem simpleTableItem = new DynamoHistoryBackupItem("table_history", simpleTableAttributes, mapper); + + db.restoreHistoryBackup(List.of(drinkItem, simpleTableItem)).get(); + + final var drinkExists = db.queryHistory(QueryHistoryBuilder.create(HistoryDrink.class).id(DRINK_ID).build()).get(); + Assertions.assertNotNull(drinkExists); + Assertions.assertEquals(1, drinkExists.size()); + final var beer = drinkExists.get(0); + Assertions.assertEquals("Beer", beer.getName()); + Assertions.assertEquals(true, beer.getAlcoholic()); + + final var simpleTableExistsPromise = db.queryHistory(QueryHistoryBuilder.create(SimpleHistoryTable.class).id(SIMPLE_ID).build()); + final var simpleTableExists = simpleTableExistsPromise.join(); + Assertions.assertNotNull(simpleTableExists); + Assertions.assertEquals(1, simpleTableExists.size()); + final var avocado = simpleTableExists.get(0); + Assertions.assertEquals("avocado", avocado.getName()); + Assertions.assertEquals("fruit", avocado.getGlobalLookup()); + } + @TestDatabase void testDestroyOrganisation(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); final var db1 = dynamoDbManager.getDatabase("organisation-1"); - db0.start(new CompletableFuture<>()); - db1.start(new CompletableFuture<>()); db0.put(new SimpleTable("avocado", "fruit")).get(); db1.put(new SimpleTable("avocado", "fruit")).get(); @@ -137,8 +215,6 @@ void testDestroyOrganisation(final DynamoDbManager dynamoDbManager) throws Execu void testDeleteItems(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); final var db1 = dynamoDbManager.getDatabase("organisation-1"); - db0.start(new CompletableFuture<>()); - db1.start(new CompletableFuture<>()); db0.put(new SimpleTable("avocado", "fruit")).get(); db0.put(new Drink("Whisky", true)).get(); @@ -169,8 +245,6 @@ void testDeleteItems(final DynamoDbManager dynamoDbManager) throws ExecutionExce void testBatchDestroyOrganisation(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); final var db1 = dynamoDbManager.getDatabase("organisation-1"); - db0.start(new CompletableFuture<>()); - db1.start(new CompletableFuture<>()); int count = 100; for (int i = 0; i < count; i++) { db0.put(new SimpleTable("avocado", "fruit")).get(); @@ -183,15 +257,15 @@ void testBatchDestroyOrganisation(final DynamoDbManager dynamoDbManager) throws var destroyResponse = db0.destroyOrganisation("organisation-0").get(); Assertions.assertEquals(true, destroyResponse); - //For some reason this fails on the test DynamoDB but is fine on the real one? - //response0 = db0.query(SimpleTable.class).get(); - //Assertions.assertEquals(0, response0.size()); + // For some reason this fails on the test DynamoDB but is fine on the real one? + // response0 = db0.query(SimpleTable.class).get(); + // Assertions.assertEquals(0, response0.size()); var response1 = db1.query(SimpleTable.class).get(); Assertions.assertEquals(1, response1.size()); } - private void checkResponseNameField(List queryResult, Integer rank, List names) { + private void checkResponseNameField(List queryResult, Integer rank, List names) { var jsonMap = queryResult.get(rank).getItem(); ObjectMapper om = new ObjectMapper(); var itemMap = om.convertValue(jsonMap, new TypeReference>() {}); @@ -219,6 +293,14 @@ public Boolean getAlcoholic() { } } + @History + public static class HistoryDrink extends Drink { + + public HistoryDrink(String name, Boolean alcoholic) { + super(name, alcoholic); + } + } + public static class SimpleTable extends Table { private String name; @@ -241,4 +323,12 @@ public String getGlobalLookup() { return globalLookup; } } + + @History + public static class SimpleHistoryTable extends SimpleTable { + + public SimpleHistoryTable(String name, String globalLookup) { + super(name, globalLookup); + } + } } diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbDataWriterTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbDataWriterTest.java index b3b91596..b818fbfe 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbDataWriterTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbDataWriterTest.java @@ -5,7 +5,6 @@ import com.fleetpin.graphql.database.manager.DataWriter; import com.fleetpin.graphql.database.manager.DatabaseDriver; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import org.junit.jupiter.api.Assertions; import org.mockito.Mockito; @@ -15,7 +14,7 @@ final class DynamoDbDataWriterTest { void testDispatchSize() { DatabaseDriver my = Mockito.mock(DatabaseDriver.class, Mockito.CALLS_REAL_METHODS); DynamoDbIndexesTest.SimpleTable entry1 = new DynamoDbIndexesTest.SimpleTable("garry", "john"); - var dataWriter = new DataWriter(my::bulkPut); + var dataWriter = new DataWriter(my::bulkPut, __ -> {}); dataWriter.put("test", entry1, true); Assertions.assertEquals(1, dataWriter.dispatchSize()); } @@ -24,7 +23,7 @@ void testDispatchSize() { void testDispatch() { DatabaseDriver my = Mockito.mock(DatabaseDriver.class, Mockito.CALLS_REAL_METHODS); DynamoDbIndexesTest.SimpleTable entry1 = new DynamoDbIndexesTest.SimpleTable("garry", "john"); - var dataWriter = new DataWriter(my::bulkPut); + var dataWriter = new DataWriter(my::bulkPut, __ -> {}); dataWriter.put("test", entry1, true); dataWriter.dispatch(); verify(my, times(1)).bulkPut(Mockito.anyList()); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbHistoryTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbHistoryTest.java index fc461bc3..5e28514f 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbHistoryTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbHistoryTest.java @@ -17,10 +17,6 @@ import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.annotations.History; import com.fleetpin.graphql.database.manager.annotations.TableName; -import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; -import java.time.Instant; -import java.util.Comparator; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbIndexesTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbIndexesTest.java index 603c5c87..b384ca54 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbIndexesTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbIndexesTest.java @@ -21,12 +21,10 @@ import com.fleetpin.graphql.database.manager.dynamo.DynamoDbManager; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseOrganisation; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import com.fleetpin.graphql.database.manager.util.BackupItem; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; @@ -75,7 +73,6 @@ void testGlobalInheritance(@DatabaseNames({ "prod", "stage" }) final Database db void testSecondary(final Database db) throws InterruptedException, ExecutionException { var list = db.querySecondary(SimpleTable.class, "garry").get(); Assertions.assertEquals(0, list.size()); - SimpleTable entry1 = new SimpleTable("garry", "john"); entry1 = db.put(entry1).get(); Assertions.assertEquals("garry", entry1.getName()); @@ -145,8 +142,6 @@ void testGlobalUnique(final Database db) throws InterruptedException, ExecutionE void testMultiOrganisationSecondaryIndexWithDynamoDbManager(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); final var db1 = dynamoDbManager.getDatabase("organisation-1"); - db0.start(new CompletableFuture<>()); - db1.start(new CompletableFuture<>()); final var putAvocado = db0.put(new SimpleTable("avocado", "fruit")).get(); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbInheritanceTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbInheritanceTest.java index df165175..17c477e7 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbInheritanceTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbInheritanceTest.java @@ -18,7 +18,6 @@ import com.fleetpin.graphql.database.manager.Database; import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.annotations.TableName; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.Comparator; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbLinkTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbLinkTest.java index a19eb50c..1ac240bd 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbLinkTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbLinkTest.java @@ -16,7 +16,6 @@ import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseOrganisation; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.Comparator; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPermission.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPermission.java index 05c14b51..be26a515 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPermission.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPermission.java @@ -14,7 +14,6 @@ import com.fleetpin.graphql.database.manager.Database; import com.fleetpin.graphql.database.manager.Table; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.Comparator; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPutGetDeleteTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPutGetDeleteTest.java index 1fe25161..fc8dedb5 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPutGetDeleteTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPutGetDeleteTest.java @@ -18,8 +18,6 @@ import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseOrganisation; import com.fleetpin.graphql.database.manager.test.annotations.GlobalEnabled; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; @@ -65,7 +63,7 @@ void testGlobalPutGetDelete(final Database db, final Database dbProd) throws Int db.delete(entry1, false).get(); - //will not actually delete as is in global space + // will not actually delete as is in global space entry1 = db.get(SimpleTable.class, id).get(); Assertions.assertEquals("garry", entry1.getName()); Assertions.assertEquals(id, entry1.getId()); @@ -142,7 +140,7 @@ void testClimbingGlobalPutGetDelete(@DatabaseNames({ "prod", "stage" }) final Da Assertions.assertEquals("garry", entry1.getName()); Assertions.assertEquals(id, entry1.getId()); - //is global so should do nothing + // is global so should do nothing db.delete(entry1, false).get(); entry1 = db.get(SimpleTable.class, id).get(); @@ -193,9 +191,6 @@ void testTwoManagedDatabasesOnSameOrganisationPutGetDelete(final DynamoDbManager final var db = dynamoDbManager.getDatabase("test"); final var db2 = dynamoDbManager.getDatabase("test"); - db.start(new CompletableFuture<>()); - db2.start(new CompletableFuture<>()); - final var joPutEntry = db.put(new SimpleTable("jo")).get(); Assertions.assertEquals("jo", joPutEntry.getName()); Assertions.assertNotNull(joPutEntry.getId()); @@ -217,9 +212,6 @@ void testTwoManagedDatabasesPutGetDelete(final DynamoDbManager dynamoDbManager) final var db = dynamoDbManager.getDatabase("test"); final var db2 = dynamoDbManager.getDatabase("test2"); - db.start(new CompletableFuture<>()); - db2.start(new CompletableFuture<>()); - final var janePutEntry = db.put(new SimpleTable("jane")).get(); Assertions.assertEquals("jane", janePutEntry.getName()); Assertions.assertNotNull(janePutEntry.getId()); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPutValueTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPutValueTest.java index cceee52d..00637123 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPutValueTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbPutValueTest.java @@ -2,7 +2,6 @@ import com.fleetpin.graphql.database.manager.PutValue; import com.fleetpin.graphql.database.manager.RevisionMismatchException; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.concurrent.CompletableFuture; import org.junit.jupiter.api.Assertions; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbQueryBuilderTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbQueryBuilderTest.java index b72e3f1b..5152668f 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbQueryBuilderTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbQueryBuilderTest.java @@ -14,8 +14,8 @@ import com.fleetpin.graphql.database.manager.Database; import com.fleetpin.graphql.database.manager.Table; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; -import java.util.*; +import java.util.List; +import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbQueryTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbQueryTest.java index 9baf1f6f..04609b35 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbQueryTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbQueryTest.java @@ -16,7 +16,6 @@ import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; import com.fleetpin.graphql.database.manager.test.annotations.GlobalEnabled; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.Comparator; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionDeleteTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionDeleteTest.java index 8b4c6968..3dde428f 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionDeleteTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionDeleteTest.java @@ -16,7 +16,6 @@ import com.fleetpin.graphql.database.manager.RevisionMismatchException; import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.Arrays; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionLinkTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionLinkTest.java index 20d6da4a..e711d9a9 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionLinkTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionLinkTest.java @@ -16,7 +16,6 @@ import com.fleetpin.graphql.database.manager.RevisionMismatchException; import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.Arrays; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionPutTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionPutTest.java index dbae43ef..12cb1d66 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionPutTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/DynamoDbRevisionPutTest.java @@ -16,7 +16,6 @@ import com.fleetpin.graphql.database.manager.RevisionMismatchException; import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/ObjectMapperCreator.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/ObjectMapperCreator.java new file mode 100644 index 00000000..50715d57 --- /dev/null +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/ObjectMapperCreator.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.fleetpin.graphql.database.manager.test; + +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; +import java.util.function.Supplier; + +public class ObjectMapperCreator implements Supplier { + + @Override + public ObjectMapper get() { + return new ObjectMapper() + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()) + .disable(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .disable(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) + .disable(SerializationFeature.WRITE_DURATIONS_AS_TIMESTAMPS) + .setVisibility(PropertyAccessor.FIELD, Visibility.ANY); + } +} diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/TestDatabase.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/TestDatabase.java new file mode 100644 index 00000000..4e032693 --- /dev/null +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/TestDatabase.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.fleetpin.graphql.database.manager.test; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@com.fleetpin.graphql.database.manager.test.annotations.TestDatabase( + objectMapper = ObjectMapperCreator.class, + classPath = "com.fleetpin.graphql.database.manager.test" +) +public @interface TestDatabase { +} diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/VirtualDatabaseTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/VirtualDatabaseTest.java new file mode 100644 index 00000000..c1569a3a --- /dev/null +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/VirtualDatabaseTest.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.fleetpin.graphql.database.manager.test; + +import com.fleetpin.graphql.database.manager.Database; +import com.fleetpin.graphql.database.manager.Table; +import com.fleetpin.graphql.database.manager.VirtualDatabase; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import org.junit.jupiter.api.Assertions; + +public class VirtualDatabaseTest { + + @TestDatabase + public void testConcurrency(VirtualDatabase database) throws InterruptedException, ExecutionException { + List> futures = new ArrayList<>(); + + for (int i = 0; i < 100; i++) { + var f = i; + var future = CompletableFuture.supplyAsync( + () -> { + var fc = new Simple(); + fc.setId("a:b:c:" + f); + return database.put(fc); + }, + Database.VIRTUAL_THREAD_POOL + ); + futures.add(future); + } + + futures.forEach(CompletableFuture::join); + + futures.clear(); + + for (int i = 0; i < 100; i++) { + var f = i; + var future = CompletableFuture.supplyAsync( + () -> { + var fc = database.get(Simple.class, "a:b:c:" + f); + Assertions.assertNotNull(fc); + return fc; + }, + Database.VIRTUAL_THREAD_POOL + ); + futures.add(future); + } + + futures.forEach(CompletableFuture::join); + } + + static class Simple extends Table {} +} diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbBackupTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbBackupTest.java index 519802eb..f886e460 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbBackupTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbBackupTest.java @@ -25,23 +25,19 @@ import com.fleetpin.graphql.database.manager.annotations.HashLocator.HashQueryBuilder; import com.fleetpin.graphql.database.manager.annotations.SecondaryIndex; import com.fleetpin.graphql.database.manager.dynamo.DynamoDbManager; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import com.fleetpin.graphql.database.manager.util.BackupItem; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; final class DynamoDbBackupTest { - @TestDatabase(hashed = true, classPath = "com.fleetpin.graphql.database.manager.test.hashed") + @TestDatabase void testTakeBackup(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); final var db1 = dynamoDbManager.getDatabase("organisation-1"); - db0.start(new CompletableFuture<>()); - db1.start(new CompletableFuture<>()); final var putAvocado = db0.put(new SimpleTable("Beer:avocado", "fruit")).get(); final var putBanana = db0.put(new SimpleTable("Beer:banana", "fruit")).get(); @@ -64,12 +60,10 @@ void testTakeBackup(final DynamoDbManager dynamoDbManager) throws ExecutionExcep checkResponseNameField(orgQuery2, 0, List.of(putBeer.getName(), putTomato.getName())); } - @TestDatabase(hashed = true, classPath = "com.fleetpin.graphql.database.manager.test.hashed") + @TestDatabase void testRestoreBackup(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); final var db1 = dynamoDbManager.getDatabase("organisation-1"); - db0.start(new CompletableFuture<>()); - db1.start(new CompletableFuture<>()); final var putAvocado = db0.put(new SimpleTable("Beer:avocado", "fruit")).get(); final var putBanana = db0.put(new SimpleTable("Beer:banana", "fruit")).get(); @@ -105,18 +99,16 @@ void testRestoreBackup(final DynamoDbManager dynamoDbManager) throws ExecutionEx Assertions.assertEquals("fruit", simpleTableExists.getGlobalLookup()); } - @TestDatabase(hashed = true) + @TestDatabase void testDeleteItems(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); assertThrows(UnsupportedOperationException.class, () -> db0.delete("organisation-0", SimpleTable.class)); } - @TestDatabase(hashed = true, classPath = "com.fleetpin.graphql.database.manager.test.hashed") + @TestDatabase void testBatchDestroyOrganisation(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); final var db1 = dynamoDbManager.getDatabase("organisation-1"); - db0.start(new CompletableFuture<>()); - db1.start(new CompletableFuture<>()); int count = 100; for (int i = 0; i < count; i++) { var entry = new SimpleTable("avocado", "fruit"); @@ -133,9 +125,9 @@ void testBatchDestroyOrganisation(final DynamoDbManager dynamoDbManager) throws var destroyResponse = db0.destroyOrganisation("organisation-0").get(); Assertions.assertEquals(true, destroyResponse); - //For some reason this fails on the test DynamoDB but is fine on the real one? - //response0 = db0.query(SimpleTable.class).get(); - //Assertions.assertEquals(0, response0.size()); + // For some reason this fails on the test DynamoDB but is fine on the real one? + // response0 = db0.query(SimpleTable.class).get(); + // Assertions.assertEquals(0, response0.size()); var response1 = db1.query(SimpleTable.class, q -> q.startsWith("1234")).get(); Assertions.assertEquals(1, response1.size()); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbDataWriterTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbDataWriterTest.java index eb6957b5..69a1b3c6 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbDataWriterTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbDataWriterTest.java @@ -5,26 +5,25 @@ import com.fleetpin.graphql.database.manager.DataWriter; import com.fleetpin.graphql.database.manager.DatabaseDriver; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import org.junit.jupiter.api.Assertions; import org.mockito.Mockito; final class DynamoDbDataWriterTest { - @TestDatabase(hashed = true) + @TestDatabase void testDispatchSize() { DatabaseDriver my = Mockito.mock(DatabaseDriver.class, Mockito.CALLS_REAL_METHODS); DynamoDbIndexesTest.SimpleTable entry1 = new DynamoDbIndexesTest.SimpleTable("garry", "john"); - var dataWriter = new DataWriter(my::bulkPut); + var dataWriter = new DataWriter(my::bulkPut, __ -> {}); dataWriter.put("test", entry1, true); Assertions.assertEquals(1, dataWriter.dispatchSize()); } - @TestDatabase(hashed = true) + @TestDatabase void testDispatch() { DatabaseDriver my = Mockito.mock(DatabaseDriver.class, Mockito.CALLS_REAL_METHODS); DynamoDbIndexesTest.SimpleTable entry1 = new DynamoDbIndexesTest.SimpleTable("garry", "john"); - var dataWriter = new DataWriter(my::bulkPut); + var dataWriter = new DataWriter(my::bulkPut, __ -> {}); dataWriter.put("test", entry1, true); dataWriter.dispatch(); verify(my, times(1)).bulkPut(Mockito.anyList()); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbHistoryTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbHistoryTest.java index 6f5b76e8..b77cc938 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbHistoryTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbHistoryTest.java @@ -18,13 +18,12 @@ import com.fleetpin.graphql.database.manager.annotations.Hash; import com.fleetpin.graphql.database.manager.annotations.History; import com.fleetpin.graphql.database.manager.test.HistoryProcessor; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; final class DynamoDbHistoryTest { - @TestDatabase(hashed = true) + @TestDatabase void testGetRevisionsById(final Database db, final HistoryProcessor historyProcessor) throws InterruptedException, ExecutionException { var table1 = new SimpleTable("revision1"); table1.setId("hash:testTable1"); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbIndexesTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbIndexesTest.java index d5eeb92d..f1a230b7 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbIndexesTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbIndexesTest.java @@ -24,12 +24,10 @@ import com.fleetpin.graphql.database.manager.dynamo.DynamoDbManager; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseOrganisation; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import com.fleetpin.graphql.database.manager.util.BackupItem; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; @@ -37,7 +35,7 @@ final class DynamoDbIndexesTest { ObjectMapper mapper = new ObjectMapper(); - @TestDatabase(hashed = true) + @TestDatabase void testGlobal(final Database db) throws InterruptedException, ExecutionException { var list = db.queryGlobal(SimpleTable.class, "john").get(); Assertions.assertEquals(0, list.size()); @@ -54,7 +52,7 @@ void testGlobal(final Database db) throws InterruptedException, ExecutionExcepti Assertions.assertEquals("garry", db.queryGlobalUnique(SimpleTable.class, "john").get().getName()); } - @TestDatabase(hashed = true) + @TestDatabase void testGlobalInheritance(@DatabaseNames({ "prod", "stage" }) final Database db, @DatabaseNames({ "prod" }) final Database dbProd) throws InterruptedException, ExecutionException { SimpleTable entry1 = new SimpleTable("garry", "john"); @@ -74,12 +72,12 @@ void testGlobalInheritance(@DatabaseNames({ "prod", "stage" }) final Database db Assertions.assertEquals("barry", db.queryGlobalUnique(SimpleTable.class, "john").get().getName()); } - @TestDatabase(hashed = true) + @TestDatabase void testSecondary(final Database db) throws InterruptedException, ExecutionException { assertThrows(RuntimeException.class, () -> db.querySecondary(SimpleTable.class, "garry")); } - @TestDatabase(hashed = true) + @TestDatabase void testGlobalUnique(final Database db) throws InterruptedException, ExecutionException { var entry = db.queryGlobalUnique(SimpleTable.class, "john").get(); Assertions.assertNull(entry); @@ -97,12 +95,10 @@ void testGlobalUnique(final Database db) throws InterruptedException, ExecutionE Assertions.assertTrue(t.getCause().getMessage().contains("expected single linkage")); } - @TestDatabase(hashed = true) + @TestDatabase void testMultiOrganisationSecondaryIndexWithDynamoDbManager(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db0 = dynamoDbManager.getDatabase("organisation-0"); final var db1 = dynamoDbManager.getDatabase("organisation-1"); - db0.start(new CompletableFuture<>()); - db1.start(new CompletableFuture<>()); final var putAvocado = db0.put(new SimpleTable("avocado", "fruit")).get(); @@ -114,7 +110,7 @@ void testMultiOrganisationSecondaryIndexWithDynamoDbManager(final DynamoDbManage Assertions.assertNull(nonExistent); } - @TestDatabase(hashed = true) + @TestDatabase void testMultiOrganisationSecondaryIndexWithAnnotations(@DatabaseOrganisation("newdude") final Database db0, final Database db1) throws ExecutionException, InterruptedException { final var putJohn = db0.put(new SimpleTable("john", "nhoj")).get(); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbLinkTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbLinkTest.java index 9000bfb2..d6a13b44 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbLinkTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbLinkTest.java @@ -19,13 +19,12 @@ import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.annotations.Hash; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; final class DynamoDbLinkTest { - @TestDatabase(hashed = true) + @TestDatabase void testSimpleQuery(@DatabaseNames({ "prod", "stage" }) final Database db, @DatabaseNames("prod") final Database dbProd) throws InterruptedException, ExecutionException { var garry = db.put(new SimpleTable("garry")).get(); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbPutGetDeleteTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbPutGetDeleteTest.java index d9f9153a..92c3a852 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbPutGetDeleteTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbPutGetDeleteTest.java @@ -19,21 +19,16 @@ import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseOrganisation; import com.fleetpin.graphql.database.manager.test.annotations.GlobalEnabled; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; - -import static org.junit.jupiter.api.Assertions.assertEquals; - import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; - import org.junit.jupiter.api.Assertions; final class DynamoDbPutGetDeleteTest { - @TestDatabase(hashed = true) + @TestDatabase void testSimplePutGetDelete(final Database db) throws InterruptedException, ExecutionException { SimpleTable entry1 = new SimpleTable("garry"); entry1 = db.put(entry1).get(); @@ -52,7 +47,7 @@ void testSimplePutGetDelete(final Database db) throws InterruptedException, Exec Assertions.assertNull(entry1); } - @TestDatabase(hashed = true) + @TestDatabase void testGlobalPutGetDelete(final Database db, final Database dbProd) throws InterruptedException, ExecutionException { SimpleTable entry1 = new SimpleTable("garry"); entry1 = db.putGlobal(entry1).get(); @@ -73,13 +68,13 @@ void testGlobalPutGetDelete(final Database db, final Database dbProd) throws Int db.delete(entry1, false).get(); - //will not actually delete as is in global space + // will not actually delete as is in global space entry1 = db.get(SimpleTable.class, id).get(); Assertions.assertEquals("garry", entry1.getName()); Assertions.assertEquals(id, entry1.getId()); } - @TestDatabase(hashed = true) + @TestDatabase void testGlobalDisabledPutGetDelete(@GlobalEnabled(false) final Database db, @GlobalEnabled(false) final Database dbProd) throws InterruptedException, ExecutionException { SimpleTable entry1 = new SimpleTable("garry"); @@ -98,7 +93,7 @@ void testGlobalDisabledPutGetDelete(@GlobalEnabled(false) final Database db, @Gl Assertions.assertNull(entry1); } - @TestDatabase(hashed = true) + @TestDatabase void testClimbingSimplePutGetDelete(final @DatabaseNames({ "prod", "stage" }) Database db, @DatabaseNames("prod") final Database dbProd) throws InterruptedException, ExecutionException { SimpleTable entry1 = new SimpleTable("garry"); @@ -135,7 +130,7 @@ void testClimbingSimplePutGetDelete(final @DatabaseNames({ "prod", "stage" }) Da Assertions.assertEquals(id, entry1.getId()); } - @TestDatabase(hashed = true) + @TestDatabase void testClimbingGlobalPutGetDelete(@DatabaseNames({ "prod", "stage" }) final Database db, @DatabaseNames("prod") final Database dbProd) throws InterruptedException, ExecutionException { SimpleTable entry1 = new SimpleTable("garry"); @@ -150,7 +145,7 @@ void testClimbingGlobalPutGetDelete(@DatabaseNames({ "prod", "stage" }) final Da Assertions.assertEquals("garry", entry1.getName()); Assertions.assertEquals(id, entry1.getId()); - //is global so should do nothing + // is global so should do nothing db.delete(entry1, false).get(); entry1 = db.get(SimpleTable.class, id).get(); @@ -175,7 +170,7 @@ void testClimbingGlobalPutGetDelete(@DatabaseNames({ "prod", "stage" }) final Da Assertions.assertEquals(id, entry1.getId()); } - @TestDatabase(hashed = true) + @TestDatabase void testTwoOrganisationsPutGetDelete(final Database db, @DatabaseOrganisation("org-777") final Database db2) throws InterruptedException, ExecutionException { SimpleTable entry1 = new SimpleTable("garry"); @@ -196,14 +191,11 @@ void testTwoOrganisationsPutGetDelete(final Database db, @DatabaseOrganisation(" Assertions.assertNull(entry1); } - @TestDatabase(hashed = true) + @TestDatabase void testTwoManagedDatabasesOnSameOrganisationPutGetDelete(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db = dynamoDbManager.getDatabase("test"); final var db2 = dynamoDbManager.getDatabase("test"); - db.start(new CompletableFuture<>()); - db2.start(new CompletableFuture<>()); - final var joPutEntry = db.put(new SimpleTable("jo")).get(); Assertions.assertEquals("jo", joPutEntry.getName()); Assertions.assertNotNull(joPutEntry.getId()); @@ -220,14 +212,11 @@ void testTwoManagedDatabasesOnSameOrganisationPutGetDelete(final DynamoDbManager Assertions.assertNull(joWasDeleted); } - @TestDatabase(hashed = true) + @TestDatabase void testTwoManagedDatabasesPutGetDelete(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { final var db = dynamoDbManager.getDatabase("test"); final var db2 = dynamoDbManager.getDatabase("test2"); - db.start(new CompletableFuture<>()); - db2.start(new CompletableFuture<>()); - final var janePutEntry = db.put(new SimpleTable("jane")).get(); Assertions.assertEquals("jane", janePutEntry.getName()); Assertions.assertNotNull(janePutEntry.getId()); @@ -244,7 +233,7 @@ void testTwoManagedDatabasesPutGetDelete(final DynamoDbManager dynamoDbManager) Assertions.assertNull(janeWasDeleted); } - @TestDatabase(hashed = true) + @TestDatabase void testSameIdDifferentTypes(final Database db) throws InterruptedException, ExecutionException { SimpleTable entry1 = new SimpleTable("garry"); SimpleTable2 entry2 = new SimpleTable2("bob"); @@ -264,38 +253,35 @@ void testSameIdDifferentTypes(final Database db) throws InterruptedException, Ex Assertions.assertEquals("bob", entry2Future.get().getName()); } - - @TestDatabase(hashed = true) + @TestDatabase void testLotsOfDataPerPartition(final Database db) throws InterruptedException, ExecutionException { - -//putting 400 entries split across 4 partitions - + // putting 400 entries split across 4 partitions + var futures = new ArrayList>(); - + List ids = new ArrayList<>(); - + List expected = new ArrayList<>(); - - for(int i = 0; i < 400; i++) { + + for (int i = 0; i < 400; i++) { SimpleTable entry = new SimpleTable("id" + i); entry.setId("###" + (i / 100) + ":" + i); ids.add(entry.getId()); expected.add(entry.getName()); futures.add(db.put(entry, false)); } - - for(var f: futures) { + + for (var f : futures) { f.join(); } - + var get = db.get(SimpleTable.class, ids).get(); - + var actual = get.stream().map(t -> t.getName()).collect(Collectors.toList()); - + Assertions.assertEquals(expected, actual); } - - + @Hash(SimplerHasher.class) static class SimpleTable extends Table { diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbPutValueTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbPutValueTest.java index 98fd5ac6..925435a5 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbPutValueTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbPutValueTest.java @@ -2,13 +2,12 @@ import com.fleetpin.graphql.database.manager.PutValue; import com.fleetpin.graphql.database.manager.RevisionMismatchException; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.concurrent.CompletableFuture; import org.junit.jupiter.api.Assertions; public class DynamoDbPutValueTest { - @TestDatabase(hashed = true) + @TestDatabase void testSuccess() { DynamoDbIndexesTest.SimpleTable entry1 = new DynamoDbIndexesTest.SimpleTable("garry", "john"); @@ -24,7 +23,7 @@ void testSuccess() { Assertions.assertEquals(1, putValue.getEntity().getRevision()); } - @TestDatabase(hashed = true) + @TestDatabase void testFailure() { DynamoDbIndexesTest.SimpleTable entry1 = new DynamoDbIndexesTest.SimpleTable("garry", "john"); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbQueryBuilderTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbQueryBuilderTest.java index 23ad55ff..27666d03 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbQueryBuilderTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbQueryBuilderTest.java @@ -15,8 +15,8 @@ import com.fleetpin.graphql.database.manager.Database; import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.annotations.Hash; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; -import java.util.*; +import java.util.List; +import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -45,7 +45,43 @@ public String toString() { } } - @TestDatabase(hashed = true) + static class BigData extends Table { + + private String name; + private Double[][] matrix; + + public BigData(String id, String name, Double[][] matrix) { + setId(id); + this.name = name; + this.matrix = matrix; + } + } + + private Double[][] createMatrix(Integer size) { + Double[][] m = new Double[size][size]; + Random r = new Random(); + Double k = r.nextDouble(); + m[0][0] = r.nextDouble(); + for (int i = 0; i < m.length; i++) { + for (int j = 0; j < m[i].length; j++) { + if (i == 0 && j == 0) continue; else if (j == 0) { + m[i][j] = m[i - 1][m[i - 1].length - 1] + k; + } else m[i][j] = m[i][j - 1] + k; + } + } + + return m; + } + + private void swallow(CompletableFuture f) { + try { + f.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(); + } + } + + @TestDatabase void testAfter(final Database db) throws InterruptedException, ExecutionException { db.put(new Ticket("budgetId1:sales;trinkets:2020/10", "6 trinkets")).get(); db.put(new Ticket("budgetId1:sales;trinkets:2020/11", "7 trinkets")).get(); @@ -70,4 +106,64 @@ void testAfter(final Database db) throws InterruptedException, ExecutionExceptio static String getId(int i) { return String.format("%04d", i); } + + @TestDatabase + void parallelQuery(final Database db) throws InterruptedException, ExecutionException { + var n = 20; + List ids = Stream.iterate(1, i -> i + 1).map(i -> getId(i)).limit(n).collect(Collectors.toList()); + + var l = Stream + .iterate(1, i -> i + 1) + .limit(n) + // Must pick a sufficiently sized matrix in order to force multiple pages to test limit, 100 works well + .map(i -> new BigData(ids.get(i - 1), "bigdata-" + i.toString(), createMatrix(100))) + .collect(Collectors.toList()); + + l.stream().map(db::put).forEach(this::swallow); + + var allItems = db.query(BigData.class, builder -> builder).get(); + var result1 = db.query(BigData.class, builder -> builder.threadCount(2).threadIndex(0)).get(); + var result2 = db.query(BigData.class, builder -> builder.threadCount(2).threadIndex(1)).get(); + + Assertions.assertEquals(20, allItems.size()); + Assertions.assertEquals(20, result1.size() + result2.size()); + Assertions.assertNotEquals(20, result1.size()); + Assertions.assertNotEquals(20, result2.size()); + + var allItemsPage = Stream.concat(result1.stream(), result2.stream()).map(s -> s.name).collect(Collectors.toList()); + Assertions.assertEquals(true, allItems.stream().map(s -> s.name).collect(Collectors.toList()).containsAll(allItemsPage)); + } + + @TestDatabase + void parallelPagingQuery(final Database db) throws InterruptedException, ExecutionException { + var n = 20; + List ids = Stream.iterate(1, i -> i + 1).map(i -> getId(i)).limit(n).collect(Collectors.toList()); + + var l = Stream + .iterate(1, i -> i + 1) + .limit(n) + // Must pick a sufficiently sized matrix in order to force multiple pages to test limit, 100 works well + .map(i -> new BigData(ids.get(i - 1), "bigdata-" + i.toString(), createMatrix(100))) + .collect(Collectors.toList()); + + l.stream().map(db::put).forEach(this::swallow); + + var allItems = db.query(BigData.class, builder -> builder).get(); + var result1Page1 = db.query(BigData.class, builder -> builder.threadCount(2).threadIndex(0).limit(5)).get(); + var result2Page2 = db.query(BigData.class, builder -> builder.threadCount(2).threadIndex(1).limit(5)).get(); + + Assertions.assertEquals(20, allItems.size()); + + var lastPageIndex1 = result1Page1.get(result1Page1.size() - 1).getId(); + var result1Page3 = db.query(BigData.class, builder -> builder.threadCount(2).after(lastPageIndex1).threadIndex(0).limit(5)).get(); + + var lastPageIndex2 = result2Page2.get(result2Page2.size() - 1).getId(); + var result2Page4 = db.query(BigData.class, builder -> builder.threadCount(2).after(lastPageIndex2).threadIndex(1).limit(5)).get(); + + var firstSide = Stream.concat(result1Page1.stream(), result2Page2.stream()); + var secondSide = Stream.concat(result1Page3.stream(), result2Page4.stream()); + var allItemsPage = Stream.concat(firstSide, secondSide).map(s -> s.name).collect(Collectors.toList()); + + Assertions.assertTrue(allItems.stream().map(s -> s.name).collect(Collectors.toList()).containsAll(allItemsPage)); + } } diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbQueryTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbQueryTest.java index 9e72f00f..8fedc50d 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbQueryTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbQueryTest.java @@ -17,12 +17,11 @@ import com.fleetpin.graphql.database.manager.Database; import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.annotations.Hash; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.concurrent.ExecutionException; final class DynamoDbQueryTest { - @TestDatabase(hashed = true) + @TestDatabase void testSimpleQuery(final Database db) throws InterruptedException, ExecutionException { db.put(new SimpleTable("garry")).get(); db.put(new SimpleTable("bob")).get(); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbRevisionDeleteTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbRevisionDeleteTest.java index 59299ba3..61df04ec 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbRevisionDeleteTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbRevisionDeleteTest.java @@ -17,7 +17,6 @@ import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.annotations.Hash; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.Arrays; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; @@ -29,7 +28,7 @@ */ public class DynamoDbRevisionDeleteTest { - @TestDatabase(hashed = true) + @TestDatabase public void testDelete(final Database db) throws InterruptedException, ExecutionException { var entry = new SimpleTable("12345", "garry"); @@ -56,7 +55,7 @@ public void testDelete(final Database db) throws InterruptedException, Execution Assertions.assertNull(db.get(SimpleTable.class, "12345").get()); } - @TestDatabase(hashed = true) + @TestDatabase public void testMultipleEnv(final @DatabaseNames({ "prod", "stage" }) Database db, @DatabaseNames("prod") final Database dbProd) throws InterruptedException, ExecutionException { var entry = new SimpleTable("12345", "garry"); @@ -81,7 +80,7 @@ public void testMultipleEnv(final @DatabaseNames({ "prod", "stage" }) Database d Assertions.assertEquals(2, dbProd.get(SimpleTable.class, "12345").get().getRevision()); } - @TestDatabase(hashed = true) + @TestDatabase public void testMultipleEnvCheckLinks(final @DatabaseNames({ "prod", "stage" }) Database db, @DatabaseNames("prod") final Database dbProd) throws InterruptedException, ExecutionException { var entry = new SimpleTable("12345", "garry"); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbRevisionPutTest.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbRevisionPutTest.java index 8e2249fb..c79429d2 100644 --- a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbRevisionPutTest.java +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/DynamoDbRevisionPutTest.java @@ -17,13 +17,12 @@ import com.fleetpin.graphql.database.manager.Table; import com.fleetpin.graphql.database.manager.annotations.Hash; import com.fleetpin.graphql.database.manager.test.annotations.DatabaseNames; -import com.fleetpin.graphql.database.manager.test.annotations.TestDatabase; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Assertions; public class DynamoDbRevisionPutTest { - @TestDatabase(hashed = true) + @TestDatabase public void testCreateNewObject(final Database db) throws InterruptedException, ExecutionException { var entry = new SimpleTable("12345", "garry"); db.put(entry).get(); @@ -47,7 +46,7 @@ public void testCreateNewObject(final Database db) throws InterruptedException, Assertions.assertEquals(1, entries.get(0).getRevision()); } - @TestDatabase(hashed = true) + @TestDatabase public void testRevisionMustStartAt0(final Database db) throws InterruptedException, ExecutionException { var entry = new SimpleTable("12345", "garry"); entry.setRevision(1); @@ -56,7 +55,7 @@ public void testRevisionMustStartAt0(final Database db) throws InterruptedExcept Assertions.assertEquals(RevisionMismatchException.class, cause.getCause().getClass()); } - @TestDatabase(hashed = true) + @TestDatabase public void testIncrementsAfterExists(final Database db) throws InterruptedException, ExecutionException { var entry = new SimpleTable("12345", "garry"); @@ -94,7 +93,7 @@ public void testIncrementsAfterExists(final Database db) throws InterruptedExcep } } - @TestDatabase(hashed = true) + @TestDatabase public void testMultipleEnv(final @DatabaseNames({ "prod", "stage" }) Database db, @DatabaseNames("prod") final Database dbProd) throws InterruptedException, ExecutionException { var entry = new SimpleTable("12345", "garry"); @@ -116,7 +115,7 @@ public void testMultipleEnv(final @DatabaseNames({ "prod", "stage" }) Database d Assertions.assertEquals(RevisionMismatchException.class, cause.getCause().getClass()); } - @TestDatabase(hashed = true) + @TestDatabase public void testMultipleEnvConfirmRevisionIgnored(final @DatabaseNames({ "prod", "stage" }) Database db, @DatabaseNames("prod") final Database dbProd) throws InterruptedException, ExecutionException { var entry = new SimpleTable("12345", "garry"); @@ -137,7 +136,7 @@ public void testMultipleEnvConfirmRevisionIgnored(final @DatabaseNames({ "prod", Assertions.assertEquals(RevisionMismatchException.class, cause.getCause().getClass()); } - @TestDatabase(hashed = true) + @TestDatabase public void testMultipleEnvCreatedInDbBeforePut(final @DatabaseNames({ "prod", "stage" }) Database db, @DatabaseNames("prod") final Database dbProd) throws InterruptedException, ExecutionException { var entry = new SimpleTable("12345", "garry"); diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/TestDatabase.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/TestDatabase.java new file mode 100644 index 00000000..969595ae --- /dev/null +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/TestDatabase.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.fleetpin.graphql.database.manager.test.hashed; + +import com.fleetpin.graphql.database.manager.test.ObjectMapperCreator; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@com.fleetpin.graphql.database.manager.test.annotations.TestDatabase( + objectMapper = ObjectMapperCreator.class, + classPath = "com.fleetpin.graphql.database.manager.test.hashed", + hashed = true +) +public @interface TestDatabase { +} diff --git a/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/TestScanLogic.java b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/TestScanLogic.java new file mode 100644 index 00000000..7a37d0f3 --- /dev/null +++ b/graphql-database-manager-test/src/test/java/com/fleetpin/graphql/database/manager/test/hashed/TestScanLogic.java @@ -0,0 +1,134 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.fleetpin.graphql.database.manager.test.hashed; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.fleetpin.graphql.database.manager.Table; +import com.fleetpin.graphql.database.manager.annotations.Hash; +import com.fleetpin.graphql.database.manager.dynamo.DynamoDbManager; +import java.util.concurrent.ExecutionException; + +final class TestScanLogic { + + @TestDatabase + void testTakeBackup(final DynamoDbManager dynamoDbManager) throws ExecutionException, InterruptedException { + var org1 = dynamoDbManager.getVirtualDatabase("123"); + + org1.put(new Cat("mittens", "loud")).getRevision(); + org1.put(new Cat("boots", "quite")); + org1.put(new Dog("hash_otis", "small")); + + var org2 = dynamoDbManager.getVirtualDatabase("321"); + + org2.put(new Cat("horse", "medium")); + org2.put(new Dog("hash_dog", "small")); + org2.put(new Dog("hash_lassy", "loud")); + + var scan = dynamoDbManager.startTableScan(b -> + b + .updater( + Cat.class, + (context, cat) -> { + if (cat.getName().equals("boots")) { + context.delete(); + } else if (cat.getName().equals("mittens")) { + cat.setPur("gone"); + context.replace(cat); + } else { + context.getVirtualDatabase().put(cat); + } + } + ) + .updater( + Dog.class, + (context, dog) -> { + if (dog.getName().equals("hash_otis")) { + context.delete(); + } else if (dog.getName().equals("hash_dog")) { + dog.setBark("gone"); + context.replace(dog); + } else { + context.getVirtualDatabase().put(dog); + } + } + ) + ); + + scan.start().join(); + + assertNull(org1.get(Cat.class, "boots")); + var mittens = org1.get(Cat.class, "mittens"); + assertEquals(1, mittens.getRevision()); + assertEquals("gone", mittens.getPur()); + var horse = org2.get(Cat.class, "horse"); + assertEquals(2, horse.getRevision()); + + assertNull(org1.get(Dog.class, "hash_otis")); + var dog = org2.get(Dog.class, "hash_dog"); + assertEquals(1, dog.getRevision()); + assertEquals("gone", dog.getBark()); + var lassy = org2.get(Dog.class, "hash_lassy"); + assertEquals(2, lassy.getRevision()); + } + + public static class Cat extends Table { + + private String name; + private String pur; + + public Cat(String name, String pur) { + this.name = name; + this.pur = pur; + setId(name); + } + + public String getName() { + return name; + } + + public String getPur() { + return pur; + } + + public void setPur(String pur) { + this.pur = pur; + } + } + + @Hash(SimplerHasher.class) + public static class Dog extends Table { + + private String name; + private String bark; + + public Dog(String name, String bark) { + this.name = name; + this.bark = bark; + setId(name); + } + + public String getName() { + return name; + } + + public String getBark() { + return bark; + } + + public void setBark(String bark) { + this.bark = bark; + } + } +} diff --git a/pom.xml b/pom.xml index a82d9eb2..74af136c 100644 --- a/pom.xml +++ b/pom.xml @@ -2,12 +2,16 @@ 4.0.0 com.fleetpin graphql-database-manager - 0.2.30-SNAPSHOT + 3.0.4-SNAPSHOT GraphQL Builder library to make working with dynamodb and graphql easy - https://github.com/fleetpin/graphql-dynamodb-manager + https://github.com/ashley-taylor/graphql-dynamodb-manager pom + + 2.17.2 + + graphql-database-manager-core graphql-database-manager-test @@ -17,9 +21,9 @@ - https://github.com/fleetpin/graphql-dynamodb-manager - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git - scm:git:https://github.com/fleetpin/graphql-dynamodb-manager.git + https://github.com/ashley-taylor/graphql-dynamodb-manager + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git + scm:git:https://github.com/ashley-taylor/graphql-dynamodb-manager.git HEAD @@ -51,7 +55,7 @@ software.amazon.awssdk bom - 2.10.62 + 2.28.8 pom import @@ -63,21 +67,21 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.13.0 - 11 - 11 + 21 + 21 -parameters maven-surefire-plugin - 2.22.2 + 3.5.0 com.hubspot.maven.plugins prettier-maven-plugin - 0.15 + 0.22 1.4.0 160 @@ -108,7 +112,7 @@ attach-sources - jar + jar-no-fork @@ -116,7 +120,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.1.1 + 3.10.0 attach-javadocs @@ -129,7 +133,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + 3.2.6 sign-artifacts @@ -143,7 +147,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.7.0 true sonatype diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..dbc10d25 --- /dev/null +++ b/renovate.json @@ -0,0 +1,3 @@ +{ + "forkProcessing": "enabled" +}