Skip to content
This repository has been archived by the owner on Sep 15, 2023. It is now read-only.

Commit

Permalink
Merge pull request #37 from admin-ch/feature/revocation-list-sync-hot…
Browse files Browse the repository at this point in the history 
…-fix

fix revocation list insert and delete batch statements
  • Loading branch information
@ubhaller
ubhaller authored Jul 26, 2021
2 parents 4b9dea2 + 17fbc48 commit 27da307
Show file tree
Hide file tree
Showing 6 changed files with 61 additions and 25 deletions.
3 changes: 2 additions & 1 deletion ...main/java/ch/admin/bag/covidcertificate/backend/verifier/data/RevokedCertDataService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@
import ch.admin.bag.covidcertificate.backend.verifier.model.DbRevokedCert;
import ch.admin.bag.covidcertificate.backend.verifier.model.cert.db.RevokedCertsUpdateResponse;
import java.util.List;
import java.util.Set;

public interface RevokedCertDataService {

/** upserts the given revoked uvcis into the db */
public RevokedCertsUpdateResponse replaceRevokedCerts(List<String> revokedUvcis);
public RevokedCertsUpdateResponse replaceRevokedCerts(Set<String> revokedUvcis);

/** returns the next batch of revoked certs after `since` */
public List<DbRevokedCert> findRevokedCerts(Long since);
Expand Down
56 changes: 42 additions & 14 deletions ...admin/bag/covidcertificate/backend/verifier/data/impl/JdbcRevokedCertDataServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@
import ch.admin.bag.covidcertificate.backend.verifier.data.mapper.RevokedCertRowMapper;
import ch.admin.bag.covidcertificate.backend.verifier.model.DbRevokedCert;
import ch.admin.bag.covidcertificate.backend.verifier.model.cert.db.RevokedCertsUpdateResponse;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import org.apache.commons.collections4.ListUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.EmptyResultDataAccessException;
Expand Down Expand Up @@ -45,30 +48,40 @@ public JdbcRevokedCertDataServiceImpl(DataSource dataSource, int revokedCertBatc

@Transactional(readOnly = false)
@Override
public RevokedCertsUpdateResponse replaceRevokedCerts(List<String> revokedUvcis) {
int insertCount = insertNewRevokedCerts(revokedUvcis);
int removeCount = removeRevokedCertsNotIn(revokedUvcis);
public RevokedCertsUpdateResponse replaceRevokedCerts(Set<String> revokedUvcis) {
Set<String> existingUvcis = findAllRevokedUvcis();
int insertCount = insertNewRevokedCerts(revokedUvcis, existingUvcis);
int removeCount = removeRevokedCertsNotIn(revokedUvcis, existingUvcis);
return new RevokedCertsUpdateResponse(insertCount, removeCount);
}

private int insertNewRevokedCerts(List<String> revokedUvcis) {
private int insertNewRevokedCerts(Set<String> revokedUvcis, Set<String> existingUvcis) {
if (revokedUvcis != null && !revokedUvcis.isEmpty()) {
List<String> existingUvcis =
jt.queryForList(
"select uvci from t_revoked_cert",
new MapSqlParameterSource(),
String.class);
List<String> toInsert =
revokedUvcis.stream()
.filter(uvci -> !existingUvcis.contains(uvci))
.collect(Collectors.toList());
revokedCertInsert.executeBatch(createParams(toInsert));

// insert in batches
final int maxBatchSize = 10000;
for (List<String> batchToInsert : ListUtils.partition(toInsert, maxBatchSize)) {
revokedCertInsert.executeBatch(createParams(batchToInsert));
}

return toInsert.size();
} else {
return 0;
}
}

private Set<String> findAllRevokedUvcis() {
return new HashSet<>(
jt.queryForList(
"select distinct uvci from t_revoked_cert",
new MapSqlParameterSource(),
String.class));
}

private MapSqlParameterSource[] createParams(List<String> revokedUvcis) {
if (revokedUvcis == null) {
return null;
Expand All @@ -82,12 +95,27 @@ private MapSqlParameterSource[] createParams(List<String> revokedUvcis) {
return params;
}

private int removeRevokedCertsNotIn(List<String> revokedUvcis) {
private int removeRevokedCertsNotIn(Set<String> uvcisToKeep, Set<String> existingUvcis) {
String sql = "delete from t_revoked_cert";
if (revokedUvcis != null && !revokedUvcis.isEmpty()) {
sql += " where uvci not in (:to_keep)";
if (uvcisToKeep != null && !uvcisToKeep.isEmpty()) {
sql += " where uvci in (:to_delete)";

List<String> toDelete =
existingUvcis.stream()
.filter(uvci -> !uvcisToKeep.contains(uvci))
.collect(Collectors.toList());

// delete in batches
int deleteCount = 0;
final int maxBatchSize = 10000;
for (List<String> batchToDelete : ListUtils.partition(toDelete, maxBatchSize)) {
deleteCount +=
jt.update(sql, new MapSqlParameterSource("to_delete", batchToDelete));
}
return deleteCount;
} else {
return jt.update(sql, new MapSqlParameterSource());
}
return jt.update(sql, new MapSqlParameterSource("to_keep", revokedUvcis));
}

@Transactional(readOnly = true)
Expand Down
16 changes: 9 additions & 7 deletions ...n/java/ch/admin/bag/covidcertificate/backend/verifier/ws/client/RevocationListSyncer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@

import ch.admin.bag.covidcertificate.backend.verifier.data.RevokedCertDataService;
import ch.admin.bag.covidcertificate.backend.verifier.model.cert.db.RevokedCertsUpdateResponse;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.HashSet;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -42,28 +42,30 @@ public void updateRevokedCerts() {
logger.info("updating revoked certs");

try {
List<String> revokedCerts = downloadRevokedCerts();
long start = System.currentTimeMillis();
Set<String> revokedCerts = downloadRevokedCerts();
logger.info("downloaded {} revoked certs", revokedCerts.size());

RevokedCertsUpdateResponse updateResponse =
revokedCertDataService.replaceRevokedCerts(revokedCerts);

logger.info(
"finished updating revoked certs. inserted {}, removed {}",
"finished updating revoked certs. inserted {}, removed {}. took {}ms",
updateResponse.getInsertCount(),
updateResponse.getRemoveCount());
updateResponse.getRemoveCount(),
System.currentTimeMillis() - start);
} catch (Exception e) {
logger.error("revoked certs update failed", e);
}
}

private List<String> downloadRevokedCerts() {
private Set<String> downloadRevokedCerts() {
final var requestEndpoint = baseurl + endpoint;
final var uri = UriComponentsBuilder.fromHttpUrl(requestEndpoint).build().toUri();
final RequestEntity<Void> requestEntity =
RequestEntity.get(uri).headers(createDownloadHeaders()).build();
final var response = rt.exchange(requestEntity, String[].class).getBody();
return new ArrayList<>(Arrays.asList(response));
return new HashSet<>(Arrays.asList(response));
}

private HttpHeaders createDownloadHeaders() {
Expand Down
4 changes: 2 additions & 2 deletions .../main/java/ch/admin/bag/covidcertificate/backend/verifier/ws/config/SchedulingConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ public void syncRevocationListOnStartup() {
revocationListSyncer.updateRevokedCerts();
}

// Sync revocation list every full hour (default)
@Scheduled(cron = "${revocationList.sync.cron:0 0 * ? * *}")
// Sync revocation list every 5 minutes from the full hour (default)
@Scheduled(cron = "${revocationList.sync.cron:0 0/5 * ? * *}")
@SchedulerLock(name = "revocation_list_sync", lockAtLeastFor = "PT15S")
public void syncRevocationList() {
LockAssert.assertLocked();
Expand Down
2 changes: 1 addition & 1 deletion .../java/ch/admin/bag/covidcertificate/backend/verifier/ws/controller/dev/DevController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class DevController {
@GetMapping(value = "/v1/revocation-list")
public @ResponseBody ResponseEntity<List<String>> getMockRevokedCerts() {
List<String> response = new ArrayList<>();
for (int i = 0; i < 10; i++) {
for (int i = 0; i < 100010; i++) {
response.add("urn:uvci:01:CH:MOCK" + i);
}
return ResponseEntity.ok(response);
Expand Down
5 changes: 5 additions & 0 deletions ch-covidcertificate-backend-verifier/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
Expand Down

0 comments on commit 27da307

Please sign in to comment.