Merge pull request #34 from admin-ch/feature/revocation-list-paging

Feature/revocation list paging

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-data/src/main/java/ch/admin/bag/covidcertificate/backend/verifier/data/RevokedCertDataService.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2021 Ubique Innovation AG <https://www.ubique.ch>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package ch.admin.bag.covidcertificate.backend.verifier.data;
+
+import ch.admin.bag.covidcertificate.backend.verifier.model.DbRevokedCert;
+import ch.admin.bag.covidcertificate.backend.verifier.model.cert.db.RevokedCertsUpdateResponse;
+import java.util.List;
+
+public interface RevokedCertDataService {
+
+    /** upserts the given revoked uvcis into the db */
+    public RevokedCertsUpdateResponse replaceRevokedCerts(List<String> revokedUvcis);
+
+    /** returns the next batch of revoked certs after `since` */
+    public List<DbRevokedCert> findRevokedCerts(Long since);
+
+    /** returns the highest revoked cert pk id */
+    public long findMaxRevokedCertPkId();
+}

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-data/src/main/java/ch/admin/bag/covidcertificate/backend/verifier/data/impl/JdbcRevokedCertDataServiceImpl.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2021 Ubique Innovation AG <https://www.ubique.ch>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package ch.admin.bag.covidcertificate.backend.verifier.data.impl;
+
+import ch.admin.bag.covidcertificate.backend.verifier.data.RevokedCertDataService;
+import ch.admin.bag.covidcertificate.backend.verifier.data.mapper.RevokedCertRowMapper;
+import ch.admin.bag.covidcertificate.backend.verifier.model.DbRevokedCert;
+import ch.admin.bag.covidcertificate.backend.verifier.model.cert.db.RevokedCertsUpdateResponse;
+import java.util.Arrays;
+import java.util.List;
+import javax.sql.DataSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.EmptyResultDataAccessException;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.transaction.annotation.Transactional;
+
+public class JdbcRevokedCertDataServiceImpl implements RevokedCertDataService {
+
+    private static final Logger logger =
+            LoggerFactory.getLogger(JdbcRevokedCertDataServiceImpl.class);
+
+    private static final int MAX_REVOKED_CERT_BATCH_COUNT = 1000;
+    private final NamedParameterJdbcTemplate jt;
+
+    public JdbcRevokedCertDataServiceImpl(DataSource dataSource) {
+        this.jt = new NamedParameterJdbcTemplate(dataSource);
+    }
+
+    @Transactional(readOnly = false)
+    @Override
+    public RevokedCertsUpdateResponse replaceRevokedCerts(List<String> revokedUvcis) {
+        int insertCount = upsertRevokedCerts(revokedUvcis);
+        int removeCount = removeRevokedCertsNotIn(revokedUvcis);
+        return new RevokedCertsUpdateResponse(insertCount, removeCount);
+    }
+
+    private int upsertRevokedCerts(List<String> revokedUvcis) {
+        if (revokedUvcis != null && !revokedUvcis.isEmpty()) {
+            String sql =
+                    "insert into t_revoked_cert"
+                            + " (uvci)"
+                            + " values (:uvci)"
+                            + " on conflict (uvci)"
+                            + " do nothing";
+            int[] updateCounts = jt.batchUpdate(sql, createParams(revokedUvcis));
+            return Arrays.stream(updateCounts).sum();
+        } else {
+            return 0;
+        }
+    }
+
+    private MapSqlParameterSource[] createParams(List<String> revokedUvcis) {
+        if (revokedUvcis == null) {
+            return null;
+        }
+
+        int size = revokedUvcis.size();
+        MapSqlParameterSource[] params = new MapSqlParameterSource[size];
+        for (int i = 0; i < size; i++) {
+            params[i] = new MapSqlParameterSource("uvci", revokedUvcis.get(i));
+        }
+        return params;
+    }
+
+    private int removeRevokedCertsNotIn(List<String> revokedUvcis) {
+        String sql = "delete from t_revoked_cert";
+        if (revokedUvcis != null && !revokedUvcis.isEmpty()) {
+            sql += " where uvci not in (:to_keep)";
+        }
+        return jt.update(sql, new MapSqlParameterSource("to_keep", revokedUvcis));
+    }
+
+    @Transactional(readOnly = true)
+    @Override
+    public List<DbRevokedCert> findRevokedCerts(Long since) {
+        if (since == null) {
+            since = 0L;
+        }
+        String sql =
+                "select pk_revoked_cert_id, uvci from t_revoked_cert"
+                        + " where pk_revoked_cert_id > :since"
+                        + " order by pk_revoked_cert_id asc"
+                        + " limit :max_batch_count";
+        MapSqlParameterSource params = new MapSqlParameterSource();
+        params.addValue("since", since);
+        params.addValue("max_batch_count", MAX_REVOKED_CERT_BATCH_COUNT);
+        return jt.query(sql, params, new RevokedCertRowMapper());
+    }
+
+    @Transactional(readOnly = true)
+    @Override
+    public long findMaxRevokedCertPkId() {
+        try {
+            String sql =
+                    "select pk_revoked_cert_id from t_revoked_cert"
+                            + " order by pk_revoked_cert_id desc"
+                            + " limit 1";
+            return jt.queryForObject(sql, new MapSqlParameterSource(), Long.class);
+        } catch (EmptyResultDataAccessException e) {
+            return 0L;
+        }
+    }
+}

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-data/src/main/java/ch/admin/bag/covidcertificate/backend/verifier/data/mapper/RevokedCertRowMapper.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2021 Ubique Innovation AG <https://www.ubique.ch>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package ch.admin.bag.covidcertificate.backend.verifier.data.mapper;
+
+import ch.admin.bag.covidcertificate.backend.verifier.model.DbRevokedCert;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.springframework.jdbc.core.RowMapper;
+
+public class RevokedCertRowMapper implements RowMapper<DbRevokedCert> {
+
+    @Override
+    public DbRevokedCert mapRow(ResultSet resultSet, int i) throws SQLException {
+        var revokedCert = new DbRevokedCert();
+        revokedCert.setPkId(resultSet.getLong("pk_revoked_cert_id"));
+        revokedCert.setUvci(resultSet.getString("uvci"));
+        return revokedCert;
+    }
+}

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-data/src/main/resources/db/migration/pgsql/V0_5__revoked_certs.sql

@@ -0,0 +1,12 @@
+/*
+ * Created by Ubique Innovation AG
+ * https://www.ubique.ch
+ * Copyright (c) 2021. All rights reserved.
+ */
+
+CREATE TABLE t_revoked_cert
+(
+    pk_revoked_cert_id serial                       NOT NULL,
+    uvci               character varying(50) UNIQUE NOT NULL,
+    CONSTRAINT pk_t_revoked_cert PRIMARY KEY (pk_revoked_cert_id)
+);

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-data/src/main/resources/db/migration/pgsql_cluster/V0_5__revoked_certs.sql

@@ -0,0 +1,12 @@
+/*
+ * Created by Ubique Innovation AG
+ * https://www.ubique.ch
+ * Copyright (c) 2021. All rights reserved.
+ */
+
+CREATE TABLE t_revoked_cert
+(
+    pk_revoked_cert_id serial                       NOT NULL,
+    uvci               character varying(50) UNIQUE NOT NULL,
+    CONSTRAINT pk_t_revoked_cert PRIMARY KEY (pk_revoked_cert_id)
+);

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-model/src/main/java/ch/admin/bag/covidcertificate/backend/verifier/model/DbRevokedCert.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021 Ubique Innovation AG <https://www.ubique.ch>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package ch.admin.bag.covidcertificate.backend.verifier.model;
+
+public class DbRevokedCert {
+    private Long pkId;
+    private String uvci;
+
+    public Long getPkId() {
+        return pkId;
+    }
+
+    public void setPkId(Long pkId) {
+        this.pkId = pkId;
+    }
+
+    public String getUvci() {
+        return uvci;
+    }
+
+    public void setUvci(String uvci) {
+        this.uvci = uvci;
+    }
+}

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-model/src/main/java/ch/admin/bag/covidcertificate/backend/verifier/model/RevocationResponse.java

@@ -15,6 +15,12 @@ public class RevocationResponse {
             example = "172800000")
     private Duration validDuration = Duration.ofHours(48);
 
+    public RevocationResponse() {}
+
+    public RevocationResponse(List<String> revokedCerts) {
+        this.revokedCerts = revokedCerts;
+    }
+
     public List<String> getRevokedCerts() {
         return revokedCerts;
     }

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-model/src/main/java/ch/admin/bag/covidcertificate/backend/verifier/model/cert/db/RevokedCertsUpdateResponse.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2021 Ubique Innovation AG <https://www.ubique.ch>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package ch.admin.bag.covidcertificate.backend.verifier.model.cert.db;
+
+public class RevokedCertsUpdateResponse {
+    private final int insertCount;
+    private final int removeCount;
+
+    public RevokedCertsUpdateResponse(int insertCount, int removeCount) {
+        this.insertCount = insertCount;
+        this.removeCount = removeCount;
+    }
+
+    public int getInsertCount() {
+        return insertCount;
+    }
+
+    public int getRemoveCount() {
+        return removeCount;
+    }
+}

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-ws/pom.xml

@@ -237,7 +237,7 @@
                             ch.admin.bag.covidcertificate.backend.verifier.ws.controller.KeyControllerV2
                         </controller>
                         <controller>
-                            ch.admin.bag.covidcertificate.backend.verifier.ws.controller.RevocationListController
+                            ch.admin.bag.covidcertificate.backend.verifier.ws.controller.RevocationListControllerV2
                         </controller>
                         <controller>
                             ch.admin.bag.covidcertificate.backend.verifier.ws.controller.VerificationRulesController

ch-covidcertificate-backend-verifier/ch-covidcertificate-backend-verifier-ws/src/main/java/ch/admin/bag/covidcertificate/backend/verifier/ws/client/RevocationListSyncer.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2021 Ubique Innovation AG <https://www.ubique.ch>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package ch.admin.bag.covidcertificate.backend.verifier.ws.client;
+
+import ch.admin.bag.covidcertificate.backend.verifier.data.RevokedCertDataService;
+import ch.admin.bag.covidcertificate.backend.verifier.model.cert.db.RevokedCertsUpdateResponse;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.RequestEntity;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+public class RevocationListSyncer {
+
+    private static final Logger logger = LoggerFactory.getLogger(RevocationListSyncer.class);
+
+    private final String baseurl;
+    private final String endpoint = "/v1/revocation-list";
+    private final RevokedCertDataService revokedCertDataService;
+    @Autowired private RestTemplate rt;
+
+    public RevocationListSyncer(
+            String revokedCertsBaseUrl, RevokedCertDataService revokedCertDataService) {
+        this.baseurl = revokedCertsBaseUrl;
+        this.revokedCertDataService = revokedCertDataService;
+    }
+
+    public void updateRevokedCerts() {
+        logger.info("updating revoked certs");
+
+        try {
+            List<String> revokedCerts = downloadRevokedCerts();
+            logger.info("downloaded {} revoked certs", revokedCerts.size());
+
+            RevokedCertsUpdateResponse updateResponse =
+                    revokedCertDataService.replaceRevokedCerts(revokedCerts);
+
+            logger.info(
+                    "finished updating revoked certs. inserted {}, removed {}",
+                    updateResponse.getInsertCount(),
+                    updateResponse.getRemoveCount());
+        } catch (Exception e) {
+            logger.error("revoked certs update failed", e);
+        }
+    }
+
+    private List<String> downloadRevokedCerts() {
+        final var requestEndpoint = baseurl + endpoint;
+        final var uri = UriComponentsBuilder.fromHttpUrl(requestEndpoint).build().toUri();
+        final RequestEntity<Void> requestEntity =
+                RequestEntity.get(uri).headers(createDownloadHeaders()).build();
+        final var response = rt.exchange(requestEntity, String[].class).getBody();
+        return new ArrayList<>(Arrays.asList(response));
+    }
+
+    private HttpHeaders createDownloadHeaders() {
+        HttpHeaders headers = new HttpHeaders();
+        headers.add(HttpHeaders.ACCEPT, "application/json");
+        return headers;
+    }
+}