Dataset link simplification

geonetwork · Nov 14, 2024 · 75a446f · 75a446f
1 parent b80d724
commit 75a446f
Show file tree

Hide file tree

Showing 3 changed files with 118 additions and 11 deletions.
diff --git a/...nkchecker/src/main/java/net/geocat/database/linkchecker/entities/helper/DocumentLink.java b/...nkchecker/src/main/java/net/geocat/database/linkchecker/entities/helper/DocumentLink.java
@@ -92,8 +92,11 @@ public DocumentLink() {
             "OGC Web Feature Service".toLowerCase(),
             "OGC Web Map Service".toLowerCase(),
             "OGC Web Map Tile Service".toLowerCase(),
+            "OGC:WMS".toLowerCase(),
             "wms",
+            "OGC:WMTS".toLowerCase(),
             "wmts",
+            "OGC:WFS".toLowerCase(),
             "wfs",
             "atom",
             "http://www.opengeospatial.org/standards/wms",
@@ -102,6 +105,33 @@ public DocumentLink() {
             "INSPIRE Atom".toLowerCase()
     });
 
+    public static List<String> validViewProtocols = Arrays.asList(new String[]{
+            "http://www.opengis.net/def/serviceType/ogc/wms".toLowerCase(),
+            "http://www.opengis.net/def/serviceType/ogc/wmts".toLowerCase(),
+            "OGC Web Feature Service".toLowerCase(),
+            "OGC Web Map Service".toLowerCase(),
+            "OGC Web Map Tile Service".toLowerCase(),
+            "OGC:WMS".toLowerCase(),
+            "wms",
+            "OGC:WMTS".toLowerCase(),
+            "wmts",
+            "http://www.opengeospatial.org/standards/wms",
+            "http://www.opengeospatial.org/standards/wmts"
+    });
+
+    public static List<String> validDownloadProtocols = Arrays.asList(new String[]{
+            "http://www.opengis.net/def/serviceType/ogc/wfs".toLowerCase(),
+            "https://tools.ietf.org/html/rfc4287".toLowerCase(),
+            "ATOM Syndication Format".toLowerCase(),
+            "OGC Web Feature Service".toLowerCase(),
+            "OGC:WFS".toLowerCase(),
+            "wfs",
+            "atom",
+            "http://www.opengeospatial.org/standards/wfs",
+            "INSPIRE Atom".toLowerCase()
+    });
+
+
     public static List<String> validAtomProtocols = Arrays.asList(new String[]{
             "https://tools.ietf.org/html/rfc4287".toLowerCase(),
             "ATOM Syndication Format".toLowerCase(),
@@ -117,17 +147,14 @@ public DocumentLink() {
     });
 
     public boolean isInspireSimplifiedLink() {
-        if ((rawURL == null) || (protocol == null) || (applicationProfile == null))
+        if ((rawURL == null) || (protocol == null))
             return false;
-        if (rawURL.isEmpty() || protocol.isEmpty() || applicationProfile.isEmpty())
+        if (rawURL.isEmpty() || protocol.isEmpty())
             return false;
 
         if (!validProtocols.contains(protocol.toLowerCase()))
             return false;
 
-        if (!validAppProfiles.contains(applicationProfile.toLowerCase()))
-            return false;
-
         return true;
     }
 

diff --git a/...ker/src/main/java/net/geocat/database/linkchecker/service/ServiceDocumentLinkService.java b/...ker/src/main/java/net/geocat/database/linkchecker/service/ServiceDocumentLinkService.java
@@ -72,11 +72,64 @@ public DatasetDocumentLink create(DatasetMetadataRecord datasetMetadataRecord, O
         result.setFunction(onlineResource.getFunction());
         result.setOperationName(onlineResource.getOperationName());
         result.setRawURL(onlineResource.getRawURL());
-        result.setProtocol(onlineResource.getProtocol());
+
+        String protocolFromUrl = inferProtocolFromUrl(onlineResource.getRawURL());
+
+        if ((onlineResource.getProtocol() == null) && (protocolFromUrl != null)) {
+            // If no protocol defined, try to infer the protocol from the URL
+            result.setProtocol(protocolFromUrl);
+        } else {
+            result.setProtocol(onlineResource.getProtocol());
+
+            // TODO: review
+            // Check if protocol inferred is not the same type to the one defined in the protocol field and update it based on the URL inferred protocol
+            if (protocolFromUrl != null) {
+                boolean isDownloadProtocol = ServiceDocumentLink.validDownloadProtocols.contains(onlineResource.getProtocol().toLowerCase());
+                boolean isDownloadUrlProtocol = ServiceDocumentLink.validDownloadProtocols.contains(protocolFromUrl.toLowerCase());
+                boolean isViewProtocol = ServiceDocumentLink.validViewProtocols.contains(onlineResource.getProtocol().toLowerCase());
+                boolean isViewUrlProtocol = ServiceDocumentLink.validViewProtocols.contains(protocolFromUrl);
+
+                if (isDownloadProtocol) {
+                   if (!isDownloadUrlProtocol && isViewUrlProtocol) {
+                       result.setProtocol(protocolFromUrl);
+                   }
+                } else if (isViewProtocol) {
+                    if (!isViewUrlProtocol && isDownloadUrlProtocol) {
+                        result.setProtocol(protocolFromUrl);
+                    }
+                }
+            }
+        }
+
         result.setApplicationProfile(onlineResource.getApplicationProfile());
 
         result.setLinkCheckJobId(datasetMetadataRecord.getLinkCheckJobId());
 
         return result;
     }
+
+
+    private String inferProtocolFromUrl(String url) {
+        String normalizedUrl = url.toLowerCase();
+
+        if (normalizedUrl.indexOf("wms") > -1) {
+            return "wms";
+        } else if (normalizedUrl.indexOf("wmts") > -1) {
+            return "wmts";
+        } else if (normalizedUrl.indexOf("wfs") > -1) {
+            return "wfs";
+        } else if (normalizedUrl.indexOf("atom") > -1) {
+            return "atom";
+        } else if (normalizedUrl.indexOf("wcs") > -1) {
+            return "wcs";
+        } else if (normalizedUrl.indexOf("sos") > -1) {
+            return "sos";
+        } else if (normalizedUrl.indexOf("api features") > -1) {
+            return "api features";
+        } else if (normalizedUrl.indexOf("sensorthings") > -1) {
+            return "sensorthings";
+        }
+
+        return null;
+    }
 }
diff --git a/...eventprocessor/processors/postprocess/EventProcessor_PostProcessDatasetDocumentEvent.java b/...eventprocessor/processors/postprocess/EventProcessor_PostProcessDatasetDocumentEvent.java
@@ -323,13 +323,40 @@ private void process() {
                 .filter(x -> (x.getCapabilitiesDocumentType() == CapabilitiesType.WFS) || (x.getCapabilitiesDocumentType() == CapabilitiesType.Atom))
                 .collect(Collectors.toList());
 
-        if (!viewLinks.isEmpty())
+
+        if (!viewLinks.isEmpty())  {
             localDatasetMetadataRecord.setINDICATOR_VIEW_LINK_TO_DATA(IndicatorStatus.PASS);
-        if (!downloadLinks.isEmpty())
-            localDatasetMetadataRecord.setINDICATOR_DOWNLOAD_LINK_TO_DATA(IndicatorStatus.PASS);
+            localDatasetMetadataRecord.setNumberOfViewDataLinks(viewLinks.size());
+        } else {
+            // Dataset link simplification
+            if (!localDatasetMetadataRecord.getDocumentLinks().isEmpty()) {
+                List<DocumentLink> viewLinksMetadataOnlineResources = localDatasetMetadataRecord.getDocumentLinks().stream()
+                        .filter(x -> (x.getLinkState().equals(LinkState.Complete) && x.getLinkHTTPStatusCode() == 200) && (DocumentLink.validViewProtocols.contains(x.getProtocol().toLowerCase())))
+                        .collect(Collectors.toList());
+
+                if (!viewLinksMetadataOnlineResources.isEmpty()) {
+                    localDatasetMetadataRecord.setINDICATOR_VIEW_LINK_TO_DATA(IndicatorStatus.PASS);
+                    localDatasetMetadataRecord.setNumberOfViewDataLinks(viewLinksMetadataOnlineResources.size());
+                }
+            }
+        }
 
-        localDatasetMetadataRecord.setNumberOfViewDataLinks(viewLinks.size());
-        localDatasetMetadataRecord.setNumberOfDownloadDataLinks(downloadLinks.size());
+        if (!downloadLinks.isEmpty()) {
+            localDatasetMetadataRecord.setINDICATOR_DOWNLOAD_LINK_TO_DATA(IndicatorStatus.PASS);
+            localDatasetMetadataRecord.setNumberOfDownloadDataLinks(downloadLinks.size());
+        } else {
+            // Dataset link simplification
+            if (!localDatasetMetadataRecord.getDocumentLinks().isEmpty()) {
+                List<DocumentLink> downloadLinksMetadataOnlineResources = localDatasetMetadataRecord.getDocumentLinks().stream()
+                        .filter(x -> (x.getLinkState().equals(LinkState.Complete) && x.getLinkHTTPStatusCode() == 200) && (DocumentLink.validDownloadProtocols.contains(x.getProtocol().toLowerCase())))
+                        .collect(Collectors.toList());
+
+                if (!downloadLinksMetadataOnlineResources.isEmpty()) {
+                    localDatasetMetadataRecord.setINDICATOR_DOWNLOAD_LINK_TO_DATA(IndicatorStatus.PASS);
+                    localDatasetMetadataRecord.setNumberOfDownloadDataLinks(downloadLinksMetadataOnlineResources.size());
+                }
+            }
+        }
 
 //        List<ServiceDocSearchResult> serviceLinks =  new ArrayList<>();
 //        List<CapabilitiesLinkResult> capLinks =  new ArrayList<>();