From c1baae32dcc157a0409f9a52c856d76e6f08e71f Mon Sep 17 00:00:00 2001
From: rahmab <basim.rahmat-alla@kadaster.nl>
Date: Fri, 26 Jan 2024 13:15:32 +0100
Subject: [PATCH 1/3] PDOK-15939 PDOK Viewer toont soms een service niet.

---
 ngr_spider/cli.py        |  7 +++++
 ngr_spider/csw_client.py | 58 +++++++++++++++++++++++-----------------
 2 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/ngr_spider/cli.py b/ngr_spider/cli.py
index aa68ca4..c48e9c1 100755
--- a/ngr_spider/cli.py
+++ b/ngr_spider/cli.py
@@ -70,6 +70,7 @@ def main_services(args):
     if protocols:
         protocol_list = protocols.split(",")
 
+    LOGGER.info("main_services start.")
     if not show_warnings:
         cm = warnings.catch_warnings()
         warnings.simplefilter("ignore")
@@ -119,6 +120,8 @@ def main_services(args):
         content = get_output(pretty, yaml_output, config, no_updated, jq_filter)
         write_output(output_file, az_conn_string, az_container, yaml_output, content)
 
+        LOGGER.info("main_services end.")
+
         LOGGER.info(f"output written to {output_file}")
 
 
@@ -143,6 +146,8 @@ def main_layers(args):
     setup_logger(log_level)
     protocol_list = PROTOCOLS
 
+    LOGGER.info("main_layers start.")
+
     csw_client = CSWClient(csw_url)
 
     if protocols:
@@ -244,6 +249,8 @@ def main_layers(args):
             LOGGER.info(f"failed to index {len(service_errors)} services")
             message = "\n".join(service_errors_string)
             LOGGER.info(f"failed service urls:\n{message}")
+
+        LOGGER.info("main_layers end.")
         LOGGER.info(f"output written to {output_file}")
 
 
diff --git a/ngr_spider/csw_client.py b/ngr_spider/csw_client.py
index 89e14fe..e688030 100644
--- a/ngr_spider/csw_client.py
+++ b/ngr_spider/csw_client.py
@@ -9,7 +9,6 @@
 
 LOGGER = logging.getLogger(__name__)
 
-
 class CSWClient:
     def __init__(self, csw_url):
         self.csw_url = csw_url
@@ -33,32 +32,41 @@ def _get_csw_records(
         self, query: str, maxresults: int = 0, no_filter: bool = False
     ) -> list[CswServiceRecord]:
         csw = CatalogueServiceWeb(self.csw_url)
-        result: list[CswServiceRecord] = []
-        start = 1
-        maxrecord = maxresults if (maxresults < 100 and maxresults != 0) else 100
 
         while True:
-            csw.getrecords2(
-                maxrecords=maxrecord,
-                cql=query,
-                startposition=start,
-                esn="full",
-                outputschema="http://www.isotc211.org/2005/gmd",
-            )
-            records = [CswServiceRecord(rec[1].xml) for rec in csw.records.items()]
-            result.extend(records)
-            if (
-                maxresults != 0 and len(result) >= maxresults
-            ):  # break only early when maxresults set
-                break
-            if csw.results["nextrecord"] != 0:
-                start = csw.results["nextrecord"]
-                continue
-            break
-        result_out: list[CswServiceRecord] = result
-        if not no_filter:
-            result_out = self._filter_service_records(result)
-        return sorted(result_out, key=lambda x: x.title)
+            result: list[CswServiceRecord] = []
+            start = 1
+            maxrecord = maxresults if (maxresults < 100 and maxresults != 0) else 100
+            matched = 0
+            while True:
+                csw.getrecords2(
+                    maxrecords=maxrecord,
+                    cql=query,
+                    startposition=start,
+                    esn="full",
+                    outputschema="http://www.isotc211.org/2005/gmd",
+                    sortby="CreationDate:A"
+                )
+                if start == 1:
+                    matched = csw.results["matches"]
+                    LOGGER.info("Number of matched servcies before filtering: " + str(matched))
+                elif matched != csw.results["matches"]:
+                    LOGGER.info("Number of matched servcies has been changed: old = " + str(matched) + ", new = " + str(csw.results["matches"]))
+                    break # inner loop
+
+                records = [CswServiceRecord(rec[1].xml) for rec in csw.records.items()]
+                result.extend(records)
+                if (
+                    maxresults != 0 and len(result) >= maxresults
+                ):  # break only early when maxresults set
+                    break
+                if csw.results["nextrecord"] != 0:
+                    start = csw.results["nextrecord"]
+                    continue
+                result_out: list[CswServiceRecord] = result
+                if not no_filter:
+                    result_out = self._filter_service_records(result)
+                return sorted(result_out, key=lambda x: x.title)
 
     def _get_csw_records_by_protocol(
         self,

From 18a27a310cce7ede0c85e9ae4514cd9bd8dfdc25 Mon Sep 17 00:00:00 2001
From: rahmab <basim.rahmat-alla@kadaster.nl>
Date: Fri, 26 Jan 2024 15:19:47 +0100
Subject: [PATCH 2/3] PDOK-15939 PDOK Viewer toont soms een service niet.

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 42bee59..426d4bc 100644
--- a/README.md
+++ b/README.md
@@ -103,3 +103,4 @@ When running the `layers` command in flat mode (`--mode flat`), it is possible t
   { "index": 60, "names": ["ahn3+"], "types": ["OGC:WMTS"] }
 ]
 ```
+

From 3c3ca8bf39147f5b49eb11a3abef9022e957c59a Mon Sep 17 00:00:00 2001
From: rahmab <basim.rahmat-alla@kadaster.nl>
Date: Fri, 26 Jan 2024 15:22:48 +0100
Subject: [PATCH 3/3] PDOK-15939 PDOK Viewer toont soms een service niet.

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index 426d4bc..42bee59 100644
--- a/README.md
+++ b/README.md
@@ -103,4 +103,3 @@ When running the `layers` command in flat mode (`--mode flat`), it is possible t
   { "index": 60, "names": ["ahn3+"], "types": ["OGC:WMTS"] }
 ]
 ```
-