Office365: save end date of date range in the checkpoint

Office365/CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 2024-01-10 - 2.18.7
+
+### Fixed
+
+- Save the end date of each date range in the checkpoint, in order to always progress
+
 ## 2024-01-10 - 2.18.6
 
 ### Changed

Office365/manifest.json

@@ -8,7 +8,7 @@
   "name": "Microsoft Office365",
   "uuid": "2dc2855e-3f9a-441c-af2a-30c64e0d0f4a",
   "slug": "office365",
-  "version": "2.18.6",
+  "version": "2.18.7",
   "categories": [
     "Applicative"
   ]

Office365/office365/management_api/connector.py

@@ -70,9 +70,9 @@
                     for event in events:
                         pulled_events.append(json.dumps(event))
 
-                if len(pulled_events) > self.limit_of_events_to_push:
-                    yield pulled_events
-                    pulled_events = []
+                    if len(pulled_events) > self.limit_of_events_to_push:
+                        yield pulled_events
+                        pulled_events = []
 
         if len(pulled_events) > 0:
             yield pulled_events
@@ -106,14 +106,27 @@
         for start_date, end_date in split_date_range(
             start_pull_date, end_pull_date, timedelta(minutes=self.time_range_interval)
         ):
+            intermediate_start_time = time.time()
+
+            # Get events for the current date range
             async for list_of_events in self.pull_content(start_date, end_date):
                 await self.send_events(list_of_events)
 
+            # get the ending time and compute the duration to forward the events
+            intermediate_end_time = time.time()
+            intermediate_batch_duration = intermediate_end_time - intermediate_start_time
+            FORWARD_EVENTS_DURATION.labels(intake_key=self.configuration.intake_key).observe(
+                intermediate_batch_duration
+            )
+
+            # save intermediate end date
+            checkpoint.offset = end_date
+
         # get the ending time and compute the duration to forward the events
         end_time = time.time()
         batch_duration = end_time - start_time
-        FORWARD_EVENTS_DURATION.labels(intake_key=self.configuration.intake_key).observe(batch_duration)
 
+        # save end date
         checkpoint.offset = end_pull_date
 
         # compute the remaining sleeping time. If greater than 0, sleep