support new lidl sublinks scraping

StefanBratanov · Oct 6, 2023 · 1e8d298 · 1e8d298
1 parent 81b6eb0
commit 1e8d298
Showing 1 changed file with 2 additions and 10 deletions.
diff --git a/src/main/kotlin/com/stefanbratanov/sofiasupermarketsapi/links/LidlSublinksScraper.kt b/src/main/kotlin/com/stefanbratanov/sofiasupermarketsapi/links/LidlSublinksScraper.kt
@@ -18,19 +18,11 @@ class LidlSublinksScraper(
     log.info("Scraping {} for sublinks", baseUrl)
 
     return getHtmlDocument(baseUrl)
-      .select("li[data-Creative=Main Navigation] > a")
+      .select("li.AHeroStageItems__Item > a")
       .map {
         val href = it.attr("href")
-        val sublink = baseUrl.toURI().resolve(href).toURL()
-        getHtmlDocument(sublink)
-      }
-      .flatMap {
-        it
-          .select("li.nuc-m-header-sub-nav-item > a")
-          .map { elem -> elem.attr("href") }
-          .filter { href -> !href.contains("lidl-plus") }
+        baseUrl.toURI().resolve(href).toURL()
       }
       .distinct()
-      .map { baseUrl.toURI().resolve(it).toURL() }
   }
 }