From e1c610e34740ed384ab8a4dd3dc08c2ee77fe00a Mon Sep 17 00:00:00 2001 From: Michael Duffett Date: Tue, 16 Apr 2024 07:56:49 +0930 Subject: [PATCH 1/7] Add bergamot --- recipe_scrapers/__init__.py | 3 + recipe_scrapers/bergamot.py | 77 ++++++++++++++++++++++++ tests/legacy/test_bergamot.py | 61 +++++++++++++++++++ tests/legacy/test_data/bergamot.testhtml | 1 + tests/legacy/test_data/bergamot.testjson | 1 + 5 files changed, 143 insertions(+) create mode 100644 recipe_scrapers/bergamot.py create mode 100644 tests/legacy/test_bergamot.py create mode 100644 tests/legacy/test_data/bergamot.testhtml create mode 100644 tests/legacy/test_data/bergamot.testjson diff --git a/recipe_scrapers/__init__.py b/recipe_scrapers/__init__.py index e0f0a0c31..82af06a46 100644 --- a/recipe_scrapers/__init__.py +++ b/recipe_scrapers/__init__.py @@ -30,6 +30,7 @@ from .barefootcontessa import BareFootContessa from .bbcfood import BBCFood from .bbcgoodfood import BBCGoodFood +from .bergamot import Bergamot from .bestrecipes import BestRecipes from .bettybossi import BettyBossi from .bettycrocker import BettyCrocker @@ -338,6 +339,8 @@ BakingSense.host(): BakingSense, BakingMischief.host(): BakingMischief, BareFootContessa.host(): BareFootContessa, + Bergamot.host(): Bergamot, + Bergamot.host(): Bergamot, BestRecipes.host(): BestRecipes, BettyBossi.host(): BettyBossi, BettyCrocker.host(): BettyCrocker, diff --git a/recipe_scrapers/bergamot.py b/recipe_scrapers/bergamot.py new file mode 100644 index 000000000..651d6b32c --- /dev/null +++ b/recipe_scrapers/bergamot.py @@ -0,0 +1,77 @@ +# mypy: allow-untyped-defs +import requests + +from ._abstract import HEADERS, AbstractScraper +from ._utils import url_path_to_dict + + +class Bergamot(AbstractScraper): + def __init__(self, url, proxies=None, timeout=None, *args, **kwargs): + super().__init__(url=url, proxies=proxies, timeout=timeout, *args, **kwargs) + + url_dict = url_path_to_dict(url) + path = url_dict.get("path") + recipe_id = path.split("/")[-1] + + data_url = f"https://api.bergamot.app/recipes/shared?r={recipe_id}" + response = requests.get( + data_url, headers=HEADERS, proxies=proxies, timeout=timeout + ) + self.data = response.json() + + @classmethod + def host(cls): + return "dashboard.bergamot.app" + + def canonical_url(self): + source_url = self.data.get("sourceUrl") + return source_url if source_url else self.url + + def author(self): + return None + + def title(self): + return self.data.get("title") + + def category(self): + return None + + def total_time(self): + return self.data.get("time").get("totalTime") + + def yields(self): + servings = self.data.get("servings") + return f"{servings} servings" + + def image(self): + photos = self.data.get("photos") + if not photos: + return + + photo = photos[0] + return photo.get("sourceUrl") + + def ingredients(self): + return self._map_list("ingredients") + + def instructions(self): + instructions_list = self._map_list("instructions") + return "\n".join(instructions_list) + + def ratings(self): + return None + + def cuisine(self): + return None + + def description(self): + return self.data.get("description") + + def prep_time(self): + return self.data.get("time").get("prepTime") + + def _map_list(self, data_key): + output = [] + for entry in self.data.get(data_key): + output.extend(entry.get("data")) + return output diff --git a/tests/legacy/test_bergamot.py b/tests/legacy/test_bergamot.py new file mode 100644 index 000000000..9a7ced876 --- /dev/null +++ b/tests/legacy/test_bergamot.py @@ -0,0 +1,61 @@ +from responses import GET + +from recipe_scrapers.bergamot import Bergamot +from tests.legacy import ScraperTest + + +class TestBergamotScraper(ScraperTest): + scraper_class = Bergamot + + @classmethod + def expected_requests(cls): + yield GET, "https://dashboard.bergamot.app/shared/mIB4jYQtZU1A97", "tests/legacy/test_data/bergamot.testhtml" + yield GET, "https://api.bergamot.app/recipes/shared?r=mIB4jYQtZU1A97", "tests/legacy/test_data/bergamot.testjson" + + def test_canonical_url(self): + self.assertEqual( + self.harvester_class.canonical_url(), + "https://www.elle.fr/Elle-a-Table/Recettes-de-cuisine/Soupe-miso-aux-oignons-nouveaux-tofu-et-saumon-emiette-4188650", + ) + + def test_host(self): + self.assertEqual("dashboard.bergamot.app", self.harvester_class.host()) + + def test_title(self): + self.assertEqual( + "Soupe miso aux oignons nouveaux, tofu et saumon émietté", + self.harvester_class.title(), + ) + + def test_author(self): + self.assertEqual(None, self.harvester_class.author()) + + def test_total_time(self): + self.assertEqual(50, self.harvester_class.total_time()) + + def test_yields(self): + self.assertEqual("4 servings", self.harvester_class.yields()) + + def test_ingredients(self): + self.assertEqual( + [ + "100 g de saumon frais", + "6 oignons nouveaux", + "30 g d'algues wakame séchées", + "70 g de pâte de miso blanc", + "quelques cives", + "300 g de tofu soyeux", + "1 cuillère(s) à soupe d'huile de sésame", + "1 cuillère(s) à soupe de graines de sésame", + ], + self.harvester_class.ingredients(), + ) + + def test_instructions(self): + return self.assertEqual( + "Dans une poêle bien chaude, faites cuire le saumon côté peau pendant 5 mn, puis laissez-le refroidir avant de l’émietter.\nDans une casserole, versez 1,5l d’eau, la moitié des oignons lavés et coupés en deux dans la hauteur, et les algues, puis portez à ébullition, réduisez ensuite le feu et laissez mijoter pendant 20 mn. Filtrez et ajoutez le miso, mélangez soigneusement.\nAjoutez le reste des oignons coupés en quatre, les cives lavées et émincées, le tofu coupé en dés et les miettes de saumon. Arrosez d’huile de sésame et parsemez de graines de sésame. Dégustez bien chaud.", + self.harvester_class.instructions(), + ) + + def test_ratings(self): + self.assertEqual(None, self.harvester_class.ratings()) diff --git a/tests/legacy/test_data/bergamot.testhtml b/tests/legacy/test_data/bergamot.testhtml new file mode 100644 index 000000000..56ee25ab0 --- /dev/null +++ b/tests/legacy/test_data/bergamot.testhtml @@ -0,0 +1 @@ +Bergamot
\ No newline at end of file diff --git a/tests/legacy/test_data/bergamot.testjson b/tests/legacy/test_data/bergamot.testjson new file mode 100644 index 000000000..d7a84c311 --- /dev/null +++ b/tests/legacy/test_data/bergamot.testjson @@ -0,0 +1 @@ +{"id":210338,"shortId":"mIB4jYQtZU1A97","userId":585,"userFavorite":0,"sourceId":10,"sourceUrl":"https://www.elle.fr/Elle-a-Table/Recettes-de-cuisine/Soupe-miso-aux-oignons-nouveaux-tofu-et-saumon-emiette-4188650","lang":"","title":"Soupe miso aux oignons nouveaux, tofu et saumon émietté","description":"La soupe miso enrichie de saumon.","userNote":null,"ingredients":[{"data":["100 g de saumon frais","6 oignons nouveaux","30 g d'algues wakame séchées","70 g de pâte de miso blanc","quelques cives","300 g de tofu soyeux","1 cuillère(s) à soupe d'huile de sésame","1 cuillère(s) à soupe de graines de sésame"]}],"instructions":[{"data":["Dans une poêle bien chaude, faites cuire le saumon côté peau pendant 5 mn, puis laissez-le refroidir avant de l’émietter.","Dans une casserole, versez 1,5l d’eau, la moitié des oignons lavés et coupés en deux dans la hauteur, et les algues, puis portez à ébullition, réduisez ensuite le feu et laissez mijoter pendant 20 mn. Filtrez et ajoutez le miso, mélangez soigneusement.","Ajoutez le reste des oignons coupés en quatre, les cives lavées et émincées, le tofu coupé en dés et les miettes de saumon. Arrosez d’huile de sésame et parsemez de graines de sésame. Dégustez bien chaud."]}],"time":{"prepTime":20,"cookTime":null,"totalTime":50},"nutrition":{},"servings":4,"createdAt":"2024-01-16T16:16:24.000Z","updatedAt":"2024-01-16T16:16:24.000Z","deletedAt":null,"photos":[{"id":198989,"recipeId":210338,"reference":"210338PZAE79ER","order":0,"status":"uploaded","isUserUploaded":0,"sourceUrl":"https://resize.elle.fr/portrait_1280/var/plain_site/storage/images/elle-a-table/recettes-de-cuisine/soupe-miso-aux-oignons-nouveaux-tofu-et-saumon-emiette-4188650/101348896-2-fre-FR/Soupe-miso-aux-oignons-nouveaux-tofu-et-saumon-emiette.jpg","filenameExtension":"jpg","createdAt":"2024-01-16T16:16:24.000Z","updatedAt":"2024-01-16T16:16:24.000Z","deletedAt":null,"photoUrl":"https://aihkimhfpo.cloudimg.io/v7/foodbox/210338PZAE79ER.jpg?w=1280","photoThumbUrl":"https://aihkimhfpo.cloudimg.io/v7/foodbox/210338PZAE79ER.jpg?w=600&h=338"}],"sourceDomain":"elle.fr"} \ No newline at end of file From 71c4648af3ae71a3423b26b232c412043a007044 Mon Sep 17 00:00:00 2001 From: Michael Duffett Date: Tue, 16 Apr 2024 14:13:02 +0930 Subject: [PATCH 2/7] Check for time being non null --- recipe_scrapers/bergamot.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/recipe_scrapers/bergamot.py b/recipe_scrapers/bergamot.py index 651d6b32c..817805066 100644 --- a/recipe_scrapers/bergamot.py +++ b/recipe_scrapers/bergamot.py @@ -37,7 +37,7 @@ def category(self): return None def total_time(self): - return self.data.get("time").get("totalTime") + return self._get_time_value("totalTime") def yields(self): servings = self.data.get("servings") @@ -68,10 +68,17 @@ def description(self): return self.data.get("description") def prep_time(self): - return self.data.get("time").get("prepTime") + return self._get_time_value("prepTime") def _map_list(self, data_key): output = [] for entry in self.data.get(data_key): output.extend(entry.get("data")) return output + + def _get_time_value(self, time_key): + time_values = self.data.get("time") + if not time_values: + return None + + return time_values.get(time_key) From d91d39229bf0577d926d7c04481878764c867c69 Mon Sep 17 00:00:00 2001 From: Michael Duffett Date: Tue, 16 Apr 2024 17:59:59 +0930 Subject: [PATCH 3/7] Add to README --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index aab825ebe..2515ed5c8 100644 --- a/README.rst +++ b/README.rst @@ -109,6 +109,7 @@ Scrapers available for: - `https://bbc.com/ `_ - `https://bbc.co.uk/ `_ - `https://bbcgoodfood.com/ `_ +- `https://dashboard.bergamot.app/ `_ - `https://bestrecipes.com.au/ `_ - `https://bettybossi.ch/ `_ - `https://bettycrocker.com/ `_ From d4bc5181728f7868ba2e5ae909dc6f309641b791 Mon Sep 17 00:00:00 2001 From: mlduff <46545313+mlduff@users.noreply.github.com> Date: Thu, 18 Apr 2024 07:57:36 +0930 Subject: [PATCH 4/7] Update recipe_scrapers/__init__.py Co-authored-by: James Addison <55152140+jayaddison@users.noreply.github.com> --- recipe_scrapers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/recipe_scrapers/__init__.py b/recipe_scrapers/__init__.py index 82af06a46..7df2b036e 100644 --- a/recipe_scrapers/__init__.py +++ b/recipe_scrapers/__init__.py @@ -340,7 +340,6 @@ BakingMischief.host(): BakingMischief, BareFootContessa.host(): BareFootContessa, Bergamot.host(): Bergamot, - Bergamot.host(): Bergamot, BestRecipes.host(): BestRecipes, BettyBossi.host(): BettyBossi, BettyCrocker.host(): BettyCrocker, From 2dc375f85ea6f09aaf5df73ea5c7e25ee58a8d94 Mon Sep 17 00:00:00 2001 From: mlduff <46545313+mlduff@users.noreply.github.com> Date: Thu, 18 Apr 2024 07:57:58 +0930 Subject: [PATCH 5/7] Update recipe_scrapers/bergamot.py Co-authored-by: James Addison <55152140+jayaddison@users.noreply.github.com> --- recipe_scrapers/bergamot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipe_scrapers/bergamot.py b/recipe_scrapers/bergamot.py index 817805066..14a83e2b5 100644 --- a/recipe_scrapers/bergamot.py +++ b/recipe_scrapers/bergamot.py @@ -24,8 +24,7 @@ def host(cls): return "dashboard.bergamot.app" def canonical_url(self): - source_url = self.data.get("sourceUrl") - return source_url if source_url else self.url + return self.data.get("sourceUrl") or self.url def author(self): return None From 48baa93c3db9639a52cf887e369660eb691a9ab1 Mon Sep 17 00:00:00 2001 From: Michael Duffett Date: Thu, 18 Apr 2024 08:05:18 +0930 Subject: [PATCH 6/7] Use source domain as author --- recipe_scrapers/bergamot.py | 2 +- tests/legacy/test_bergamot.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe_scrapers/bergamot.py b/recipe_scrapers/bergamot.py index 14a83e2b5..ae1f88c8c 100644 --- a/recipe_scrapers/bergamot.py +++ b/recipe_scrapers/bergamot.py @@ -27,7 +27,7 @@ def canonical_url(self): return self.data.get("sourceUrl") or self.url def author(self): - return None + return self.data.get("sourceDomain") def title(self): return self.data.get("title") diff --git a/tests/legacy/test_bergamot.py b/tests/legacy/test_bergamot.py index 9a7ced876..96e6e5fa9 100644 --- a/tests/legacy/test_bergamot.py +++ b/tests/legacy/test_bergamot.py @@ -28,7 +28,7 @@ def test_title(self): ) def test_author(self): - self.assertEqual(None, self.harvester_class.author()) + self.assertEqual("elle.fr", self.harvester_class.author()) def test_total_time(self): self.assertEqual(50, self.harvester_class.total_time()) From 52058c2ad09288dd379f485418ef765d5d653709 Mon Sep 17 00:00:00 2001 From: Michael Duffett Date: Thu, 18 Apr 2024 08:11:04 +0930 Subject: [PATCH 7/7] Add cook time and tests --- recipe_scrapers/bergamot.py | 3 +++ tests/legacy/test_bergamot.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/recipe_scrapers/bergamot.py b/recipe_scrapers/bergamot.py index ae1f88c8c..0a18de522 100644 --- a/recipe_scrapers/bergamot.py +++ b/recipe_scrapers/bergamot.py @@ -69,6 +69,9 @@ def description(self): def prep_time(self): return self._get_time_value("prepTime") + def cook_time(self): + return self._get_time_value("cookTime") + def _map_list(self, data_key): output = [] for entry in self.data.get(data_key): diff --git a/tests/legacy/test_bergamot.py b/tests/legacy/test_bergamot.py index 96e6e5fa9..45df544ca 100644 --- a/tests/legacy/test_bergamot.py +++ b/tests/legacy/test_bergamot.py @@ -59,3 +59,9 @@ def test_instructions(self): def test_ratings(self): self.assertEqual(None, self.harvester_class.ratings()) + + def test_cook_time(self): + self.assertEqual(None, self.harvester_class.cook_time()) + + def test_prep_time(self): + self.assertEqual(20, self.harvester_class.prep_time())