Skip to content

Commit

Permalink
Deprecate target settings, replaced with script
Browse files Browse the repository at this point in the history
  • Loading branch information
RobertoBochet committed Jul 8, 2024
1 parent eb420d1 commit 250f611
Showing 1 changed file with 25 additions and 3 deletions.
28 changes: 25 additions & 3 deletions scraper_bot/settings/task.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,33 @@
from datetime import timedelta
from typing import Annotated
from logging import getLogger
from typing import Annotated, Any

from pydantic import BaseModel, Field, HttpUrl, PositiveInt
from pydantic import BaseModel, Field, HttpUrl, PositiveInt, model_validator


class TaskSettings(BaseModel):
name: Annotated[str, Field(description="A human readable label for teh task")]
url: Annotated[HttpUrl, Field(description="The url to the page to be scraped")]

target: Annotated[
script: Annotated[
str,
Field(
description="Javascript script to retrieve the target entities. "
"The script have to return a object(dict) or a list of them. "
"The attributes of the object will be accessible in the notification message template"
),
]
target: Annotated[
str | None,
Field(
deprecated=True,
description="'target' is deprecated, use 'script' instead. "
"Javascript script to retrieve the target entities. "
"The script have to return a object(dict) or a list of them. "
"The attributes of the object will be accessible in the notification message template",
default=None,
),
]
waitingForTarget: Annotated[
str | None, Field(description="CSS selector for a target to wait before start the scraping", default=None)
]
Expand All @@ -34,3 +46,13 @@ class TaskSettings(BaseModel):
maxPages: Annotated[
PositiveInt | None, Field(description="The maximum number of pages to scrape per task", default=None)
]

@model_validator(mode="before")
@classmethod
def retro_compatibility(cls, values: dict[str, Any]) -> dict[str, Any]:
if (target := values.get("target")) is not None:
getLogger(__package__).warning("'target' is deprecated, use 'script' instead")
if values.get("script") is None:
values["script"] = target

return values

0 comments on commit 250f611

Please sign in to comment.