From 3e50ee2d4fdc00fc94f8505c7bf41570daef2229 Mon Sep 17 00:00:00 2001 From: joskfg Date: Mon, 3 Feb 2020 09:53:08 +0100 Subject: [PATCH] Add context for scrape request. (#19) Add context for scrape request. --- .travis.yml | 2 + README.md | 53 ++++++++++++++++++--- src/Scraper/Events/InvalidConfiguration.php | 4 +- src/Scraper/Events/ScrapeFailed.php | 4 +- src/Scraper/Events/ScrapeRequest.php | 19 ++++---- src/Scraper/Events/Scraped.php | 7 +-- src/Scraper/helpers.php | 4 +- src/config/scraper.php | 2 - 8 files changed, 66 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9401057..48ae003 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ matrix: env: STATIC_ANALYSIS=true VALIDATE_CODING_STYLE=true - php: 7.3 env: STATIC_ANALYSIS=true VALIDATE_CODING_STYLE=true + - php: 7.4 + env: STATIC_ANALYSIS=true VALIDATE_CODING_STYLE=true - php: master env: STATIC_ANALYSIS=true VALIDATE_CODING_STYLE=false allow_failures: diff --git a/README.md b/README.md index 09379b9..b42073f 100644 --- a/README.md +++ b/README.md @@ -199,14 +199,25 @@ After configure the scraper, you will be able to request an specific scrape usin scrape('https://test.c/p/my-objective', 'Item-definition-1'); ``` +There is an optional parameter called `context` that allows you to set a context to the scrapeRequest so you will +be able to access that context in your listener. This is useful if you need some additional data (out of the scraped +data) to work in your listener. + +```php + 'my-objective']); +``` + The scrape will produce a `\Softonic\LaravelIntelligentScraper\Scraper\Events\Scraped` event if all worked as expected. So attach a listener to that event to receive the data. ```php -$event->scrapeRequest->url // Url scraped -$event->scrapeRequest->type // Request type -$event->data // Contains all the data in a [ 'fieldName' => 'value' ] format. -$event->variant // Contains the page variation sha1 hash. +$event->scrapeRequest->url; // Url scraped +$event->scrapeRequest->type; // Request type +$event->scrapeRequest->context; // Context +$event->data; // Contains all the data in a [ 'fieldName' => 'value' ] format. +$event->variant; // Contains the page variation sha1 hash. ``` All the output fields are arrays that can contain one or more results. @@ -214,8 +225,38 @@ All the output fields are arrays that can contain one or more results. If the scrape fails a `\Softonic\LaravelIntelligentScraper\Scraper\Events\ScrapeFailed` event is fired with the scrape request information. ```php -$event->scrapeRequest->url // Url scraped -$event->scrapeRequest->type // Request type +$event->scrapeRequest->url; // Url scraped +$event->scrapeRequest->type; // Request type +$event->scrapeRequest->context; // Context +``` + +To attach the listener, you can use the Laravel listener configuration like: +```php +// providers/EventServiceProvider + protected $listen = [ + Scraped::class => [ + MyListener::class, + ], + ScrapeFailed::class => [ + MyListenerFirFailedScrapes::class, + ], + ]; +``` + +But the scrapes from all types will go to that listeners. To simplify the listeners and just listen scrapes from a +single type, there is a `listeners` configuration available at scraper.php, so you can configure the listeners +with greater granularity. +```php + // config/scrapper.php + 'listeners' => [ + 'scraped' => [ + 'my-type-1' => ListenerForTypeOne::class, + 'my-type-2' => ListenerForTypeTwo::class, + ], + 'scrape-failed' => [ + 'my-type-1' => ListenerFailedForTypeOne::class, + ], + ]; ``` ## Advanced usage diff --git a/src/Scraper/Events/InvalidConfiguration.php b/src/Scraper/Events/InvalidConfiguration.php index c41fce3..d5163a7 100644 --- a/src/Scraper/Events/InvalidConfiguration.php +++ b/src/Scraper/Events/InvalidConfiguration.php @@ -25,10 +25,8 @@ public function __construct(ScrapeRequest $scrapeRequest) */ public function tags() { - $type = $this->scrapeRequest->type; - return [ - "reconfigure_type:$type", + "reconfigure_type:{$this->scrapeRequest->type}", ]; } } diff --git a/src/Scraper/Events/ScrapeFailed.php b/src/Scraper/Events/ScrapeFailed.php index 94a6669..e7eba4d 100644 --- a/src/Scraper/Events/ScrapeFailed.php +++ b/src/Scraper/Events/ScrapeFailed.php @@ -35,10 +35,8 @@ public function __construct(ScrapeRequest $scrapeRequest) */ public function tags() { - $type = $this->scrapeRequest->type; - return [ - "failed_type:$type", + "failed_type:{$this->scrapeRequest->type}", ]; } } diff --git a/src/Scraper/Events/ScrapeRequest.php b/src/Scraper/Events/ScrapeRequest.php index 9cec128..256b986 100644 --- a/src/Scraper/Events/ScrapeRequest.php +++ b/src/Scraper/Events/ScrapeRequest.php @@ -19,16 +19,23 @@ class ScrapeRequest */ public $type; + /** + * @var array + */ + public $context; + /** * Create a new event instance. * * @param string $url * @param string $type + * @param array $context */ - public function __construct(string $url, string $type) + public function __construct(string $url, string $type, array $context = []) { - $this->url = $url; - $this->type = $type; + $this->url = $url; + $this->type = $type; + $this->context = $context; } /** @@ -42,10 +49,6 @@ public function __construct(string $url, string $type) */ public function tags() { - $type = $this->type; - - return [ - "request_type:$type", - ]; + return ["request_type:{$this->type}"]; } } diff --git a/src/Scraper/Events/Scraped.php b/src/Scraper/Events/Scraped.php index 958c854..faa9010 100644 --- a/src/Scraper/Events/Scraped.php +++ b/src/Scraper/Events/Scraped.php @@ -49,12 +49,9 @@ public function __construct(ScrapeRequest $scrapeRequest, array $data, string $v */ public function tags() { - $type = $this->scrapeRequest->type; - $variant = $this->variant; - return [ - "scraped_type:$type", - "scraped_variant:$variant", + "scraped_type:{$this->scrapeRequest->type}", + "scraped_variant:{$this->variant}", ]; } } diff --git a/src/Scraper/helpers.php b/src/Scraper/helpers.php index a92998c..402f66d 100644 --- a/src/Scraper/helpers.php +++ b/src/Scraper/helpers.php @@ -8,8 +8,8 @@ function regexp($regexp) } if (!function_exists('scrape')) { - function scrape($url, $type) + function scrape($url, $type, $context = []) { - event(new \Softonic\LaravelIntelligentScraper\Scraper\Events\ScrapeRequest($url, $type)); + event(new \Softonic\LaravelIntelligentScraper\Scraper\Events\ScrapeRequest($url, $type, $context)); } } diff --git a/src/config/scraper.php b/src/config/scraper.php index 7a1f126..d587149 100644 --- a/src/config/scraper.php +++ b/src/config/scraper.php @@ -35,10 +35,8 @@ */ 'listeners' => [ 'scraped' => [ - // ], 'scrape-failed' => [ - // ], ], ];