Skip to content
This repository has been archived by the owner. It is now read-only.

Commit

Permalink
Add context for scrape request. (#19)
Browse files Browse the repository at this point in the history
Add context for scrape request.
  • Loading branch information
joskfg authored Feb 3, 2020
1 parent b90f468 commit 3e50ee2
Show file tree
Hide file tree
Showing 8 changed files with 66 additions and 29 deletions.
2 changes: 2 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ matrix:
env: STATIC_ANALYSIS=true VALIDATE_CODING_STYLE=true
- php: 7.3
env: STATIC_ANALYSIS=true VALIDATE_CODING_STYLE=true
- php: 7.4
env: STATIC_ANALYSIS=true VALIDATE_CODING_STYLE=true
- php: master
env: STATIC_ANALYSIS=true VALIDATE_CODING_STYLE=false
allow_failures:
Expand Down
53 changes: 47 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -199,23 +199,64 @@ After configure the scraper, you will be able to request an specific scrape usin
scrape('https://test.c/p/my-objective', 'Item-definition-1');
```

There is an optional parameter called `context` that allows you to set a context to the scrapeRequest so you will
be able to access that context in your listener. This is useful if you need some additional data (out of the scraped
data) to work in your listener.

```php
<?php

scrape('https://test.c/p/my-objective', 'Item-definition-1', ['id' => 'my-objective']);
```

The scrape will produce a `\Softonic\LaravelIntelligentScraper\Scraper\Events\Scraped` event if all worked as expected.
So attach a listener to that event to receive the data.

```php
$event->scrapeRequest->url // Url scraped
$event->scrapeRequest->type // Request type
$event->data // Contains all the data in a [ 'fieldName' => 'value' ] format.
$event->variant // Contains the page variation sha1 hash.
$event->scrapeRequest->url; // Url scraped
$event->scrapeRequest->type; // Request type
$event->scrapeRequest->context; // Context
$event->data; // Contains all the data in a [ 'fieldName' => 'value' ] format.
$event->variant; // Contains the page variation sha1 hash.
```

All the output fields are arrays that can contain one or more results.

If the scrape fails a `\Softonic\LaravelIntelligentScraper\Scraper\Events\ScrapeFailed` event is fired with the
scrape request information.
```php
$event->scrapeRequest->url // Url scraped
$event->scrapeRequest->type // Request type
$event->scrapeRequest->url; // Url scraped
$event->scrapeRequest->type; // Request type
$event->scrapeRequest->context; // Context
```

To attach the listener, you can use the Laravel listener configuration like:
```php
// providers/EventServiceProvider
protected $listen = [
Scraped::class => [
MyListener::class,
],
ScrapeFailed::class => [
MyListenerFirFailedScrapes::class,
],
];
```

But the scrapes from all types will go to that listeners. To simplify the listeners and just listen scrapes from a
single type, there is a `listeners` configuration available at scraper.php, so you can configure the listeners
with greater granularity.
```php
// config/scrapper.php
'listeners' => [
'scraped' => [
'my-type-1' => ListenerForTypeOne::class,
'my-type-2' => ListenerForTypeTwo::class,
],
'scrape-failed' => [
'my-type-1' => ListenerFailedForTypeOne::class,
],
];
```

## Advanced usage
Expand Down
4 changes: 1 addition & 3 deletions src/Scraper/Events/InvalidConfiguration.php
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,8 @@ public function __construct(ScrapeRequest $scrapeRequest)
*/
public function tags()
{
$type = $this->scrapeRequest->type;

return [
"reconfigure_type:$type",
"reconfigure_type:{$this->scrapeRequest->type}",
];
}
}
4 changes: 1 addition & 3 deletions src/Scraper/Events/ScrapeFailed.php
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,8 @@ public function __construct(ScrapeRequest $scrapeRequest)
*/
public function tags()
{
$type = $this->scrapeRequest->type;

return [
"failed_type:$type",
"failed_type:{$this->scrapeRequest->type}",
];
}
}
19 changes: 11 additions & 8 deletions src/Scraper/Events/ScrapeRequest.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,23 @@ class ScrapeRequest
*/
public $type;

/**
* @var array
*/
public $context;

/**
* Create a new event instance.
*
* @param string $url
* @param string $type
* @param array $context
*/
public function __construct(string $url, string $type)
public function __construct(string $url, string $type, array $context = [])
{
$this->url = $url;
$this->type = $type;
$this->url = $url;
$this->type = $type;
$this->context = $context;
}

/**
Expand All @@ -42,10 +49,6 @@ public function __construct(string $url, string $type)
*/
public function tags()
{
$type = $this->type;

return [
"request_type:$type",
];
return ["request_type:{$this->type}"];
}
}
7 changes: 2 additions & 5 deletions src/Scraper/Events/Scraped.php
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,9 @@ public function __construct(ScrapeRequest $scrapeRequest, array $data, string $v
*/
public function tags()
{
$type = $this->scrapeRequest->type;
$variant = $this->variant;

return [
"scraped_type:$type",
"scraped_variant:$variant",
"scraped_type:{$this->scrapeRequest->type}",
"scraped_variant:{$this->variant}",
];
}
}
4 changes: 2 additions & 2 deletions src/Scraper/helpers.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ function regexp($regexp)
}

if (!function_exists('scrape')) {
function scrape($url, $type)
function scrape($url, $type, $context = [])
{
event(new \Softonic\LaravelIntelligentScraper\Scraper\Events\ScrapeRequest($url, $type));
event(new \Softonic\LaravelIntelligentScraper\Scraper\Events\ScrapeRequest($url, $type, $context));
}
}
2 changes: 0 additions & 2 deletions src/config/scraper.php
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,8 @@
*/
'listeners' => [
'scraped' => [
//
],
'scrape-failed' => [
//
],
],
];

0 comments on commit 3e50ee2

Please sign in to comment.