diff --git a/app/Http/Controllers/GithubController.php b/app/Http/Controllers/GithubController.php index a7a6845a..c3898900 100644 --- a/app/Http/Controllers/GithubController.php +++ b/app/Http/Controllers/GithubController.php @@ -4,8 +4,13 @@ namespace App\Http\Controllers; +use App\Http\Integrations\GithubConnector; +use App\Jobs\FetchDataFromApi; use App\Models\User; use App\Services\AssignUserToOrganizationsService; +use App\Services\FetchRepositoriesService; +use App\Services\FetchWorkflowJobsService; +use App\Services\FetchWorkflowRunsService; use Illuminate\Support\Facades\Auth; use Laravel\Socialite\Facades\Socialite; use Symfony\Component\HttpFoundation\RedirectResponse; @@ -40,4 +45,19 @@ public function callback(): RedirectResponse return redirect("/"); } + + public function fetchData($organizationId): RedirectResponse + { + $userId = Auth::user()->id; + + FetchDataFromApi::dispatch( + (int)$organizationId, + $githubConnector = new GithubConnector(), + new FetchRepositoriesService($githubConnector, $userId), + new FetchWorkflowRunsService($githubConnector, $userId), + new FetchWorkflowJobsService($githubConnector, $userId), + ); + + return redirect()->back(); + } } diff --git a/app/Http/Integrations/GithubConnector.php b/app/Http/Integrations/GithubConnector.php index c32e5ce5..11a28d4a 100644 --- a/app/Http/Integrations/GithubConnector.php +++ b/app/Http/Integrations/GithubConnector.php @@ -4,12 +4,20 @@ namespace App\Http\Integrations; +use Illuminate\Support\Facades\Cache; use Saloon\Http\Connector; +use Saloon\Http\Response; +use Saloon\RateLimitPlugin\Contracts\RateLimitStore; +use Saloon\RateLimitPlugin\Helpers\RetryAfterHelper; +use Saloon\RateLimitPlugin\Limit; +use Saloon\RateLimitPlugin\Stores\LaravelCacheStore; +use Saloon\RateLimitPlugin\Traits\HasRateLimits; use Saloon\Traits\Plugins\AlwaysThrowOnErrors; class GithubConnector extends Connector { use AlwaysThrowOnErrors; + use HasRateLimits; public function resolveBaseUrl(): string { @@ -23,4 +31,27 @@ protected function defaultHeaders(): array "Accept" => "application/json", ]; } + + protected function resolveLimits(): array + { + return [ + Limit::allow(config("services.rate_limit"))->everyHour(), + ]; + } + + protected function resolveRateLimitStore(): RateLimitStore + { + return new LaravelCacheStore(Cache::store("redis")); + } + + protected function handleTooManyAttempts(Response $response, Limit $limit): void + { + if ($response->status() !== 429 && $response->status() !== 403) { + return; + } + + $limit->exceeded( + releaseInSeconds: RetryAfterHelper::parse($response->header("Retry-After")), + ); + } } diff --git a/app/Http/Integrations/Requests/GetRepositoriesRequest.php b/app/Http/Integrations/Requests/GetRepositoriesRequest.php index a2fde275..c4e1e4e7 100644 --- a/app/Http/Integrations/Requests/GetRepositoriesRequest.php +++ b/app/Http/Integrations/Requests/GetRepositoriesRequest.php @@ -7,8 +7,8 @@ use App\DTO\OrganizationDTO; use App\DTO\RepositoryDTO; use App\Models\Organization; +use App\Models\User; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Auth; use Saloon\Enums\Method; use Saloon\Http\Request; use Saloon\Http\Response; @@ -19,11 +19,12 @@ class GetRepositoriesRequest extends Request public function __construct( protected OrganizationDTO $organizationDto, + protected User $user, ) {} public function resolveEndpoint(): string { - return "/orgs" . $this->organizationDto->name . "/repos"; + return "/orgs/" . $this->organizationDto->name . "/repos"; } public function createDtoFromResponse(Response $response): Collection @@ -49,7 +50,7 @@ public function createDtoFromResponse(Response $response): Collection protected function defaultHeaders(): array { return [ - "Authorization" => "Bearer " . Auth::user()->github_token, + "Authorization" => "Bearer " . $this->user->github_token, ]; } } diff --git a/app/Http/Integrations/Requests/GetWorkflowJobsRequest.php b/app/Http/Integrations/Requests/GetWorkflowJobsRequest.php index 02ec832f..5ef46c76 100644 --- a/app/Http/Integrations/Requests/GetWorkflowJobsRequest.php +++ b/app/Http/Integrations/Requests/GetWorkflowJobsRequest.php @@ -6,11 +6,11 @@ use App\DTO\WorkflowJobDTO; use App\DTO\WorkflowRunDTO; +use App\Models\User; use App\Models\WorkflowRun; use App\Services\CalculateJobTimeService; use App\Services\GetRunnerDataService; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Auth; use Saloon\Enums\Method; use Saloon\Http\Request; use Saloon\Http\Response; @@ -23,6 +23,7 @@ public function __construct( protected WorkflowRunDTO $workflowRunDto, protected string $organizationName, protected string $repositoryName, + protected User $user, ) {} public function resolveEndpoint(): string @@ -62,7 +63,7 @@ public function createDtoFromResponse(Response $response): Collection protected function defaultHeaders(): array { return [ - "Authorization" => "Bearer " . Auth::user()->github_token, + "Authorization" => "Bearer " . $this->user->github_token, ]; } } diff --git a/app/Http/Integrations/Requests/GetWorkflowRunsRequest.php b/app/Http/Integrations/Requests/GetWorkflowRunsRequest.php index 44c831ea..7ea059de 100644 --- a/app/Http/Integrations/Requests/GetWorkflowRunsRequest.php +++ b/app/Http/Integrations/Requests/GetWorkflowRunsRequest.php @@ -8,9 +8,9 @@ use App\DTO\WorkflowRunDTO; use App\Models\Organization; use App\Models\Repository; +use App\Models\User; use DateTime; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Auth; use Saloon\Enums\Method; use Saloon\Http\Request; use Saloon\Http\Response; @@ -21,6 +21,7 @@ class GetWorkflowRunsRequest extends Request public function __construct( protected RepositoryDTO $repositoryDto, + protected User $user, ) {} public function resolveEndpoint(): string @@ -55,7 +56,7 @@ public function createDtoFromResponse(Response $response): Collection protected function defaultHeaders(): array { return [ - "Authorization" => "Bearer " . Auth::user()->github_token, + "Authorization" => "Bearer " . $this->user->github_token, ]; } } diff --git a/app/Jobs/FetchDataFromApi.php b/app/Jobs/FetchDataFromApi.php new file mode 100644 index 00000000..f9905d67 --- /dev/null +++ b/app/Jobs/FetchDataFromApi.php @@ -0,0 +1,63 @@ +where("id", $this->organizationId)->firstOrFail(); + $organizationDto = new OrganizationDTO( + $organization->name, + $organization->github_id, + $organization->avatar_url, + ); + + $repositories = collect(); + $repositories = $this->fetchRepositoriesService->fetchRepositories($organizationDto); + + $workflowRuns = collect(); + + foreach ($repositories as $repositoryDto) { + $workflowRuns = $workflowRuns->union($this->fetchWorkflowRunsService->fetchWorkflowRuns($repositoryDto)); + } + + foreach ($workflowRuns as $workflowRunDto) { + $this->fetchWorkflowJobsService->fetchWorkflowJobs($workflowRunDto); + } + } + + public function middleware(): array + { + return [new ApiRateLimited()]; + } +} diff --git a/app/Services/FetchRepositoriesService.php b/app/Services/FetchRepositoriesService.php index c69d71c9..6b00e097 100644 --- a/app/Services/FetchRepositoriesService.php +++ b/app/Services/FetchRepositoriesService.php @@ -13,19 +13,19 @@ use App\Models\User; use Exception; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Auth; use Illuminate\Validation\UnauthorizedException; class FetchRepositoriesService { public function __construct( protected GithubConnector $githubConnector, + protected int $userId, ) {} - public function fetchRepositories(OrganizationDTO $organizationDto): void + public function fetchRepositories(OrganizationDTO $organizationDto): Collection { $organization = Organization::query()->where("github_id", $organizationDto->githubId)->firstOrFail(); - $user = User::query()->where("id", Auth::user()->id)->firstOrFail(); + $user = User::query()->where("id", $this->userId)->firstOrFail(); $userOrganizationExists = $user->organizations() ->where("organization_id", $organization->id) @@ -34,11 +34,13 @@ public function fetchRepositories(OrganizationDTO $organizationDto): void if ($userOrganizationExists) { try { - $request = new GetRepositoriesRequest($organizationDto); + $request = new GetRepositoriesRequest($organizationDto, $user); $response = $this->githubConnector->send($request); $this->storeRepositories($response->dto()); + + return $response->dto(); } catch (Exception $exception) { throw new FetchingRepositoriesErrorException( message: "Error ocurred while fetching repositories", diff --git a/app/Services/FetchWorkflowJobsService.php b/app/Services/FetchWorkflowJobsService.php index 695a902c..0b88576d 100644 --- a/app/Services/FetchWorkflowJobsService.php +++ b/app/Services/FetchWorkflowJobsService.php @@ -15,13 +15,13 @@ use App\Models\WorkflowRun; use Exception; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Auth; use Illuminate\Validation\UnauthorizedException; class FetchWorkflowJobsService { public function __construct( protected GithubConnector $githubConnector, + protected int $userId, ) {} public function fetchWorkflowJobs(WorkflowRunDTO $workflowRunDto): void @@ -40,7 +40,7 @@ public function fetchWorkflowJobs(WorkflowRunDTO $workflowRunDto): void ->where("id", $repository->organization_id) ->firstOrFail(); - $user = User::query()->where("id", Auth::user()->id)->firstOrFail(); + $user = User::query()->where("id", $this->userId)->firstOrFail(); $userOrganizationExists = $user->organizations() ->where("organization_id", $organization->id) @@ -49,7 +49,7 @@ public function fetchWorkflowJobs(WorkflowRunDTO $workflowRunDto): void if ($userOrganizationExists) { try { - $request = new GetWorkflowJobsRequest($workflowRunDto, $organization->name, $repository->name); + $request = new GetWorkflowJobsRequest($workflowRunDto, $organization->name, $repository->name, $user); $response = $this->githubConnector->send($request); diff --git a/app/Services/FetchWorkflowRunsService.php b/app/Services/FetchWorkflowRunsService.php index a653db8d..442cf3ed 100644 --- a/app/Services/FetchWorkflowRunsService.php +++ b/app/Services/FetchWorkflowRunsService.php @@ -13,19 +13,19 @@ use App\Models\WorkflowRun; use Exception; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Auth; use Illuminate\Validation\UnauthorizedException; class FetchWorkflowRunsService { public function __construct( protected GithubConnector $githubConnector, + protected int $userId, ) {} - public function fetchWorkflowRuns(RepositoryDTO $repositoryDto): void + public function fetchWorkflowRuns(RepositoryDTO $repositoryDto): Collection { $organization = Organization::query()->where("id", $repositoryDto->organizationId)->firstOrFail(); - $user = User::query()->where("id", Auth::user()->id)->firstOrFail(); + $user = User::query()->where("id", $this->userId)->firstOrFail(); $userOrganizationExists = $user->organizations() ->where("organization_id", $organization->id) @@ -34,11 +34,13 @@ public function fetchWorkflowRuns(RepositoryDTO $repositoryDto): void if ($userOrganizationExists) { try { - $request = new GetWorkflowRunsRequest($repositoryDto); + $request = new GetWorkflowRunsRequest($repositoryDto, $user); $response = $this->githubConnector->send($request); $this->storeWorkflowRuns($response->dto()); + + return $response->dto(); } catch (Exception $exception) { throw new FetchingWorkflowRunsErrorException( message: "Error ocurred while fetching workflow runs", diff --git a/composer.json b/composer.json index 4cc90431..9298f493 100755 --- a/composer.json +++ b/composer.json @@ -14,6 +14,7 @@ "laravel/socialite": "^5.14", "laravel/tinker": "^2.9", "nesbot/carbon": "^3.6", + "saloonphp/rate-limit-plugin": "^2.0", "saloonphp/saloon": "^3.0" }, "require-dev": { diff --git a/composer.lock b/composer.lock index 316ade4c..71dbabcb 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fa706c72c030c2fff8e237e2f57575bf", + "content-hash": "74c972e582a60bcb746b08f19f4b4912", "packages": [ { "name": "brick/math", @@ -3686,6 +3686,60 @@ ], "time": "2024-04-27T21:32:50+00:00" }, + { + "name": "saloonphp/rate-limit-plugin", + "version": "v2.0.0", + "source": { + "type": "git", + "url": "https://github.com/saloonphp/rate-limit-plugin.git", + "reference": "651be17f6bd835cabc8bb2fea63c473f380dc0ca" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/saloonphp/rate-limit-plugin/zipball/651be17f6bd835cabc8bb2fea63c473f380dc0ca", + "reference": "651be17f6bd835cabc8bb2fea63c473f380dc0ca", + "shasum": "" + }, + "require": { + "php": "^8.1", + "saloonphp/saloon": "^3.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.5", + "mockery/mockery": "^1.5", + "orchestra/testbench": "^8.5", + "pestphp/pest": "^2.6", + "phpstan/phpstan": "^1.9", + "predis/predis": "^2.1", + "psr/simple-cache": "^3.0", + "spatie/ray": "^1.33" + }, + "type": "library", + "autoload": { + "psr-4": { + "Saloon\\RateLimitPlugin\\": "src/", + "Saloon\\RateLimitPlugin\\Tests\\": "tests/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Sam Carré", + "email": "29132017+Sammyjo20@users.noreply.github.com", + "role": "Developer" + } + ], + "description": "Handle rate limits beautifully in your Saloon API integrations or SDKs", + "homepage": "https://github.com/sammyjo20", + "support": { + "issues": "https://github.com/saloonphp/rate-limit-plugin/issues", + "source": "https://github.com/saloonphp/rate-limit-plugin/tree/v2.0.0" + }, + "time": "2023-10-02T16:30:26+00:00" + }, { "name": "saloonphp/saloon", "version": "v3.9.1", diff --git a/config/services.php b/config/services.php index c9f8d4dd..f9a680e9 100755 --- a/config/services.php +++ b/config/services.php @@ -55,4 +55,5 @@ ], ], ], + "rate_limit" => 5000, ]; diff --git a/routes/web.php b/routes/web.php index 5bd938fa..b8f31766 100755 --- a/routes/web.php +++ b/routes/web.php @@ -20,3 +20,5 @@ Route::get("/auth/redirect", [GithubController::class, "redirect"]); Route::get("/auth/callback", [GithubController::class, "callback"]); + +Route::get("/{organizationId}/fetch", [GithubController::class, "fetchData"]); diff --git a/tests/Feature/FetchRepositoriesTest.php b/tests/Feature/FetchRepositoriesTest.php index 95ef0bcc..627612a2 100644 --- a/tests/Feature/FetchRepositoriesTest.php +++ b/tests/Feature/FetchRepositoriesTest.php @@ -41,7 +41,7 @@ protected function setUp(): void $this->organization->github_id, $this->organization->avatar_url, ); - $this->fetchRepositoriesService = new FetchRepositoriesService($this->githubConnector); + $this->fetchRepositoriesService = new FetchRepositoriesService($this->githubConnector, $this->user->id); $this->actingAs($this->user); MockClient::destroyGlobal(); diff --git a/tests/Feature/FetchWorkflowJobsTest.php b/tests/Feature/FetchWorkflowJobsTest.php index e6e50f31..517afded 100644 --- a/tests/Feature/FetchWorkflowJobsTest.php +++ b/tests/Feature/FetchWorkflowJobsTest.php @@ -49,7 +49,7 @@ protected function setUp(): void new DateTime($this->workflowRun->github_created_at), ); $this->repository = Repository::query()->where("id", $this->workflowRun->repository_id)->firstOrFail(); - $this->fetchWorkflowJobsService = new FetchWorkflowJobsService($this->githubConnector); + $this->fetchWorkflowJobsService = new FetchWorkflowJobsService($this->githubConnector, $this->user->id); $this->actingAs($this->user); MockClient::destroyGlobal(); diff --git a/tests/Feature/FetchWorkflowRunsTest.php b/tests/Feature/FetchWorkflowRunsTest.php index edbbb4a3..0ee59439 100644 --- a/tests/Feature/FetchWorkflowRunsTest.php +++ b/tests/Feature/FetchWorkflowRunsTest.php @@ -44,7 +44,7 @@ protected function setUp(): void $this->repository->organization_id, $this->repository->is_private, ); - $this->fetchWorkflowRunsService = new FetchWorkflowRunsService($this->githubConnector); + $this->fetchWorkflowRunsService = new FetchWorkflowRunsService($this->githubConnector, $this->user->id); $this->actingAs($this->user); MockClient::destroyGlobal(); diff --git a/tests/Feature/RateLimiterTest.php b/tests/Feature/RateLimiterTest.php index 5666f5cd..e272d81e 100644 --- a/tests/Feature/RateLimiterTest.php +++ b/tests/Feature/RateLimiterTest.php @@ -8,6 +8,7 @@ use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Config; +use Illuminate\Support\Facades\Redis; use Saloon\Config as SaloonConfig; use Saloon\Enums\Method; use Saloon\Http\Faking\MockClient; @@ -45,6 +46,12 @@ public function resolveEndpoint(): string MockClient::destroyGlobal(); } + protected function tearDown(): void + { + Redis::flushall(); + parent::tearDown(); + } + public function testRateLimitNotReached(): void { $mockClient = new MockClient([