From a56d226b79d4ef292c52018d1a55b7ad7374ae80 Mon Sep 17 00:00:00 2001 From: Aleksandra Kozubal Date: Fri, 11 Aug 2023 10:51:23 +0200 Subject: [PATCH 01/41] add Link provider --- app/Importers/LinkDataImporter.php | 86 +++++++++++++++++++++++++ app/Jobs/LinkDataImporterJob.php | 15 +++++ app/Services/DataImporterService.php | 2 + app/Services/MapboxGeocodingService.php | 5 +- database/seeders/ProviderSeeder.php | 2 + 5 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 app/Importers/LinkDataImporter.php create mode 100644 app/Jobs/LinkDataImporterJob.php diff --git a/app/Importers/LinkDataImporter.php b/app/Importers/LinkDataImporter.php new file mode 100644 index 00000000..db205eef --- /dev/null +++ b/app/Importers/LinkDataImporter.php @@ -0,0 +1,86 @@ +client->get("https://superpedestrian.com/locations"); + $html = $response->getBody()->getContents(); + } catch (GuzzleException) { + $this->createImportInfoDetails("400", self::getProviderName()); + + $this->stopExecution = true; + + return $this; + } + + $crawler = new Crawler($html); + $this->sections = $crawler->filter(".Main-content .sqs-row.row > .col p > strong"); + + if (count($this->sections) === 0) { + $this->createImportInfoDetails("204", self::getProviderName()); + + $this->stopExecution = true; + } + + return $this; + } + + public function transform(): void + { + if ($this->stopExecution) { + return; + } + + $existingCityProviders = []; + + foreach ($this->sections as $section) { + foreach ($section->childNodes as $node) { + $countryName = trim($node->nodeValue); + + foreach ($node->parentNode->parentNode->parentNode->childNodes as $i => $cityName) { + if ($i === 0 || !trim($cityName->nodeValue)) { + continue; + } + + $name = $cityName->nodeValue; + + $cities = []; + + if (str_contains($name, "(") && str_contains($name, ")")) { + $names = explode("(", $name)[1]; + $names = explode(")", $names)[0]; + $names = explode(", ", $names); + + foreach ($names as $name) { + $cities[] = str_replace("*", "", $name); + } + } else { + $cities[] = $name; + } + + foreach ($cities as $name) { + $provider = $this->load($name, $countryName); + + if ($provider !== "") { + $existingCityProviders[] = $provider; + } + } + } + } + } + + $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); + } +} diff --git a/app/Jobs/LinkDataImporterJob.php b/app/Jobs/LinkDataImporterJob.php new file mode 100644 index 00000000..e7a84ea3 --- /dev/null +++ b/app/Jobs/LinkDataImporterJob.php @@ -0,0 +1,15 @@ +setImportInfo($this->importInfoId)->extract()->transform(); + } +} diff --git a/app/Services/DataImporterService.php b/app/Services/DataImporterService.php index b2821304..6042bec1 100644 --- a/app/Services/DataImporterService.php +++ b/app/Services/DataImporterService.php @@ -10,6 +10,7 @@ use App\Jobs\DottDataImporterJob; use App\Jobs\HulajDataImporterJob; use App\Jobs\LimeDataImporterJob; +use App\Jobs\LinkDataImporterJob; use App\Jobs\NeuronDataImporterJob; use App\Jobs\QuickDataImporterJob; use App\Jobs\RydeDataImporterJob; @@ -40,6 +41,7 @@ public function run(string $whoRunsIt = "admin"): void new DottDataImporterJob($this->importInfoId), new HulajDataImporterJob($this->importInfoId), new LimeDataImporterJob($this->importInfoId), + new LinkDataImporterJob($this->importInfoId), new NeuronDataImporterJob($this->importInfoId), new QuickDataImporterJob($this->importInfoId), new RydeDataImporterJob($this->importInfoId), diff --git a/app/Services/MapboxGeocodingService.php b/app/Services/MapboxGeocodingService.php index 17844e25..64385f42 100644 --- a/app/Services/MapboxGeocodingService.php +++ b/app/Services/MapboxGeocodingService.php @@ -30,11 +30,10 @@ public function getCoordinatesFromApi(string $cityName, string $countryName): ar ); $coordinates = json_decode($response->getBody()->getContents(), associative: true)["features"][0]["center"]; - - return [$coordinates[1], $coordinates[0]]; } catch (Throwable $exception) { - throw new MapboxGeocodingServiceException(previous: $exception); } + + return [$coordinates[1] ?? null, $coordinates[0] ?? null]; } /** diff --git a/database/seeders/ProviderSeeder.php b/database/seeders/ProviderSeeder.php index bea4f0cd..a6e2bdd2 100644 --- a/database/seeders/ProviderSeeder.php +++ b/database/seeders/ProviderSeeder.php @@ -10,6 +10,7 @@ use App\Importers\DottDataImporter; use App\Importers\HulajDataImporter; use App\Importers\LimeDataImporter; +use App\Importers\LinkDataImporter; use App\Importers\NeuronDataImporter; use App\Importers\QuickDataImporter; use App\Importers\RydeDataImporter; @@ -31,6 +32,7 @@ public function run(): void ["name" => DottDataImporter::getProviderName(), "color" => "#f5c604"], ["name" => HulajDataImporter::getProviderName(), "color" => "#d6213f"], ["name" => LimeDataImporter::getProviderName(), "color" => "#00de00"], + ["name" => LinkDataImporter::getProviderName(), "color" => "#def700"], ["name" => NeuronDataImporter::getProviderName(), "color" => "#445261"], ["name" => QuickDataImporter::getProviderName(), "color" => "#009ac7"], ["name" => SpinDataImporter::getProviderName(), "color" => "#ff5436"], From 23ccfd622dffcbc8c58c6222ffd132e2fd9ab564 Mon Sep 17 00:00:00 2001 From: Aleksandra Kozubal Date: Mon, 14 Aug 2023 13:40:44 +0200 Subject: [PATCH 02/41] add BinBin provider --- app/Importers/BinBinDataImporter.php | 72 +++++++++++++++++++++++++++ app/Importers/DataImporter.php | 8 +++ app/Jobs/BinBinDataImporterJob.php | 15 ++++++ app/Services/DataImporterService.php | 2 + database/seeders/ProviderSeeder.php | 2 + public/providers/binbin.png | Bin 0 -> 1696 bytes 6 files changed, 99 insertions(+) create mode 100644 app/Importers/BinBinDataImporter.php create mode 100644 app/Jobs/BinBinDataImporterJob.php create mode 100644 public/providers/binbin.png diff --git a/app/Importers/BinBinDataImporter.php b/app/Importers/BinBinDataImporter.php new file mode 100644 index 00000000..be57acf6 --- /dev/null +++ b/app/Importers/BinBinDataImporter.php @@ -0,0 +1,72 @@ +client->get("https://www.binbin.tech/lokasyonlar"); + $html = $response->getBody()->getContents(); + } catch (GuzzleException) { + $this->createImportInfoDetails("400", self::getProviderName()); + + $this->stopExecution = true; + + return $this; + } + + $crawler = new Crawler($html); + $this->sections = $crawler->filter(".location-wrapper .cities-information .div-block-35"); + + if (count($this->sections) === 0) { + $this->createImportInfoDetails("204", self::getProviderName()); + + $this->stopExecution = true; + } + + return $this; + } + + public function transform(): void + { + if ($this->stopExecution) { + return; + } + + $existingCityProviders = []; + + foreach ($this->sections as $section) { + $data = explode("|", $section->nodeValue); + + if (trim($data[1]) === "Türkiye") { + $countryName = "Turkey"; + } else { + $countryName = $this->translate(trim($data[1]), "en"); + } + + if (trim($data[0]) === "Uşak") { + $cityName = "Usak"; + } elseif (trim($data[0]) === "Murter") { + continue; + } else { + $cityName = $this->translate(trim($data[0]), "en"); + } + + $provider = $this->load($cityName, $countryName); + + if ($provider !== "") { + $existingCityProviders[] = $provider; + } + } + $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); + } +} diff --git a/app/Importers/DataImporter.php b/app/Importers/DataImporter.php index ed83994f..cc62b12c 100644 --- a/app/Importers/DataImporter.php +++ b/app/Importers/DataImporter.php @@ -12,6 +12,7 @@ use App\Models\ImportInfoDetail; use App\Services\MapboxGeocodingService; use GuzzleHttp\Client; +use Stichoza\GoogleTranslate\GoogleTranslate; abstract class DataImporter { @@ -48,6 +49,13 @@ public static function getProviderName(): string return $classNameParts[0]; } + public function translate(string $word, $language): string + { + $translate = new GoogleTranslate($language); + + return $translate->translate($word); + } + protected function countryNotFound(string $cityName, string $countryName): void { CityWithoutAssignedCountry::query()->updateOrCreate( diff --git a/app/Jobs/BinBinDataImporterJob.php b/app/Jobs/BinBinDataImporterJob.php new file mode 100644 index 00000000..eab48525 --- /dev/null +++ b/app/Jobs/BinBinDataImporterJob.php @@ -0,0 +1,15 @@ +setImportInfo($this->importInfoId)->extract()->transform(); + } +} diff --git a/app/Services/DataImporterService.php b/app/Services/DataImporterService.php index 6042bec1..431cfe29 100644 --- a/app/Services/DataImporterService.php +++ b/app/Services/DataImporterService.php @@ -4,6 +4,7 @@ namespace App\Services; +use App\Jobs\BinBinDataImporterJob; use App\Jobs\BirdDataImporterJob; use App\Jobs\BitMobilityDataImporterJob; use App\Jobs\BoltDataImporterJob; @@ -35,6 +36,7 @@ public function run(string $whoRunsIt = "admin"): void $this->importInfoId = $importInfo->id; Bus::batch([ + new BinBinDataImporterJob($this->importInfoId), new BirdDataImporterJob($this->importInfoId), new BitMobilityDataImporterJob($this->importInfoId), new BoltDataImporterJob($this->importInfoId), diff --git a/database/seeders/ProviderSeeder.php b/database/seeders/ProviderSeeder.php index a6e2bdd2..1229898c 100644 --- a/database/seeders/ProviderSeeder.php +++ b/database/seeders/ProviderSeeder.php @@ -4,6 +4,7 @@ namespace Database\Seeders; +use App\Importers\BinBinDataImporter; use App\Importers\BirdDataImporter; use App\Importers\BitMobilityDataImporter; use App\Importers\BoltDataImporter; @@ -26,6 +27,7 @@ class ProviderSeeder extends Seeder public function run(): void { $providers = [ + ["name" => BinBinDataImporter::getProviderName(), "color" => "#3dbcc8"], ["name" => BirdDataImporter::getProviderName(), "color" => "#26ccf0"], ["name" => BitMobilityDataImporter::getProviderName(), "color" => "#8da6e3"], ["name" => BoltDataImporter::getProviderName(), "color" => "#24f0a0"], diff --git a/public/providers/binbin.png b/public/providers/binbin.png new file mode 100644 index 0000000000000000000000000000000000000000..55756c4ce8d045dabd946753ef5ff6592e5b6dd5 GIT binary patch literal 1696 zcmV;R24DG!P)8QOsbjgVyH|gFzBlfk zd!Fz2bML<2d*>VOAD`zr&w1YGbH1N*&U2oNx-I$pD1Cr>rurlGS#^hcn>w!_x+lTq z+lifP`=ooOx-(Vh)#?+8?oH~fesG^4pS1~>2m7RZO``jd zx~L!eYnU-W-Jb%^O~w}iR}XOCDTR7Aa6>XR3;6mV_a3D4-a5&-R~i-qQJH(ietA2( z%nc84->Js-E^`C5bCAxv6ZlUuz8AQCfcs80wg4L|bKe-`#shO+L~I4F0CtZVehOF? z5zh{4->F7K908UCe;6}-2v{8vcMNjx69%a~BJNSo1>Ol91|E)xgM-|2IuH>%)WtNp z{0v}oMC=;$zCE3%&H&~Eha+NlFZmJ?aRB(;B)F9vhG+_dT@f*IEc&|a^IG5(V0T0u zn#kIS5O}l?9OeP@Xza|=&i_#ho9C!sR38Ig2A&3XsRz~Hs@JI}wIp*+=Ky=7dd38d zn5*8TK136ShJj~*L+Zoo=hWA<)mOpE>Q(A}>Px^LnnLSf=fHhyCvvpmJ6rv@x>~)Y zEde#urRrVki@-kMU%&x%yZRk9ug)un^fq`({ZJ2L7pgCeS+i4peG7xGR-bFZVUPOB zmM%nJ0;2$ z$H3uB19qM^DPvrnV3S5N%&c%JBho~$S$!4s@2IdweRDFNg_^Xlsa_0x`4~8?Qr~q9 z>)YTmU}Z913#pl-X9DYixs&9u5cmr8cHT3jJ( z0IUI)0ZV~z0{auW0B)`5?uA1%wp~GE>uT!Vkb^vNL>($*A5G-$O>~~sSzltOdSN1ef%#ix}wj?t5 zC;C27(YLeO$a+$y9l_HIZs_(8%n(KwVH<(0K=0-YxGc+d8kW zn8DFk8KZa$a8tHGJsJ33Mc;kJbK9IZ70B!axUFMThLz-HeH%Mr3Y_=j%G_7g@7pQ8 zZTfeY0T0n!@s!va5u1y#4>Q3t3Mz=yUO??r)|#bD^7@r zP3q6c!^5?}wKg61*}#cibgfRtjvME7@CEY9Z4u4a^s;&wn3Kqi;_ispRE$kq*aY|t z&2>l(bLj54p94Ql#*a7Wtp|SDW&R|(JMKA}cU5BgoEH&K(tLRbCZXdI;L1+vUgCJl zd85aR<-lJiv3x)9;fToo;b_`&9sm*XN1FKm#01u_1Kt-AFO?IW{E=PLOy`$JhwEax z9L^JL11^n-4VC$6$$3W|2O{FHzye?e&Fy)HBfth=NklB~bO16;ODJiBh}a(y9|10> z`OmK`YCv=JSph7Fh`)7FIW4wUFEDK`Hy6 Date: Tue, 22 Aug 2023 10:06:26 +0200 Subject: [PATCH 03/41] show providers name, url, color --- app/Http/Controllers/ProviderController.php | 50 ++++ app/Http/Requests/ProviderRequest.php | 28 +++ app/Models/Provider.php | 9 +- database/seeders/ProviderSeeder.php | 2 + resources/js/Pages/Cities/Index.vue | 8 + resources/js/Pages/Providers/Index.vue | 218 ++++++++++++++++ resources/js/Shared/Components/Provider.vue | 235 ++++++++++++++++++ .../js/Shared/Layout/AdminNavigation.vue | 3 +- routes/web.php | 2 + 9 files changed, 553 insertions(+), 2 deletions(-) create mode 100644 app/Http/Controllers/ProviderController.php create mode 100644 app/Http/Requests/ProviderRequest.php create mode 100644 resources/js/Pages/Providers/Index.vue create mode 100644 resources/js/Shared/Components/Provider.vue diff --git a/app/Http/Controllers/ProviderController.php b/app/Http/Controllers/ProviderController.php new file mode 100644 index 00000000..6a1ec06c --- /dev/null +++ b/app/Http/Controllers/ProviderController.php @@ -0,0 +1,50 @@ +paginate(15) + ->withQueryString(); + // $countries = Country::all(); + + return Inertia::render("Providers/Index", [ + // "cities" => CityResource::collection($cities), + "providers" => ProviderResource::collection($providers), + // "countries" => CountryResource::collection($countries), + ]); + } + + public function store(ProviderRequest $request): void + { + Provider::query()->create($request->validated()); + } + + public function update(ProviderRequest $request, Provider $provider): void + { + $provider->update($request->validated()); + } + + public function destroy(Provider $provider): void + { + $provider->delete(); + } +} diff --git a/app/Http/Requests/ProviderRequest.php b/app/Http/Requests/ProviderRequest.php new file mode 100644 index 00000000..69028413 --- /dev/null +++ b/app/Http/Requests/ProviderRequest.php @@ -0,0 +1,28 @@ + ["required", "string", "regex:/^[A-Z\s]/", "max:100", $this->uniqueRuleForProvider("name")], + "url" => ["string"], + "color" => ["required", "string"], + ]; + } + + protected function uniqueRuleForProvider(string $column): Unique + { + $currentProviderId = $this->route(param: "provider"); + + return Rule::unique(table: "providers", column: $column)->ignore($currentProviderId); + } +} diff --git a/app/Models/Provider.php b/app/Models/Provider.php index cf5841b6..4b89c498 100644 --- a/app/Models/Provider.php +++ b/app/Models/Provider.php @@ -5,6 +5,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Model; +use Illuminate\Database\Eloquent\Relations\BelongsTo; /** * @property string $name @@ -17,7 +18,13 @@ class Provider extends Model protected $primaryKey = "name"; protected $keyType = "string"; - public function cityProvider() + protected $fillable = [ + "name", + "url", + "color", + ]; + + public function cityProvider(): BelongsTo { return $this->belongsTo(CityProvider::class); } diff --git a/database/seeders/ProviderSeeder.php b/database/seeders/ProviderSeeder.php index 2c5f5141..610b2680 100644 --- a/database/seeders/ProviderSeeder.php +++ b/database/seeders/ProviderSeeder.php @@ -28,6 +28,8 @@ class ProviderSeeder extends Seeder { public function run(): void { + + $providers = [ ["name" => BinBinDataImporter::getProviderName(), "color" => "#3dbcc8"], ["name" => BerylDataImporter::getProviderName(), "color" => "#00e3c2"], ["name" => BirdDataImporter::getProviderName(), "color" => "#26ccf0"], diff --git a/resources/js/Pages/Cities/Index.vue b/resources/js/Pages/Cities/Index.vue index 84c567a1..c3026924 100644 --- a/resources/js/Pages/Cities/Index.vue +++ b/resources/js/Pages/Cities/Index.vue @@ -156,6 +156,8 @@ function toggleSortDialog() { {{ __('Create city') }} + +
@@ -169,6 +171,8 @@ function toggleSortDialog() {
+ +
@@ -196,6 +200,8 @@ function toggleSortDialog() {
+ +
@@ -213,10 +219,12 @@ function toggleSortDialog() { {{ __('Providers') }} + +
diff --git a/resources/js/Pages/Providers/Index.vue b/resources/js/Pages/Providers/Index.vue new file mode 100644 index 00000000..ce287f1f --- /dev/null +++ b/resources/js/Pages/Providers/Index.vue @@ -0,0 +1,218 @@ + + + diff --git a/resources/js/Shared/Components/Provider.vue b/resources/js/Shared/Components/Provider.vue new file mode 100644 index 00000000..ad8e0405 --- /dev/null +++ b/resources/js/Shared/Components/Provider.vue @@ -0,0 +1,235 @@ + + + diff --git a/resources/js/Shared/Layout/AdminNavigation.vue b/resources/js/Shared/Layout/AdminNavigation.vue index c84ed209..354c07b6 100644 --- a/resources/js/Shared/Layout/AdminNavigation.vue +++ b/resources/js/Shared/Layout/AdminNavigation.vue @@ -1,6 +1,6 @@