From 6bacd26a2b424948788b6b0f88ac703352df0288 Mon Sep 17 00:00:00 2001 From: BarTracz Date: Fri, 8 Dec 2023 16:01:43 +0100 Subject: [PATCH 1/3] link refactor --- app/Importers/LinkDataImporter.php | 68 +++++++++++++++------------- app/Services/DataImporterService.php | 42 ++++++++--------- 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/app/Importers/LinkDataImporter.php b/app/Importers/LinkDataImporter.php index db205eef..496c91da 100644 --- a/app/Importers/LinkDataImporter.php +++ b/app/Importers/LinkDataImporter.php @@ -10,7 +10,6 @@ class LinkDataImporter extends DataImporter { protected Crawler $sections; - private string $countryName = ""; public function extract(): static { @@ -26,7 +25,7 @@ public function extract(): static } $crawler = new Crawler($html); - $this->sections = $crawler->filter(".Main-content .sqs-row.row > .col p > strong"); + $this->sections = $crawler->filter(".Main-content .sqs-row.row > .col p"); if (count($this->sections) === 0) { $this->createImportInfoDetails("204", self::getProviderName()); @@ -39,48 +38,55 @@ public function extract(): static public function transform(): void { + $countryName = ""; + + $cityName = ""; + + $states = [ + "Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", "Connecticut", "Delaware", "Florida", + "Georgia", "Hawaii", "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana", "Maine", + "Maryland", "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", "Montana", "Nebraska", + "Nevada", "New Hampshire", "New Jersey", "New Mexico", "New York", "North Carolina", "North Dakota", + "Ohio", "Oklahoma", "Oregon", "Pennsylvania", "Rhode Island", "South Carolina", "South Dakota", "Tennessee", + "Texas", "Utah", "Vermont", "Virginia", "Washington", "West Virginia", "Wisconsin", "Wyoming" + ]; if ($this->stopExecution) { return; } + $existingCityProviders = []; + $skipFirstIteration = true; + foreach ($this->sections as $section) { + if ($skipFirstIteration) { + $skipFirstIteration = false; + continue; + } foreach ($section->childNodes as $node) { - $countryName = trim($node->nodeValue); - - foreach ($node->parentNode->parentNode->parentNode->childNodes as $i => $cityName) { - if ($i === 0 || !trim($cityName->nodeValue)) { - continue; - } - - $name = $cityName->nodeValue; - - $cities = []; - - if (str_contains($name, "(") && str_contains($name, ")")) { - $names = explode("(", $name)[1]; - $names = explode(")", $names)[0]; - $names = explode(", ", $names); - - foreach ($names as $name) { - $cities[] = str_replace("*", "", $name); - } + if ($node->nodeName == "strong") { + if (!in_array($node->nodeValue, $states)) { + $countryName = trim($node->nodeValue); } else { - $cities[] = $name; - } - - foreach ($cities as $name) { - $provider = $this->load($name, $countryName); - - if ($provider !== "") { - $existingCityProviders[] = $provider; - } + $countryName = "United States"; } } + if ($node->nodeName == "#text") { + $cityName = $node->nodeValue; + } else if ($node->nodeName == "a") { + $cityName = $node->nodeValue; + } + if($cityName == " ") { + continue; + } else { + $provider = $this->load($cityName, $countryName); + } + if ($provider !== "") { + $existingCityProviders[] = $provider; + } } } - $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } } diff --git a/app/Services/DataImporterService.php b/app/Services/DataImporterService.php index 1dd1b8f0..7dc872da 100644 --- a/app/Services/DataImporterService.php +++ b/app/Services/DataImporterService.php @@ -43,28 +43,28 @@ public function run(string $whoRunsIt = "admin"): void $this->importInfoId = $importInfo->id; Bus::batch([ - new BeamDataImporterJob($this->importInfoId), - new BerylDataImporterJob($this->importInfoId), - new BinBinDataImporterJob($this->importInfoId), - new BirdDataImporterJob($this->importInfoId), - new BitMobilityDataImporterJob($this->importInfoId), - new BoltDataImporterJob($this->importInfoId), - new DottDataImporterJob($this->importInfoId), - new HulajDataImporterJob($this->importInfoId), - new LimeDataImporterJob($this->importInfoId), + //new BeamDataImporterJob($this->importInfoId), + //new BerylDataImporterJob($this->importInfoId), + //new BinBinDataImporterJob($this->importInfoId), + //new BirdDataImporterJob($this->importInfoId), + //new BitMobilityDataImporterJob($this->importInfoId), + //new BoltDataImporterJob($this->importInfoId), + //new DottDataImporterJob($this->importInfoId), + //new HulajDataImporterJob($this->importInfoId), + //new LimeDataImporterJob($this->importInfoId), new LinkDataImporterJob($this->importInfoId), - new NeuronDataImporterJob($this->importInfoId), - new QuickDataImporterJob($this->importInfoId), - new RydeDataImporterJob($this->importInfoId), - new SpinDataImporterJob($this->importInfoId), - new TierDataImporterJob($this->importInfoId), - new UrentDataImporterJob($this->importInfoId), - new VoiDataImporterJob($this->importInfoId), - new VeoDataImporterJob($this->importInfoId), - new WindDataImporterJob($this->importInfoId), - new WheeMoveDataImporterJob($this->importInfoId), - new HopDataImporterJob($this->importInfoId), - new ZwingsDataImporterJob($this->importInfoId), + //new NeuronDataImporterJob($this->importInfoId), + //new QuickDataImporterJob($this->importInfoId), + //new RydeDataImporterJob($this->importInfoId), + //new SpinDataImporterJob($this->importInfoId), + //new TierDataImporterJob($this->importInfoId), + //new UrentDataImporterJob($this->importInfoId), + //new VoiDataImporterJob($this->importInfoId), + //new VeoDataImporterJob($this->importInfoId), + //new WindDataImporterJob($this->importInfoId), + //new WheeMoveDataImporterJob($this->importInfoId), + //new HopDataImporterJob($this->importInfoId), + //new ZwingsDataImporterJob($this->importInfoId), ])->finally(function (): void { ImportInfo::query()->where("id", $this->importInfoId)->update([ "status" => "finished", From 0fd82c0e0be49c3e834677bc7a1a61f1cbafa16b Mon Sep 17 00:00:00 2001 From: BarTracz Date: Wed, 3 Jan 2024 12:27:30 +0100 Subject: [PATCH 2/3] link refactor --- app/Services/DataImporterService.php | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/app/Services/DataImporterService.php b/app/Services/DataImporterService.php index 7dc872da..1dd1b8f0 100644 --- a/app/Services/DataImporterService.php +++ b/app/Services/DataImporterService.php @@ -43,28 +43,28 @@ public function run(string $whoRunsIt = "admin"): void $this->importInfoId = $importInfo->id; Bus::batch([ - //new BeamDataImporterJob($this->importInfoId), - //new BerylDataImporterJob($this->importInfoId), - //new BinBinDataImporterJob($this->importInfoId), - //new BirdDataImporterJob($this->importInfoId), - //new BitMobilityDataImporterJob($this->importInfoId), - //new BoltDataImporterJob($this->importInfoId), - //new DottDataImporterJob($this->importInfoId), - //new HulajDataImporterJob($this->importInfoId), - //new LimeDataImporterJob($this->importInfoId), + new BeamDataImporterJob($this->importInfoId), + new BerylDataImporterJob($this->importInfoId), + new BinBinDataImporterJob($this->importInfoId), + new BirdDataImporterJob($this->importInfoId), + new BitMobilityDataImporterJob($this->importInfoId), + new BoltDataImporterJob($this->importInfoId), + new DottDataImporterJob($this->importInfoId), + new HulajDataImporterJob($this->importInfoId), + new LimeDataImporterJob($this->importInfoId), new LinkDataImporterJob($this->importInfoId), - //new NeuronDataImporterJob($this->importInfoId), - //new QuickDataImporterJob($this->importInfoId), - //new RydeDataImporterJob($this->importInfoId), - //new SpinDataImporterJob($this->importInfoId), - //new TierDataImporterJob($this->importInfoId), - //new UrentDataImporterJob($this->importInfoId), - //new VoiDataImporterJob($this->importInfoId), - //new VeoDataImporterJob($this->importInfoId), - //new WindDataImporterJob($this->importInfoId), - //new WheeMoveDataImporterJob($this->importInfoId), - //new HopDataImporterJob($this->importInfoId), - //new ZwingsDataImporterJob($this->importInfoId), + new NeuronDataImporterJob($this->importInfoId), + new QuickDataImporterJob($this->importInfoId), + new RydeDataImporterJob($this->importInfoId), + new SpinDataImporterJob($this->importInfoId), + new TierDataImporterJob($this->importInfoId), + new UrentDataImporterJob($this->importInfoId), + new VoiDataImporterJob($this->importInfoId), + new VeoDataImporterJob($this->importInfoId), + new WindDataImporterJob($this->importInfoId), + new WheeMoveDataImporterJob($this->importInfoId), + new HopDataImporterJob($this->importInfoId), + new ZwingsDataImporterJob($this->importInfoId), ])->finally(function (): void { ImportInfo::query()->where("id", $this->importInfoId)->update([ "status" => "finished", From c5a4959effd832678864276f9e0895771ed2b708 Mon Sep 17 00:00:00 2001 From: BarTracz Date: Wed, 3 Jan 2024 12:45:53 +0100 Subject: [PATCH 3/3] linter --- app/Importers/LinkDataImporter.php | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/app/Importers/LinkDataImporter.php b/app/Importers/LinkDataImporter.php index 496c91da..c2c861c5 100644 --- a/app/Importers/LinkDataImporter.php +++ b/app/Importers/LinkDataImporter.php @@ -48,13 +48,13 @@ public function transform(): void "Maryland", "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", "Montana", "Nebraska", "Nevada", "New Hampshire", "New Jersey", "New Mexico", "New York", "North Carolina", "North Dakota", "Ohio", "Oklahoma", "Oregon", "Pennsylvania", "Rhode Island", "South Carolina", "South Dakota", "Tennessee", - "Texas", "Utah", "Vermont", "Virginia", "Washington", "West Virginia", "Wisconsin", "Wyoming" + "Texas", "Utah", "Vermont", "Virginia", "Washington", "West Virginia", "Wisconsin", "Wyoming", ]; + if ($this->stopExecution) { return; } - $existingCityProviders = []; $skipFirstIteration = true; @@ -62,26 +62,30 @@ public function transform(): void foreach ($this->sections as $section) { if ($skipFirstIteration) { $skipFirstIteration = false; + continue; } + foreach ($section->childNodes as $node) { - if ($node->nodeName == "strong") { - if (!in_array($node->nodeValue, $states)) { + if ($node->nodeName === "strong") { + if (!in_array($node->nodeValue, $states, true)) { $countryName = trim($node->nodeValue); } else { $countryName = "United States"; } } - if ($node->nodeName == "#text") { + + if ($node->nodeName === "#text") { $cityName = $node->nodeValue; - } else if ($node->nodeName == "a") { + } else if ($node->nodeName === "a") { $cityName = $node->nodeValue; } - if($cityName == " ") { + + if ($cityName === " ") { continue; - } else { - $provider = $this->load($cityName, $countryName); - } + } + $provider = $this->load($cityName, $countryName); + if ($provider !== "") { $existingCityProviders[] = $provider; }