new champions are not showing [clni52700000nyr0k0ffc9zps]

pauloendoh · Nov 23, 2023 · f38787b · f38787b
1 parent 38e04af
commit f38787b
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 64 deletions.
diff --git a/src/repositories/lolrates/LolRateRepository.ts b/src/repositories/lolrates/LolRateRepository.ts
@@ -1,3 +1,4 @@
+import { Not } from "typeorm"
 import { dataSource } from "../../dataSource"
 import { LolRateDto } from "../../dtos/lolrates/LolRateDto"
 import { WinratesUpdatedAtDTO } from "../../dtos/lolrates/WinratesUpdatedAtDTO"
@@ -24,22 +25,22 @@ const LolRateRepository = dataSource.getRepository(LolRate).extend({
       }
 
       const winrates = await this.query(`
-           select avgs.*,
-                   (avgs."avgPick" + avgs."avgWin")/2  as "avgAvg"
-	          from (select "championName",
-                           "role",
-                           "iconUrl",
-                           "opggPick",
-                           "opggWin",
-                           "lolgraphsPick",
-                           "lolgraphsWin",
-						               "uggPick",
-						               "uggWin",
-                           ("opggPick" + "lolgraphsPick" + "uggPick")/3 as "avgPick",
-                           ("opggWin" + "lolgraphsWin" + "uggWin")/3 as "avgWin"
-                      from "lol_rate") as avgs
-	          where  "avgWin" > 0
-         order by "avgAvg" desc 
+      select avgs.*,
+      (avgs."avgPick" + avgs."avgWin")/2  as "avgAvg"
+from (select "championName",
+              "role",
+              "iconUrl",
+              "opggPick",
+              "opggWin",
+              "lolgraphsPick",
+              "lolgraphsWin",
+              "uggPick",
+              "uggWin",
+              (COALESCE("opggPick", 0) + COALESCE("lolgraphsPick", 0) + COALESCE("uggPick", 0))/3 as "avgPick",
+              (COALESCE("opggWin", 0)   + COALESCE ("lolgraphsWin", 0)  + COALESCE("uggWin", 0))/3 as "avgWin"
+         from "lol_rate") as avgs
+where  "avgWin" > 0
+order by "avgAvg" desc 
         `)
 
       await myRedis.set("winrates", JSON.stringify(winrates), "EX", 3600)
@@ -75,14 +76,23 @@ const LolRateRepository = dataSource.getRepository(LolRate).extend({
       for (const champion of champions) {
         const { name: championName, iconUrl } = champion
 
-        const exists = await this.findOne({
-          where: { championName, iconUrl },
-        })
+        const roles: RoleTypes[] = ["TOP", "JUNGLE", "MID", "BOT", "SUP"]
+        for (const role of roles) {
+          const exists = await this.findOne({
+            where: { championName, role },
+          })
 
-        if (!exists) {
-          const roles: RoleTypes[] = ["TOP", "JUNGLE", "MID", "BOT", "SUP"]
-          for (const role of roles) {
-            await this.save({ championName, iconUrl, role })
+          if (!exists) {
+            const id = await this.getNextId()
+            const lolRateRepo = dataSource.getRepository(LolRate)
+
+            await lolRateRepo.query(
+              `
+            INSERT INTO "lol_rate" ("id","championName", "iconUrl", "role") 
+              values($1,$2, $3, $4)
+            `,
+              [id, championName, iconUrl, role]
+            )
           }
         }
       }
@@ -95,7 +105,21 @@ const LolRateRepository = dataSource.getRepository(LolRate).extend({
         })
 
         if (!exists) {
-          await championRepo.save({ name, iconUrl })
+          const [last] = await championRepo.find({
+            order: {
+              id: "DESC",
+            },
+          })
+
+          const nextId = last ? last.id + 1 : 1
+
+          await championRepo.query(
+            `
+          INSERT INTO "champion" ("id", "name", "iconUrl")
+            values($1, $2, $3)
+          `,
+            [nextId, name, iconUrl]
+          )
         }
       }
 
@@ -105,6 +129,20 @@ const LolRateRepository = dataSource.getRepository(LolRate).extend({
     }
   },
 
+  async getNextId() {
+    const last = await this.findOne({
+      where: {
+        championName: Not(""),
+      },
+      order: { id: "DESC" },
+    })
+    if (!last) {
+      return 1
+    }
+
+    return last.id + 1
+  },
+
   async saveOpgg(results: ScrapeResult[]) {
     try {
       // resetting all from op.gg

diff --git a/src/utils/lolrates/scrapeLolRates.ts b/src/utils/lolrates/scrapeLolRates.ts
@@ -27,9 +27,10 @@ export async function scrapeLolRates() {
       )
       await page.setViewport({ width: 1000, height: 1000 })
 
+      await scrapeChampions(page)
+
       await scrapeAram(page)
 
-      await scrapeChampions(page)
       await scrapeOpgg(page)
       await scrapeLolGraphs(page)
       await scrapeUgg(page)

diff --git a/src/utils/lolrates/scrapeLolRates/scrapeChampions.ts b/src/utils/lolrates/scrapeLolRates/scrapeChampions.ts
@@ -1,4 +1,7 @@
+import axios from "axios"
+import { JSDOM } from "jsdom"
 import { Page } from "puppeteer"
+import LolRateRepository from "../../../repositories/lolrates/LolRateRepository"
 import { myConsoleError } from "../../myConsoleError"
 
 export interface IChampion {
@@ -8,54 +11,32 @@ export interface IChampion {
 
 export async function scrapeChampions(page: Page) {
   try {
-    await page.goto("https://blitz.gg/lol/champions/overview")
+    // u.gg
+    const html = await axios
+      .get<string>("https://u.gg/lol/champions")
+      .then((res) => res.data)
 
-    await page.waitForSelector(".infinite-table")
+    const dom = new JSDOM(html)
+    const document = dom.window.document
 
-    // scroll to end of page at least 5 times
-    for (let i = 0; i < 10; i++) {
-      await page.evaluate(() => {
-        window.scrollBy(0, window.innerHeight)
-      })
-      // 1 second
-      await page.waitForTimeout(1000)
-    }
-
-    const champions = await page.evaluate(() => {
-      const champions: IChampion[] = []
-
-      const table = document.querySelector(".infinite-table")
-
-      const divsParent = table?.querySelector("div[style='flex: 1 1 0%;']")
+    const champions: IChampion[] = []
 
-      let championDivs = divsParent?.children || []
+    const championsLinks = Array.from(
+      document.querySelectorAll(".champion-link")
+    )
 
-      for (const div of Array.from(championDivs)) {
-        if (!div.className.includes("⚡")) continue
-        console.log({
-          div,
-        })
-        const championDiv = div.children[3]
-
-        const img = championDiv.querySelector("img")
-        if (!img) continue
-        const iconUrl = img.getAttribute("src") || ""
-        const name = championDiv?.querySelector("span")?.textContent || ""
-
-        // 23/apr 2022 - it was returning fallback images
-        // like https://blitz-cdn-plain.blitz.gg/blitz/ui/img/fallback.svg
-        // and it was causing repeated champions
-        if (iconUrl.includes("fallback")) continue
+    for (const championLink of championsLinks) {
+      const name = championLink.querySelector(".champion-name")?.textContent
+      const iconUrl = championLink.querySelector("img")?.getAttribute("src")
 
+      if (name && iconUrl) {
         champions.push({ name, iconUrl })
       }
+    }
 
-      return champions
-    })
-
-    // const saved = await LolRateRepository.saveChampions(champions)
+    const saved = await LolRateRepository.saveChampions(champions)
 
-    // return saved
+    return saved
   } catch (err) {
     myConsoleError(err.message)
   }