diff --git a/src/repositories/lolrates/LolRateRepository.ts b/src/repositories/lolrates/LolRateRepository.ts index 7b23cd1..9048e03 100644 --- a/src/repositories/lolrates/LolRateRepository.ts +++ b/src/repositories/lolrates/LolRateRepository.ts @@ -1,3 +1,4 @@ +import { Not } from "typeorm" import { dataSource } from "../../dataSource" import { LolRateDto } from "../../dtos/lolrates/LolRateDto" import { WinratesUpdatedAtDTO } from "../../dtos/lolrates/WinratesUpdatedAtDTO" @@ -24,22 +25,22 @@ const LolRateRepository = dataSource.getRepository(LolRate).extend({ } const winrates = await this.query(` - select avgs.*, - (avgs."avgPick" + avgs."avgWin")/2 as "avgAvg" - from (select "championName", - "role", - "iconUrl", - "opggPick", - "opggWin", - "lolgraphsPick", - "lolgraphsWin", - "uggPick", - "uggWin", - ("opggPick" + "lolgraphsPick" + "uggPick")/3 as "avgPick", - ("opggWin" + "lolgraphsWin" + "uggWin")/3 as "avgWin" - from "lol_rate") as avgs - where "avgWin" > 0 - order by "avgAvg" desc + select avgs.*, + (avgs."avgPick" + avgs."avgWin")/2 as "avgAvg" +from (select "championName", + "role", + "iconUrl", + "opggPick", + "opggWin", + "lolgraphsPick", + "lolgraphsWin", + "uggPick", + "uggWin", + (COALESCE("opggPick", 0) + COALESCE("lolgraphsPick", 0) + COALESCE("uggPick", 0))/3 as "avgPick", + (COALESCE("opggWin", 0) + COALESCE ("lolgraphsWin", 0) + COALESCE("uggWin", 0))/3 as "avgWin" + from "lol_rate") as avgs +where "avgWin" > 0 +order by "avgAvg" desc `) await myRedis.set("winrates", JSON.stringify(winrates), "EX", 3600) @@ -75,14 +76,23 @@ const LolRateRepository = dataSource.getRepository(LolRate).extend({ for (const champion of champions) { const { name: championName, iconUrl } = champion - const exists = await this.findOne({ - where: { championName, iconUrl }, - }) + const roles: RoleTypes[] = ["TOP", "JUNGLE", "MID", "BOT", "SUP"] + for (const role of roles) { + const exists = await this.findOne({ + where: { championName, role }, + }) - if (!exists) { - const roles: RoleTypes[] = ["TOP", "JUNGLE", "MID", "BOT", "SUP"] - for (const role of roles) { - await this.save({ championName, iconUrl, role }) + if (!exists) { + const id = await this.getNextId() + const lolRateRepo = dataSource.getRepository(LolRate) + + await lolRateRepo.query( + ` + INSERT INTO "lol_rate" ("id","championName", "iconUrl", "role") + values($1,$2, $3, $4) + `, + [id, championName, iconUrl, role] + ) } } } @@ -95,7 +105,21 @@ const LolRateRepository = dataSource.getRepository(LolRate).extend({ }) if (!exists) { - await championRepo.save({ name, iconUrl }) + const [last] = await championRepo.find({ + order: { + id: "DESC", + }, + }) + + const nextId = last ? last.id + 1 : 1 + + await championRepo.query( + ` + INSERT INTO "champion" ("id", "name", "iconUrl") + values($1, $2, $3) + `, + [nextId, name, iconUrl] + ) } } @@ -105,6 +129,20 @@ const LolRateRepository = dataSource.getRepository(LolRate).extend({ } }, + async getNextId() { + const last = await this.findOne({ + where: { + championName: Not(""), + }, + order: { id: "DESC" }, + }) + if (!last) { + return 1 + } + + return last.id + 1 + }, + async saveOpgg(results: ScrapeResult[]) { try { // resetting all from op.gg diff --git a/src/utils/lolrates/scrapeLolRates.ts b/src/utils/lolrates/scrapeLolRates.ts index 003f1d7..d4cf5c9 100644 --- a/src/utils/lolrates/scrapeLolRates.ts +++ b/src/utils/lolrates/scrapeLolRates.ts @@ -27,9 +27,10 @@ export async function scrapeLolRates() { ) await page.setViewport({ width: 1000, height: 1000 }) + await scrapeChampions(page) + await scrapeAram(page) - await scrapeChampions(page) await scrapeOpgg(page) await scrapeLolGraphs(page) await scrapeUgg(page) diff --git a/src/utils/lolrates/scrapeLolRates/scrapeChampions.ts b/src/utils/lolrates/scrapeLolRates/scrapeChampions.ts index 959c3cd..9fa4f07 100644 --- a/src/utils/lolrates/scrapeLolRates/scrapeChampions.ts +++ b/src/utils/lolrates/scrapeLolRates/scrapeChampions.ts @@ -1,4 +1,7 @@ +import axios from "axios" +import { JSDOM } from "jsdom" import { Page } from "puppeteer" +import LolRateRepository from "../../../repositories/lolrates/LolRateRepository" import { myConsoleError } from "../../myConsoleError" export interface IChampion { @@ -8,54 +11,32 @@ export interface IChampion { export async function scrapeChampions(page: Page) { try { - await page.goto("https://blitz.gg/lol/champions/overview") + // u.gg + const html = await axios + .get("https://u.gg/lol/champions") + .then((res) => res.data) - await page.waitForSelector(".infinite-table") + const dom = new JSDOM(html) + const document = dom.window.document - // scroll to end of page at least 5 times - for (let i = 0; i < 10; i++) { - await page.evaluate(() => { - window.scrollBy(0, window.innerHeight) - }) - // 1 second - await page.waitForTimeout(1000) - } - - const champions = await page.evaluate(() => { - const champions: IChampion[] = [] - - const table = document.querySelector(".infinite-table") - - const divsParent = table?.querySelector("div[style='flex: 1 1 0%;']") + const champions: IChampion[] = [] - let championDivs = divsParent?.children || [] + const championsLinks = Array.from( + document.querySelectorAll(".champion-link") + ) - for (const div of Array.from(championDivs)) { - if (!div.className.includes("⚡")) continue - console.log({ - div, - }) - const championDiv = div.children[3] - - const img = championDiv.querySelector("img") - if (!img) continue - const iconUrl = img.getAttribute("src") || "" - const name = championDiv?.querySelector("span")?.textContent || "" - - // 23/apr 2022 - it was returning fallback images - // like https://blitz-cdn-plain.blitz.gg/blitz/ui/img/fallback.svg - // and it was causing repeated champions - if (iconUrl.includes("fallback")) continue + for (const championLink of championsLinks) { + const name = championLink.querySelector(".champion-name")?.textContent + const iconUrl = championLink.querySelector("img")?.getAttribute("src") + if (name && iconUrl) { champions.push({ name, iconUrl }) } + } - return champions - }) - - // const saved = await LolRateRepository.saveChampions(champions) + const saved = await LolRateRepository.saveChampions(champions) - // return saved + return saved } catch (err) { myConsoleError(err.message) }