-
-
Notifications
You must be signed in to change notification settings - Fork 59
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
support piped instances in invidious list
- Loading branch information
Showing
6 changed files
with
168 additions
and
35 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| /* | ||
| This file is only ran by GitHub Actions in order to populate the Invidious instances list | ||
| This file should not be shipped with the extension | ||
| */ | ||
|
|
||
| /* | ||
| Criteria for inclusion: | ||
| Invidious | ||
| - 30d uptime >= 90% | ||
| - available for at least 80/90 days | ||
| - must have been up for at least 90 days | ||
| - HTTPS only | ||
| - url includes name (this is to avoid redirects) | ||
| Piped | ||
| - 30d uptime >= 90% | ||
| - available for at least 80/90 days | ||
| - must have been up for at least 90 days | ||
| - must not be a wildcard redirect to piped.video | ||
| - must be currently up | ||
| - must have a functioning frontend | ||
| - must have a functioning API | ||
| */ | ||
|
|
||
| import { writeFile, existsSync } from "fs" | ||
| import { join } from "path" | ||
| import { getInvidiousList } from "./invidiousCI"; | ||
| // import { getPipedList } from "./pipedCI"; | ||
|
|
||
| const checkPath = (path: string) => existsSync(path); | ||
| const fixArray = (arr: string[]) => [...new Set(arr)].sort() | ||
|
|
||
| async function generateList() { | ||
| // import file from https://api.invidious.io/instances.json | ||
| const invidiousPath = join(__dirname, "invidious_instances.json"); | ||
| // import file from https://github.com/TeamPiped/piped-uptime/raw/master/history/summary.json | ||
| const pipedPath = join(__dirname, "piped_instances.json"); | ||
|
|
||
| // check if files exist | ||
| if (!checkPath(invidiousPath) || !checkPath(pipedPath)) { | ||
| console.log("Missing files") | ||
| process.exit(1); | ||
| } | ||
|
|
||
| // static non-invidious instances | ||
| const staticInstances = ["www.youtubekids.com"]; | ||
| // invidious instances | ||
| const invidiousList = fixArray(getInvidiousList()) | ||
| // piped instnaces | ||
| // const pipedList = fixArray(await getPipedList()) | ||
|
|
||
| console.log([...staticInstances, ...invidiousList]) | ||
|
|
||
| writeFile( | ||
| join(__dirname, "./invidiouslist.json"), | ||
| JSON.stringify([...staticInstances, ...invidiousList]), | ||
| (err) => { | ||
| if (err) return console.log(err); | ||
| } | ||
| ); | ||
| } | ||
| generateList() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| import * as data from "../ci/piped_instances.json"; | ||
|
|
||
| type percent = string | ||
| type dailyMinutesDown = Record<string, number> | ||
|
|
||
| type PipedInstance = { | ||
| name: string; | ||
| url: string; | ||
| icon: string; | ||
| slug: string; | ||
| status: string; | ||
| uptime: percent; | ||
| uptimeDay: percent; | ||
| uptimeWeek: percent; | ||
| uptimeMonth: percent; | ||
| uptimeYear: percent; | ||
| time: number; | ||
| timeDay: number; | ||
| timeWeek: number; | ||
| timeMonth: number; | ||
| timeYear: number; | ||
| dailyMinutesDown: dailyMinutesDown | ||
| } | ||
|
|
||
| const percentNumber = (percent: percent) => Number(percent.replace("%", "")) | ||
| const ninetyDaysAgo = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000) | ||
|
|
||
| function dailyMinuteFilter (dailyMinutesDown: dailyMinutesDown) { | ||
| let daysDown = 0 | ||
| for (const [date, minsDown] of Object.entries(dailyMinutesDown)) { | ||
| if (new Date(date) >= ninetyDaysAgo && minsDown > 1000) { // if within 90 days and down for more than 1000 minutes | ||
| daysDown++ | ||
| } | ||
| } | ||
| // return true f less than 10 days down | ||
| return daysDown < 10 | ||
| } | ||
|
|
||
| const getHost = (url: string) => new URL(url).host | ||
|
|
||
| const getWatchPage = async (instance: PipedInstance) => | ||
| fetch(`https://${getHost(instance.url)}`, { redirect: "manual" }) | ||
| .then(res => res.headers.get("Location")) | ||
| .catch(e => { console.log (e); return null }) | ||
|
|
||
| const siteOK = async (instance) => { | ||
| // check if entire site is redirect | ||
| const notRedirect = await fetch(instance.url, { redirect: "manual" }) | ||
| .then(res => res.status == 200) | ||
| // only allow kavin to return piped.video | ||
| // if (instance.url.startsWith("https://piped.video") && instance.slug !== "kavin-rocks-official") return false | ||
| // check if frontend is OK | ||
| const watchPageStatus = await fetch(instance.frontendUrl) | ||
| .then(res => res.ok) | ||
| // test API - stream returns ok result | ||
| const streamStatus = await fetch(`${instance.apiUrl}/streams/BaW_jenozKc`) | ||
| .then(res => res.ok) | ||
| // get startTime of monitor | ||
| const age = await fetch(instance.historyUrl) | ||
| .then(res => res.text()) | ||
| .then(text => { // startTime greater than 90 days ago | ||
| const date = text.match(/startTime: (.+)/)[1] | ||
| return Date.parse(date) < ninetyDaysAgo.valueOf() | ||
| }) | ||
| // console.log(notRedirect, watchPageStatus, streamStatus, age, instance.frontendUrl, instance.apiUrl) | ||
| return notRedirect && watchPageStatus && streamStatus && age | ||
| } | ||
|
|
||
| const staticFilters = (data as PipedInstance[]) | ||
| .filter(instance => { | ||
| const isup = instance.status === "up" | ||
| const monthCheck = percentNumber(instance.uptimeMonth) >= 90 | ||
| const dailyMinuteCheck = dailyMinuteFilter(instance.dailyMinutesDown) | ||
| return isup && monthCheck && dailyMinuteCheck | ||
| }) | ||
| .map(async instance => { | ||
| // get frontend url | ||
| const frontendUrl = await getWatchPage(instance) | ||
| if (!frontendUrl) return null // return false if frontend doesn't resolve | ||
| // get api base | ||
| const apiUrl = instance.url.replace("/healthcheck", "") | ||
| const historyUrl = `https://raw.githubusercontent.com/TeamPiped/piped-uptime/master/history/${instance.slug}.yml` | ||
| const pass = await siteOK({ apiUrl, historyUrl, frontendUrl, url: instance.url }) | ||
| const frontendHost = getHost(frontendUrl) | ||
| return pass ? frontendHost : null | ||
| }) | ||
|
|
||
| export async function getPipedList(): Promise<string[]> { | ||
| const instances = await Promise.all(staticFilters) | ||
| .then(arr => arr.filter(i => i !== null)) | ||
| return instances | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters