From aef6ac42d7be2a55dc3fdb8562cbdad7242297e7 Mon Sep 17 00:00:00 2001 From: Claudemirovsky <63046606+Claudemirovsky@users.noreply.github.com> Date: Mon, 4 Dec 2023 05:56:23 -0300 Subject: [PATCH] fix(en/uhdmovies): Fix episode list & video extractor (#2604) --- src/en/uhdmovies/build.gradle | 10 +- .../en/uhdmovies/RedirectorBypasser.kt | 65 ++ .../en/uhdmovies/TokenInterceptor.kt | 87 --- .../animeextension/en/uhdmovies/UHDMovies.kt | 600 ++++++++---------- 4 files changed, 330 insertions(+), 432 deletions(-) create mode 100644 src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/RedirectorBypasser.kt delete mode 100644 src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/TokenInterceptor.kt diff --git a/src/en/uhdmovies/build.gradle b/src/en/uhdmovies/build.gradle index e481714ff9..a9e18db9c8 100644 --- a/src/en/uhdmovies/build.gradle +++ b/src/en/uhdmovies/build.gradle @@ -1,12 +1,14 @@ -apply plugin: 'com.android.application' -apply plugin: 'kotlin-android' -apply plugin: 'kotlinx-serialization' +plugins { + alias(libs.plugins.android.application) + alias(libs.plugins.kotlin.android) + alias(libs.plugins.kotlin.serialization) +} ext { extName = 'UHD Movies' pkgNameSuffix = 'en.uhdmovies' extClass = '.UHDMovies' - extVersionCode = 17 + extVersionCode = 18 libVersion = '13' } diff --git a/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/RedirectorBypasser.kt b/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/RedirectorBypasser.kt new file mode 100644 index 0000000000..3fd48a57b2 --- /dev/null +++ b/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/RedirectorBypasser.kt @@ -0,0 +1,65 @@ +package eu.kanade.tachiyomi.animeextension.en.uhdmovies + +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.POST +import eu.kanade.tachiyomi.util.asJsoup +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Cookie +import okhttp3.FormBody +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.OkHttpClient +import org.jsoup.nodes.Document + +class RedirectorBypasser(private val client: OkHttpClient, private val headers: Headers) { + fun bypass(url: String): String? { + val lastDoc = client.newCall(GET(url, headers)).execute() + .use { recursiveDoc(it.asJsoup()) } + + val script = lastDoc.selectFirst("script:containsData(/?go=):containsData(href)") + ?.data() + ?: return null + + val nextUrl = script.substringAfter("\"href\",\"").substringBefore('"') + val httpUrl = nextUrl.toHttpUrlOrNull() ?: return null + val cookieName = httpUrl.queryParameter("go") ?: return null + val cookieValue = script.substringAfter("'$cookieName', '").substringBefore("'") + val cookie = Cookie.parse(httpUrl, "$cookieName=$cookieValue")!! + val headers = headers.newBuilder().set("referer", lastDoc.location()).build() + + val doc = runBlocking(Dispatchers.IO) { + MUTEX.withLock { // Mutex to prevent overwriting cookies from parallel requests + client.cookieJar.saveFromResponse(httpUrl, listOf(cookie)) + client.newCall(GET(nextUrl, headers)).execute().use { it.asJsoup() } + } + } + + return doc.selectFirst("meta[http-equiv]")?.attr("content") + ?.substringAfter("url=") + } + + private fun recursiveDoc(doc: Document): Document { + val form = doc.selectFirst("form#landing") ?: return doc + val url = form.attr("action") + val body = FormBody.Builder().apply { + form.select("input").forEach { + add(it.attr("name"), it.attr("value")) + } + }.build() + + val headers = headers.newBuilder() + .set("referer", doc.location()) + .build() + + return client.newCall(POST(url, headers, body)).execute().use { + recursiveDoc(it.asJsoup()) + } + } + + companion object { + private val MUTEX by lazy { Mutex() } + } +} diff --git a/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/TokenInterceptor.kt b/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/TokenInterceptor.kt deleted file mode 100644 index 006732fd5b..0000000000 --- a/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/TokenInterceptor.kt +++ /dev/null @@ -1,87 +0,0 @@ -package eu.kanade.tachiyomi.animeextension.en.uhdmovies - -import android.annotation.SuppressLint -import android.app.Application -import android.os.Handler -import android.os.Looper -import android.webkit.JavascriptInterface -import android.webkit.WebView -import android.webkit.WebViewClient -import eu.kanade.tachiyomi.network.GET -import okhttp3.Interceptor -import okhttp3.Request -import okhttp3.Response -import uy.kohesive.injekt.Injekt -import uy.kohesive.injekt.api.get -import java.util.concurrent.CountDownLatch - -class TokenInterceptor : Interceptor { - - private val context = Injekt.get() - private val handler by lazy { Handler(Looper.getMainLooper()) } - - class JsObject(private val latch: CountDownLatch, var payload: String = "") { - @JavascriptInterface - fun passPayload(passedPayload: String) { - payload = passedPayload - latch.countDown() - } - } - - override fun intercept(chain: Interceptor.Chain): Response { - val originalRequest = chain.request() - - val newRequest = resolveWithWebView(originalRequest) ?: originalRequest - - return chain.proceed(newRequest) - } - - @SuppressLint("SetJavaScriptEnabled") - private fun resolveWithWebView(request: Request): Request? { - val latch = CountDownLatch(1) - - var webView: WebView? = null - - val origRequestUrl = request.url.toString() - - val jsinterface = JsObject(latch) - - // Get url with token with promise - val jsScript = """ - (async () => { - var data = await generate("direct"); - window.android.passPayload(data.url); - })();""".trim() - - val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap() - - handler.post { - val webview = WebView(context) - webView = webview - with(webview.settings) { - javaScriptEnabled = true - domStorageEnabled = true - databaseEnabled = true - useWideViewPort = false - loadWithOverviewMode = false - userAgentString = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/109.0" - webview.addJavascriptInterface(jsinterface, "android") - webview.webViewClient = object : WebViewClient() { - override fun onPageFinished(view: WebView?, url: String?) { - view?.evaluateJavascript(jsScript) {} - } - } - webView?.loadUrl(origRequestUrl, headers) - } - } - - latch.await() - - handler.post { - webView?.stopLoading() - webView?.destroy() - webView = null - } - return if (jsinterface.payload.isNotBlank()) GET(jsinterface.payload) else null - } -} diff --git a/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/UHDMovies.kt b/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/UHDMovies.kt index 8b64b2f51e..e6a1e13e4f 100644 --- a/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/UHDMovies.kt +++ b/src/en/uhdmovies/src/eu/kanade/tachiyomi/animeextension/en/uhdmovies/UHDMovies.kt @@ -1,7 +1,6 @@ package eu.kanade.tachiyomi.animeextension.en.uhdmovies import android.app.Application -import android.content.SharedPreferences import android.util.Base64 import androidx.preference.EditTextPreference import androidx.preference.ListPreference @@ -20,16 +19,13 @@ import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext -import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.Serializable import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json -import okhttp3.FormBody -import okhttp3.Headers -import okhttp3.OkHttpClient +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.MultipartBody import okhttp3.Request import okhttp3.Response -import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.nodes.Element import rx.Observable @@ -37,186 +33,134 @@ import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get import uy.kohesive.injekt.injectLazy -@ExperimentalSerializationApi class UHDMovies : ConfigurableAnimeSource, ParsedAnimeHttpSource() { override val name = "UHD Movies" - override val baseUrl by lazy { preferences.getString(PREF_DOMAIN_KEY, PREF_DEFAULT_DOMAIN)!! } - - override val lang = "en" - - override val supportsLatest = false - - override val client: OkHttpClient = network.cloudflareClient - - private val json: Json by injectLazy() - - private val preferences: SharedPreferences by lazy { - Injekt.get().getSharedPreferences("source_$id", 0x0000) - } - - private val currentBaseUrl by lazy { + override val baseUrl by lazy { + val url = preferences.getString(PREF_DOMAIN_KEY, PREF_DOMAIN_DEFAULT)!! runBlocking { withContext(Dispatchers.Default) { client.newBuilder() .followRedirects(false) .build() - .newCall(GET("$baseUrl/")).execute().let { resp -> + .newCall(GET("$url/")).execute().use { resp -> when (resp.code) { 301 -> { - (resp.headers["location"]?.substringBeforeLast("/") ?: baseUrl).also { + (resp.headers["location"]?.substringBeforeLast("/") ?: url).also { preferences.edit().putString(PREF_DOMAIN_KEY, it).apply() } } - else -> baseUrl + else -> url } } } } } - // ============================== Popular =============================== + override val lang = "en" + + override val supportsLatest = false - override fun popularAnimeRequest(page: Int): Request = GET("$currentBaseUrl/page/$page/") + override val client = network.cloudflareClient + + private val json: Json by injectLazy() + + private val preferences by lazy { + Injekt.get().getSharedPreferences("source_$id", 0x0000) + } + + // ============================== Popular =============================== + override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/page/$page/") override fun popularAnimeSelector(): String = "div#content div.gridlove-posts > div.layout-masonry" + override fun popularAnimeFromElement(element: Element) = SAnime.create().apply { + setUrlWithoutDomain(element.select("div.entry-image > a").attr("abs:href")) + thumbnail_url = element.select("div.entry-image > a > img").attr("abs:src") + title = element.select("div.entry-image > a").attr("title") + .replace("Download", "").trim() + } + override fun popularAnimeNextPageSelector(): String = "div#content > nav.gridlove-pagination > a.next" - override fun popularAnimeFromElement(element: Element): SAnime { - return SAnime.create().apply { - setUrlWithoutDomain(element.select("div.entry-image > a").attr("abs:href")) - thumbnail_url = element.select("div.entry-image > a > img").attr("abs:src") - title = element.select("div.entry-image > a").attr("title") - .replace("Download", "").trim() - } - } - // =============================== Latest =============================== - override fun latestUpdatesRequest(page: Int): Request = throw Exception("Not Used") override fun latestUpdatesSelector(): String = throw Exception("Not Used") - override fun latestUpdatesNextPageSelector(): String = throw Exception("Not Used") - override fun latestUpdatesFromElement(element: Element): SAnime = throw Exception("Not Used") - // =============================== Search =============================== + override fun latestUpdatesNextPageSelector(): String = throw Exception("Not Used") + // =============================== Search =============================== override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request { val cleanQuery = query.replace(" ", "+").lowercase() - return GET("$currentBaseUrl/page/$page/?s=$cleanQuery") + return GET("$baseUrl/page/$page/?s=$cleanQuery") } override fun searchAnimeSelector(): String = popularAnimeSelector() - override fun searchAnimeNextPageSelector(): String = popularAnimeNextPageSelector() - override fun searchAnimeFromElement(element: Element): SAnime = popularAnimeFromElement(element) - // =========================== Anime Details ============================ + override fun searchAnimeNextPageSelector(): String = popularAnimeNextPageSelector() - override fun animeDetailsParse(document: Document): SAnime { - return SAnime.create().apply { - initialized = true - title = document.selectFirst(".entry-title")?.text() - ?.replace("Download", "", true)?.trim() ?: "Movie" - status = SAnime.COMPLETED - description = document.selectFirst("pre:contains(plot)")?.text() - } + // =========================== Anime Details ============================ + override fun animeDetailsParse(document: Document) = SAnime.create().apply { + initialized = true + title = document.selectFirst(".entry-title")?.text() + ?.replace("Download", "", true)?.trim() ?: "Movie" + status = SAnime.COMPLETED + description = document.selectFirst("pre:contains(plot)")?.text() } // ============================== Episodes ============================== + private fun Regex.firstValue(text: String) = + find(text)?.groupValues?.get(1)?.let { Pair(text, it) } - override fun fetchEpisodeList(anime: SAnime): Observable> { - val resp = client.newCall(GET(currentBaseUrl + anime.url)).execute().asJsoup() - val episodeList = mutableListOf() - val episodeElements = resp.select("p:has(a[href*=?id=],a[href*=r?key=]):has(a[class*=maxbutton])[style*=center]") - val qualityRegex = "\\d{3,4}p".toRegex(RegexOption.IGNORE_CASE) - val firstText = episodeElements.first()?.text() ?: "" - if (firstText.contains("Episode", true) || - firstText.contains("Zip", true) || - firstText.contains("Pack", true) - ) { - episodeElements.map { row -> - val prevP = row.previousElementSibling()!! - val seasonRegex = "[ .]?S(?:eason)?[ .]?(\\d{1,2})[ .]?".toRegex(RegexOption.IGNORE_CASE) - val partRegex = "Part ?(\\d{1,2})".toRegex(RegexOption.IGNORE_CASE) - val result = seasonRegex.find(prevP.text()) - var part = "" - val season = ( - result?.groups?.get(1)?.value?.also { - part = partRegex.find(prevP.text())?.groups?.get(1)?.value ?: "" - } ?: let { - val prevPre = row.previousElementSiblings().prev("pre,div.mks_separator") - val preResult = seasonRegex.find(prevPre.first()?.text() ?: "") - preResult?.groups?.get(1)?.value?.also { - part = partRegex.find(prevPre.first()?.text() ?: "")?.groups?.get(1)?.value ?: "" - } ?: let { - val title = resp.select("h1.entry-title") - val titleResult = "[ .\\[(]?S(?:eason)?[ .]?(\\d{1,2})[ .\\])]?" - .toRegex(RegexOption.IGNORE_CASE) - .find(title.text()) - titleResult?.groups?.get(1)?.value?.also { - part = partRegex.find(title.text())?.groups?.get(1)?.value ?: "" - } ?: "-1" - } - } - ).replaceFirst("^0+(?!$)".toRegex(), "") + override fun episodeListParse(response: Response): List { + val doc = response.use { it.asJsoup() } + val episodeElements = doc.select(episodeListSelector()) + .asSequence() - val qualityMatch = qualityRegex.find(prevP.text()) - val quality = qualityMatch?.value ?: let { - val qualityMatchOwn = qualityRegex.find(row.text()) - qualityMatchOwn?.value ?: "HD" - } + val qualityRegex = "\\d{3,4}p".toRegex(RegexOption.IGNORE_CASE) + val seasonRegex = "[ .]?S(?:eason)?[ .]?(\\d{1,2})[ .]?".toRegex(RegexOption.IGNORE_CASE) + val seasonTitleRegex = "[ .\\[(]?S(?:eason)?[ .]?(\\d{1,2})[ .\\])]?".toRegex(RegexOption.IGNORE_CASE) + val partRegex = "Part ?(\\d{1,2})".toRegex(RegexOption.IGNORE_CASE) + + val isSerie = doc.selectFirst(episodeListSelector())?.text().orEmpty().run { + contains("Episode", true) || + contains("Zip", true) || + contains("Pack", true) + } - row.select("a").filter { it -> - !it.text().contains("Zip", true) && - !it.text().contains("Pack", true) && - !it.text().contains("Volume ", true) - }.mapIndexed { index, linkElement -> - val episode = linkElement?.text() - ?.replace("Episode", "", true) - ?.trim()?.toIntOrNull() ?: index + 1 - Triple( - season + "_$episode" + "_$part", - linkElement?.attr("href") ?: return@mapIndexed null, - quality, - ) - }.filterNotNull() - }.flatten().groupBy { it.first }.map { group -> - val (season, episode, part) = group.key.split("_") - val partText = if (part.isBlank()) "" else " Pt $part" - episodeList.add( - SEpisode.create().apply { - url = EpLinks( - urls = group.value.map { - EpUrl(url = it.second, quality = it.third) - }, - ).toJson() - name = "Season $season$partText Ep $episode" - episode_number = episode.toFloat() - }, - ) + val episodeList = episodeElements.map { row -> + val prevP = row.previousElementSibling()!!.text() + val qualityMatch = qualityRegex.find(prevP) + val quality = qualityMatch?.value ?: let { + val qualityMatchOwn = qualityRegex.find(row.text()) + qualityMatchOwn?.value ?: "HD" } - } else { - var collectionIdx = 0F - episodeElements.asSequence().filter { - !it.text().contains("Zip", true) && - !it.text().contains("Pack", true) && - !it.text().contains("Volume ", true) - }.map { row -> - val prevP = row.previousElementSibling()!! - val qualityMatch = qualityRegex.find(prevP.text()) - val quality = qualityMatch?.value ?: let { - val qualityMatchOwn = qualityRegex.find(row.text()) - qualityMatchOwn?.value ?: "HD" - } - val collectionName = row.previousElementSiblings().let { prevElem -> + val defaultName = if (isSerie) { + val (source, seasonNumber) = seasonRegex.firstValue(prevP) ?: run { + val prevPre = row.previousElementSiblings().prev("pre,div.mks_separator").first() + ?.text() + .orEmpty() + seasonRegex.firstValue(prevPre) + } ?: run { + val title = doc.selectFirst("h1.entry-title")?.text().orEmpty() + seasonTitleRegex.firstValue(title) + } ?: "" to "1" + + val part = partRegex.find(source)?.groupValues?.get(1) + ?.let { " Pt $it" } + .orEmpty() + + "Season ${seasonNumber.toIntOrNull() ?: 1 }$part" + } else { + row.previousElementSiblings().let { prevElem -> (prevElem.prev("h1,h2,h3,pre:not(:contains(plot))").first()?.text() ?: "Movie - $quality") .replace("Download", "", true).trim().let { if (it.contains("Collection", true)) { @@ -226,60 +170,68 @@ class UHDMovies : ConfigurableAnimeSource, ParsedAnimeHttpSource() { } } } + } + + row.select("a").asSequence() + .filter { el -> el.classNames().none { it.endsWith("-zip") } } + .mapIndexedNotNull { index, linkElement -> + val episode = linkElement.text() + .replace("Episode", "", true) + .trim() + .toIntOrNull() ?: index + 1 + + val url = linkElement.attr("href").takeUnless(String::isBlank) + ?: return@mapIndexedNotNull null - row.select("a").map { linkElement -> - Triple(linkElement.attr("href"), quality, collectionName) + Triple( + Pair(defaultName, episode), + url, + quality, + ) } - }.flatten().groupBy { it.third }.map { group -> - collectionIdx++ - episodeList.add( - SEpisode.create().apply { - url = EpLinks( - urls = group.value.map { - EpUrl(url = it.first, quality = it.second) - }, - ).toJson() - name = group.key - episode_number = collectionIdx + }.flatten().groupBy { it.first }.values.mapIndexed { index, items -> + val (itemName, episodeNum) = items.first().first + + SEpisode.create().apply { + url = EpLinks( + urls = items.map { triple -> + EpUrl(url = triple.second, quality = triple.third) }, - ) + ).toJson() + + name = if (isSerie) "$itemName Ep $episodeNum" else itemName + + episode_number = if (isSerie) episodeNum.toFloat() else (index + 1).toFloat() } - if (episodeList.isEmpty()) throw Exception("Only Zip Pack Available") } - return Observable.just(episodeList.reversed()) + + if (episodeList.isEmpty()) throw Exception("Only Zip Pack Available") + return episodeList.reversed() } - override fun episodeListSelector(): String = throw Exception("Not Used") + override fun episodeListSelector(): String = "p:has(a[href*=?sid=],a[href*=r?key=]):has(a[class*=maxbutton])[style*=center]" override fun episodeFromElement(element: Element): SEpisode = throw Exception("Not Used") // ============================ Video Links ============================= - override fun fetchVideoList(episode: SEpisode): Observable> { val urlJson = json.decodeFromString(episode.url) - val failedMediaUrl = mutableListOf>() - val videoList = mutableListOf