Skip to content

Commit

Permalink
#938 use url optimizers in kika crawler
Browse files Browse the repository at this point in the history
  • Loading branch information
@pidoubleyou
pidoubleyou committed Nov 12, 2023
1 parent 769698e commit b1af62d
Show file tree
Hide file tree
Showing 4 changed files with 58 additions and 16 deletions.
41 changes: 34 additions & 7 deletions src/main/java/de/mediathekview/mserver/crawler/ard/ArdUrlOptimizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,34 @@

import de.mediathekview.mserver.crawler.basic.AbstractCrawler;

public class ArdUrlOptimizer {
import java.util.HashMap;
import java.util.Map;

public static final String ARD_URL_1280 = ".xl.mp4";
public static final String ARD_URL_1920 = ".xxl.mp4";
public class ArdUrlOptimizer {

private static final String BR_URL_1280 = "_X.mp4";
private static final String BR_URL_1920 = "_HD.mp4";
private static final String HR_URL_1280 = "1280x720-50p-3200kbit.mp4";
private static final String HR_URL_1920 = "1920x1080-50p-5000kbit.mp4";
private static final String NDR_URL_1280 = ".hd.mp4";
private static final String NDR_URL_1920 = ".1080.mp4";
private static final String RBB_URL_1280 = "hd1080-avc720.mp4";
private static final String RBB_URL_1920 = "hd1080-avc1080.mp4";
private static final String SR_URL_1280 = "_P.mp4";
private static final String SR_URL_1920 = "_H.mp4";
private static final String SWR_URL_1280 = ".xl.mp4";
private static final String SWR_URL_1920 = ".xxl.mp4";

private static final Map<String, String[]> HD_OPTIMIZE = new HashMap<>();

static {
HD_OPTIMIZE.put(BR_URL_1280, new String[] {BR_URL_1920});
HD_OPTIMIZE.put(HR_URL_1280, new String[] {HR_URL_1920});
HD_OPTIMIZE.put(NDR_URL_1280, new String[] {NDR_URL_1920});
HD_OPTIMIZE.put(RBB_URL_1280, new String[] {RBB_URL_1920});
HD_OPTIMIZE.put(SR_URL_1280, new String[] {SR_URL_1920});
HD_OPTIMIZE.put(SWR_URL_1280, new String[] {SWR_URL_1920});
}

protected AbstractCrawler crawler;

Expand All @@ -15,10 +38,14 @@ public ArdUrlOptimizer(AbstractCrawler aCrawler) {
}

public String optimizeHdUrl(final String url) {
if (url.contains(ARD_URL_1280)) {
final String optimizedUrl = url.replace(ARD_URL_1280, ARD_URL_1920);
if (crawler.requestUrlExists(optimizedUrl)) {
return optimizedUrl;
for (Map.Entry<String, String[]> entry : HD_OPTIMIZE.entrySet()) {
if (url.contains(entry.getKey())) {
for (String optimizeFragment : entry.getValue()) {
final String optimizedUrl = url.replace(entry.getKey(), optimizeFragment);
if (crawler.requestUrlExists(optimizedUrl)) {
return optimizedUrl;
}
}
}
}

Expand Down
27 changes: 20 additions & 7 deletions src/main/java/de/mediathekview/mserver/crawler/kika/tasks/KikaApiFilmTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
import java.util.Set;
import java.util.UUID;

import de.mediathekview.mserver.crawler.ard.ArdUrlOptimizer;
import de.mediathekview.mserver.crawler.zdf.ZdfVideoUrlOptimizer;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand All @@ -41,8 +43,13 @@ public class KikaApiFilmTask extends AbstractJsonRestTask<Film, KikaApiVideoInfo
private static final long serialVersionUID = 1L;
private static final Logger LOG = LogManager.getLogger(KikaApiFilmTask.class);

private transient ArdUrlOptimizer ardUrlOptimizer;
private transient ZdfVideoUrlOptimizer zdfVideoUrlOptimizer;

public KikaApiFilmTask(AbstractCrawler crawler, Queue<KikaApiFilmDto> urlToCrawlDTOs) {
super(crawler, urlToCrawlDTOs, null);
ardUrlOptimizer = new ArdUrlOptimizer(crawler);
zdfVideoUrlOptimizer = new ZdfVideoUrlOptimizer(crawler);
}

@Override
Expand Down Expand Up @@ -122,7 +129,7 @@ protected AbstractRecursiveConverterTask<Film, KikaApiFilmDto> createNewOwnInsta
}

protected Optional<LocalDateTime> getAiredDateTime(KikaApiFilmDto aDTO) {
Optional<LocalDateTime> airedDate = null;
Optional<LocalDateTime> airedDate;
if (aDTO.getDate().isPresent()) {
airedDate = parseLocalDateTime(aDTO, aDTO.getDate());
} else {
Expand Down Expand Up @@ -151,12 +158,18 @@ protected Set<URL> getSubtitle(KikaApiVideoInfoDto aResponseObj, KikaApiFilmDto
protected Map<Resolution,FilmUrl> getVideoUrls(KikaApiVideoInfoDto aResponseObj, KikaApiFilmDto aDTO) {
Map<Resolution, FilmUrl> urls = new EnumMap<>(Resolution.class);
for (Map.Entry<Resolution,String> element : aResponseObj.getVideoUrls().entrySet()) {
try {
final FilmUrl filmUrl = new FilmUrl(element.getValue(), crawler.determineFileSizeInKB(element.getValue()));
urls.put(element.getKey(), filmUrl);
} catch (MalformedURLException e) {
LOG.error("Invalid video url {} for {} error {}", element.getValue(), aDTO.getUrl(), e);
}
try {
String url = element.getValue();
if (Resolution.HD.equals(element.getKey())) {
url = ardUrlOptimizer.optimizeHdUrl(url);
url = zdfVideoUrlOptimizer.getOptimizedUrlHd(url);
}

final FilmUrl filmUrl = new FilmUrl(url, crawler.determineFileSizeInKB(url));
urls.put(element.getKey(), filmUrl);
} catch (MalformedURLException e) {
LOG.error("Invalid video url {} for {} error {}", element.getValue(), aDTO.getUrl(), e);
}
}
return urls;
}
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfVideoUrlOptimizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public class ZdfVideoUrlOptimizer {
private static final String HD_3296_15_13 = "3296k_p15v13.mp4";
private static final String HD_3296_15_14 = "3296k_p15v14.mp4";
private static final String HD_3328_15_15 = "3328k_p15v15.mp4";
private static final String HD_3328_15_17 = "3328k_p15v17.mp4";
private static final String HD_3328_12 = "3328k_p36v12.mp4";
private static final String HD_3328_13 = "3328k_p36v13.mp4";
private static final String HD_3328_14 = "3328k_p36v14.mp4";
Expand Down Expand Up @@ -78,6 +79,7 @@ public class ZdfVideoUrlOptimizer {

HD_OPTIMIZE.put(HD_3360_36_17, new String[] {HD_6660_37_17, HD_6628_61_17});
HD_OPTIMIZE.put(HD_6628_61_17, new String[] {HD_6660_37_17});
HD_OPTIMIZE.put(HD_3328_15_17, new String[] {HD_6660_37_17, HD_6628_61_17, HD_3360_36_17});
HD_OPTIMIZE.put(HD_3328_15_15, new String[] {HD_3360_36_15});
HD_OPTIMIZE.put(HD_3256, new String[] {HD_3328_12});
HD_OPTIMIZE.put(HD_3296_15_14, new String[] {HD_3328_36_14});
Expand Down
4 changes: 2 additions & 2 deletions ...test/java/de/mediathekview/mserver/crawler/ard/json/ArdVideoInfoJsonDeserializerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public static Collection<Object[]> data() {
"",
"https://mediastorage01.sr-online.de/Video/UD/DOKU/1505155201_20170911_KANDIDATENCHECK_LUKSIC_M.mp4",
"https://srstorage01-a.akamaihd.net/Video/UD/DOKU/1505155201_20170911_KANDIDATENCHECK_LUKSIC_L.mp4",
"https://srstorage01-a.akamaihd.net/Video/UD/DOKU/1505155201_20170911_KANDIDATENCHECK_LUKSIC_P.mp4"
"https://srstorage01-a.akamaihd.net/Video/UD/DOKU/1505155201_20170911_KANDIDATENCHECK_LUKSIC_H.mp4"
},
{
"/ard/ard_video_use_http_url.json",
Expand Down Expand Up @@ -144,7 +144,7 @@ public static Collection<Object[]> data() {
"",
"https://srstorage01-a.akamaihd.net/Video/FS/SA/sportarena_20190815_184401_M.mp4",
"https://srstorage01-a.akamaihd.net/Video/FS/SA/sportarena_20190815_184401_L.mp4",
"https://srstorage01-a.akamaihd.net/Video/FS/SA/sportarena_20190815_184401_P.mp4"
"https://srstorage01-a.akamaihd.net/Video/FS/SA/sportarena_20190815_184401_H .mp4"
},
{
"/ndr/ndr_film_detail_m3u8.json",
Expand Down

0 comments on commit b1af62d

Please sign in to comment.