Merge pull request #997 from mediathekview/bugfix/zdfFilm2Partner

Bugfix/zdf film2 partner

MServer-Config.yaml

@@ -1,7 +1,7 @@
 #### Server configurations ####
 
 # The maximum amount of cpu threads to be used.
-maximumCpuThreads: 1
+maximumCpuThreads: 10
 
 # The maximum duration in minutes the server should run.<br>
 # If set to 0 the server runs without a time limit.
@@ -28,11 +28,11 @@ senderIncluded:
  #- FUNK
  #- KIKA
  # - DW
- - ORF
+ # - ORF
  #- PHOENIX
  #- SRF
  #- SR
- #- ZDF
+ - ZDF
 
 #SRF,SR,PHONIX,ORF,KIKA,DW,3SAT<
 
@@ -130,7 +130,7 @@ maximumCrawlDurationInMinutes: 120
 
 # Enables the topics search
 # maximumSubpages limits the depth of the topics search
-topicsSearchEnabled: true
+topicsSearchEnabled: false
 
 # The maximum amount of sub pages to be crawled.<br>
 # Example: If a Sendung overview side has 10 pages with videos for this Sendung and
@@ -176,7 +176,8 @@ senderConfigurations:
  maximumSubpages: 2
  maximumRequestsPerSecond: 8.0
  ZDF:
- maximumRequestsPerSecond: 10.0
+ maximumDaysForSendungVerpasstSection: 21
+ maximumRequestsPerSecond: 20
  FUNK:
  maximumUrlsPerTask: 99
  DREISAT:

src/main/java/de/mediathekview/mserver/base/utils/CheckUrlAvailability.java

@@ -2,6 +2,8 @@
 
 
 import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -19,9 +21,9 @@ public class CheckUrlAvailability {
  private final FileSizeDeterminer fsd;
  private int numberOfThreads = 10;
  private Long minFileSize = 2048L;
- private int removedCounter = 0;
+ private AtomicInteger removedCounter = new AtomicInteger(0);
  private long timeoutInMS = 1*60*1000L;
- private boolean timeout = false;
+ private AtomicBoolean timeout = new AtomicBoolean(false);
  private long start = 0;
 
  public CheckUrlAvailability(final long minFileSize, final long timeoutInSec, final int numberOfThreads) {
@@ -45,13 +47,13 @@ public Filmlist getAvaiableFilmlist(final Filmlist importList) {
  .join();
  customThreadPool.shutdown();
  //
- LOG.debug("checked {} urls and removed {} in {} sec and timeout was reached: {}", importList.getFilms().size(), removedCounter, ((System.currentTimeMillis()-start)/1000), timeout);
+ LOG.debug("checked {} urls and removed {} in {} sec and timeout was reached: {}", importList.getFilms().size(), removedCounter.get(), ((System.currentTimeMillis()-start)/1000), timeout.get());
  return filteredFilmlist;
  }
 
  private boolean isAvailable(Film pFilm) {
- if (timeout || System.currentTimeMillis() > (start+timeoutInMS)) {
- timeout = true;
+ if (timeout.get() || System.currentTimeMillis() > (start+timeoutInMS)) {
+ timeout.set(true);
  return true;
  }
 
@@ -61,21 +63,25 @@ private boolean isAvailable(Film pFilm) {
  if (pFilm.getThema().equalsIgnoreCase("Livestream")) {
  // do not remove livestreams
  return true;
+ } else if (ri == null) {
+ LOG.debug("Film response (null): {} # {} # {} # {} ", normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
+ removedCounter.incrementAndGet();
+ return false;
  } else if (!(ri.getCode() >= 200 && ri.getCode() < 300)) {
  LOG.debug("Film response ({}): {} # {} # {} # {} ", ri.getCode(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
- removedCounter++;
+ removedCounter.incrementAndGet();
  return false;
  } else if (ri.getContentType().equalsIgnoreCase("text/html")) {
  LOG.debug("Film content type({}): {} # {} # {} # {} ", ri.getContentType(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
- removedCounter++;
+ removedCounter.incrementAndGet();
  return false;
  } else if (ri.getSize() < minFileSize && !normalUrl.endsWith("m3u8")) {
  LOG.debug("Film small ({}): {} # {} # {} # {} ", ri.getSize() , normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
- removedCounter++;
+ removedCounter.incrementAndGet();
  return false;
  } else if (removedVideo(pFilm, ri.getPath())) {
  LOG.debug("Film url ({}): {} # {} # {} # {} ", ri.getPath(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
- removedCounter++;
+ removedCounter.incrementAndGet();
  return false;
  }
  return true;

src/main/java/de/mediathekview/mserver/crawler/dreisat/DreiSatCrawler.java

@@ -26,7 +26,7 @@ public DreiSatCrawler(
  final Collection<MessageListener> aMessageListeners,
  final Collection<SenderProgressListener> aProgressListeners,
  final MServerConfigManager rootConfig) {
- super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
+ super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig, DreisatConstants.PARTNER_TO_SENDER);
  }
 
  /**

src/main/java/de/mediathekview/mserver/crawler/dreisat/DreisatConstants.java

@@ -1,5 +1,10 @@
 package de.mediathekview.mserver.crawler.dreisat;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import de.mediathekview.mlib.daten.Sender;
+
 public final class DreisatConstants {
 
  /** Base url of the 3Sat website. */
@@ -13,5 +18,11 @@ public final class DreisatConstants {
 
  public static final String URL_HTML_DAY = URL_BASE + "/programm?airtimeDate=%s";
 
+ public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
+
+ static {
+ PARTNER_TO_SENDER.put("3sat", Sender.DREISAT);
+ }
+
  private DreisatConstants() {}
 }

src/main/java/de/mediathekview/mserver/crawler/phoenix/PhoenixConstants.java

@@ -1,5 +1,10 @@
 package de.mediathekview.mserver.crawler.phoenix;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import de.mediathekview.mlib.daten.Sender;
+
 public final class PhoenixConstants {
  private PhoenixConstants() {}
 
@@ -10,4 +15,11 @@ private PhoenixConstants() {}
  public static final String URL_FILM_DETAIL_JSON = "/response/id/";
 
  public static final String URL_VIDEO_DETAILS = "%s/php/mediaplayer/data/beitrags_details.php?id=%s";
+
+ public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
+
+ static {
+ PARTNER_TO_SENDER.put("Phoenix", Sender.PHOENIX);
+ }
+
 }

src/main/java/de/mediathekview/mserver/crawler/phoenix/tasks/PhoenixFilmDetailTask.java

@@ -72,7 +72,7 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg
  this.filmDetailHost,
  filmDetailDto.getBaseName())));
  final ZdfFilmDetailTask zdfFilmDetailTask =
- new ZdfFilmDetailTask(this.crawler, "", shows, null);
+ new ZdfFilmDetailTask(this.crawler, "", shows, null, PhoenixConstants.PARTNER_TO_SENDER);
  final Set<Film> films = zdfFilmDetailTask.invoke();
  films.forEach(
  film -> {

src/main/java/de/mediathekview/mserver/crawler/zdf/AbstractZdfCrawler.java

@@ -1,6 +1,7 @@
 package de.mediathekview.mserver.crawler.zdf;
 
 import de.mediathekview.mlib.daten.Film;
+import de.mediathekview.mlib.daten.Sender;
 import de.mediathekview.mlib.messages.listener.MessageListener;
 import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.base.messages.ServerMessages;
@@ -19,6 +20,7 @@
 import java.time.temporal.ChronoUnit;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -29,13 +31,16 @@
 public abstract class AbstractZdfCrawler extends AbstractCrawler {
 
  private static final Logger LOG = LogManager.getLogger(AbstractZdfCrawler.class);
+ private final Map<String, Sender> partner2Sender;
 
  protected AbstractZdfCrawler(
  final ForkJoinPool aForkJoinPool,
  final Collection<MessageListener> aMessageListeners,
  final Collection<SenderProgressListener> aProgressListeners,
- final MServerConfigManager rootConfig) {
+ final MServerConfigManager rootConfig,
+ final Map<String, Sender> partner2Sender) {
  super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
+ this.partner2Sender = partner2Sender;
  }
 
  @Override
@@ -60,7 +65,8 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
  this,
  getApiUrlBase(),
  new ConcurrentLinkedQueue<>(shows),
- configuration.getVideoAuthKey().orElse(null));
+ configuration.getVideoAuthKey().orElse(null),
+ partner2Sender);
  } catch (final InterruptedException ex) {
  LOG.debug("{} crawler interrupted.", getSender().getName(), ex);
  Thread.currentThread().interrupt();

src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfConstants.java

@@ -1,5 +1,10 @@
 package de.mediathekview.mserver.crawler.zdf;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import de.mediathekview.mlib.daten.Sender;
+
 public final class ZdfConstants {
 
  /** Name of the header required for authentification. */
@@ -27,6 +32,16 @@ public final class ZdfConstants {
  /** The language key of german audio description. */
  public static final String LANGUAGE_GERMAN_AD = LANGUAGE_GERMAN + LANGUAGE_SUFFIX_AD;
  public static final String LANGUAGE_GERMAN_DGS = LANGUAGE_GERMAN + LANGUAGE_SUFFIX_DGS;
+
+ public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
+
+ static {
+ PARTNER_TO_SENDER.put("ZDFinfo", Sender.ZDF);
+ PARTNER_TO_SENDER.put("ZDFneo", Sender.ZDF);
+ PARTNER_TO_SENDER.put("ZDF", Sender.ZDF); 
+ PARTNER_TO_SENDER.put("EMPTY", Sender.ZDF);
+ // IGNORED Sender [KI.KA, WDR, PHOENIX, one, HR, 3sat, SWR, arte, BR, RBB, ARD, daserste, alpha, MDR, radiobremen, funk, ZDF, NDR, SR]
+ }
 
  private ZdfConstants() {}
 }

src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfCrawler.java

@@ -30,7 +30,7 @@ public ZdfCrawler(
  final Collection<MessageListener> aMessageListeners,
  final Collection<SenderProgressListener> aProgressListeners,
  final MServerConfigManager rootConfig) {
- super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
+ super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig, ZdfConstants.PARTNER_TO_SENDER);
  }
 
  @Override

src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfFilmDto.java

@@ -6,14 +6,19 @@
 import java.util.Objects;
 import java.util.Optional;
 
-public class ZdfFilmDto extends CrawlerUrlDTO {
+public class ZdfFilmDto {
 
- private final Film film;
+ private final Optional<Film> film;
  private final Optional<String> urlSignLanguage;
+ private final Optional<String> videoUrl;
 
- public ZdfFilmDto(final Film film, final String videoUrl, String urlSignLanguage) {
- super(videoUrl);
+ public ZdfFilmDto(final Optional<Film> film, final String videoUrl, String urlSignLanguage) {
  this.film = film;
+ if (videoUrl == null) {
+ this.videoUrl = Optional.empty();
+ } else {
+ this.videoUrl = Optional.of(videoUrl);
+ }
 
  if (urlSignLanguage != null && !urlSignLanguage.isEmpty()) {
  this.urlSignLanguage = Optional.of(urlSignLanguage);
@@ -22,7 +27,11 @@ public ZdfFilmDto(final Film film, final String videoUrl, String urlSignLanguage
  }
  }
 
- public Film getFilm() {
+ public Optional<String> getUrl() {
+ return videoUrl;
+ }
+
+ public Optional<Film> getFilm() {
  return film;
  }
 

src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java

@@ -44,6 +44,7 @@ public class ZdfFilmDetailDeserializer implements JsonDeserializer<Optional<ZdfF
  private static final String JSON_ELEMENT_TITLE = "title";
  private static final String JSON_ELEMENT_TEASER_TEXT = "teasertext";
  private static final String JSON_ATTRIBUTE_TEMPLATE = "http://zdf.de/rels/streams/ptmd-template";
+ private static final String JSON_ELEMENT_TVSERVICE = "tvService";
 
  private static final String PLACEHOLDER_PLAYER_ID = "{playerId}";
  private static final String PLAYER_ID = "android_native_5";
@@ -63,11 +64,11 @@ public class ZdfFilmDetailDeserializer implements JsonDeserializer<Optional<ZdfF
  private static final String[] SEASONNUMBER = {"http://zdf.de/rels/cmdm/season", "seasonNumber"};
 
  private final String apiUrlBase;
- private final Sender sender;
+ private final Map<String, Sender> partner2Sender;
 
- public ZdfFilmDetailDeserializer(final String apiUrlBase, final Sender sender) {
+ public ZdfFilmDetailDeserializer(final String apiUrlBase, Map<String, Sender> partner2Sender) {
  this.apiUrlBase = apiUrlBase;
- this.sender = sender;
+ this.partner2Sender= partner2Sender; 
  }
 
  @Override
@@ -93,7 +94,11 @@ public Optional<ZdfFilmDto> deserialize(
  mainVideoTarget = mainVideo.get(JSON_ELEMENT_TARGET).getAsJsonObject();
  }
  }
-
+ final Optional<String> tvService = JsonUtils.getElementValueAsString(aJsonObject, JSON_ELEMENT_TVSERVICE);
+ if (!partner2Sender.containsKey(tvService.orElse("EMPTY"))) {
+ return Optional.empty();
+ }
+
  final Optional<String> title = parseTitle(rootNode, programItemTarget);
  final Optional<String> topic = parseTopic(rootNode);
  final Optional<String> description = parseDescription(rootNode);
@@ -106,12 +111,8 @@ public Optional<ZdfFilmDto> deserialize(
 
  if (title.isPresent()) {
  final Optional<Film> film =
- createFilm(topic, title.get(), description, website, time, duration);
-
- if (film.isPresent() && downloadUrl.containsKey(DOWNLOAD_URL_DEFAULT)) {
- return Optional.of(new ZdfFilmDto(film.get(), downloadUrl.get(DOWNLOAD_URL_DEFAULT), downloadUrl.get(DOWNLOAD_URL_DGS)));
- }
- LOG.error("ZdfFilmDetailDeserializer: no film or downloadUrl: {}, {}", topic, title.get());
+ createFilm(partner2Sender.get(tvService.orElse("EMPTY")), topic, title.get(), description, website, time, duration);
+ return Optional.of(new ZdfFilmDto(film, downloadUrl.get(DOWNLOAD_URL_DEFAULT), downloadUrl.get(DOWNLOAD_URL_DGS)));
  } else {
  LOG.error("ZdfFilmDetailDeserializer: no title found");
  }
@@ -157,6 +158,7 @@ private String finalizeDownloadUrl(final String url) {
  }
 
  private Optional<Film> createFilm(
+ final Sender sender,
  final Optional<String> aTopic,
  final String aTitle,
  final Optional<String> aDescription,