Extract Video Short Description in YouTube.

In Trending, and Search results.
fuzhengyin · Sep 22, 2021 · 94efe86 · 94efe86
1 parent a9d2144
commit 94efe86
Show file tree

Hide file tree

Showing 18 changed files with 1,398 additions and 0 deletions.
diff --git a/...schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/...schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java
@@ -43,6 +43,12 @@ public DateWrapper getUploadDate() throws ParsingException {
         return BandcampExtractorHelper.parseDate(getTextualUploadDate());
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
+
     @Override
     public String getName() throws ParsingException {
         return show.getString("subtitle");

diff --git a/...ervices/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/...ervices/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java
@@ -26,6 +26,12 @@ public String getUploaderAvatarUrl() {
         return null;
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
+
     @Override
     public String getName() {
         return discograph.getString("title");

diff --git a/.../services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/.../services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java
@@ -60,6 +60,12 @@ public String getUploaderAvatarUrl() {
         return null;
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
+
     /**
      * Each track can have its own cover art. Therefore, unless a substitute is provided,
      * the thumbnail is extracted using a stream extractor.

diff --git a/...or/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/...or/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java
@@ -32,6 +32,12 @@ public String getUploaderAvatarUrl() {
         return null;
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
+
     @Override
     public String getName() throws ParsingException {
         return resultInfo.getElementsByClass("heading").text();

diff --git a/...abi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java b/...abi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java
@@ -95,4 +95,10 @@ public String getTextualUploadDate() throws ParsingException {
     public DateWrapper getUploadDate() throws ParsingException {
         return null;
     }
+
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
 }
diff --git a/.../schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java b/.../schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java
@@ -92,4 +92,10 @@ public DateWrapper getUploadDate() throws ParsingException {
                 DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSzzzz"));
         return new DateWrapper(zonedDateTime.toOffsetDateTime(), false);
     }
+
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
 }
diff --git a/...pe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/...pe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java
@@ -73,6 +73,12 @@ public DateWrapper getUploadDate() throws ParsingException {
         return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date));
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
+
     @Override
     public String getName() throws ParsingException {
         return event.getString("title");

diff --git a/...chabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/...chabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java
@@ -92,6 +92,12 @@ public DateWrapper getUploadDate() throws ParsingException {
         return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
+
     @Override
     public StreamType getStreamType() {
         return item.getBoolean("isLive") ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM;

diff --git a/...i/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/...i/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java
@@ -66,6 +66,12 @@ public DateWrapper getUploadDate() throws ParsingException {
         return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate()));
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
+
     @Override
     public long getViewCount() {
         return itemObject.getLong("playback_count");

diff --git a/...rg/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/...rg/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java
@@ -77,6 +77,12 @@ public DateWrapper getUploadDate() throws ParsingException {
         }
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return null;
+    }
+
     @Override
     public String getName() {
         return entryElement.getElementsByTag("title").first().text();

diff --git a/.../schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/.../schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
@@ -288,4 +288,17 @@ private OffsetDateTime getDateFromPremiere() throws ParsingException {
             throw new ParsingException("Could not parse date from premiere:  \"" + startTime + "\"");
         }
     }
+
+    @Nullable
+    @Override
+    public String getShortDescription() throws ParsingException {
+
+        if(videoInfo.has("detailedMetadataSnippets"))
+            return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets").getObject(0).getObject("snippetText"));
+
+        if(videoInfo.has("descriptionSnippet"))
+            return  getTextFromObject(videoInfo.getObject("descriptionSnippet"));
+
+        return null;
+    }
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java
@@ -32,6 +32,7 @@ public class StreamInfoItem extends InfoItem {
     private final StreamType streamType;
 
     private String uploaderName;
+    private String shortDescription;
     private String textualUploadDate;
     @Nullable
     private DateWrapper uploadDate;
@@ -92,6 +93,14 @@ public void setUploaderAvatarUrl(final String uploaderAvatarUrl) {
         this.uploaderAvatarUrl = uploaderAvatarUrl;
     }
 
+    public String getShortDescription() {
+        return shortDescription;
+    }
+
+    public void setShortDescription(String shortDescription) {
+        this.shortDescription = shortDescription;
+    }
+
     @Nullable
     public String getTextualUploadDate() {
         return textualUploadDate;

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java
@@ -116,4 +116,14 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
     @Nullable
     DateWrapper getUploadDate() throws ParsingException;
 
+
+    /**
+     * Get the video's short description.
+     *
+     * @return The video's short description or {@code null} if not provided by the service.
+     * @throws ParsingException if there is an error in the extraction
+     */
+    @Nullable
+    String getShortDescription() throws ParsingException;
+
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java
@@ -101,6 +101,11 @@ public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws ParsingE
         } catch (Exception e) {
             addError(e);
         }
+        try {
+            resultItem.setShortDescription(extractor.getShortDescription());
+        } catch (Exception e) {
+            addError(e);
+        }
 
         return resultItem;
     }

diff --git a/...java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/...java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java
@@ -343,4 +343,34 @@ public void testUploaderAvatar() throws IOException, ExtractionException {
             }
         }
     }
+
+    public static class VideoDescription extends DefaultSearchExtractorTest {
+        private static SearchExtractor extractor;
+        private static final String QUERY = "44wLAzydRFU";
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "video_description"));
+            extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
+            extractor.fetchPage();
+        }
+
+        @Override public SearchExtractor extractor() { return extractor; }
+        @Override public StreamingService expectedService() { return YouTube; }
+        @Override public String expectedName() { return QUERY; }
+        @Override public String expectedId() { return QUERY; }
+        @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
+        @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
+        @Override public String expectedSearchString() { return QUERY; }
+        @Nullable @Override public String expectedSearchSuggestion() { return null; }
+        @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
+
+        @Test
+        public void testVideoDescription() throws IOException, ExtractionException {
+            final List<InfoItem> items = extractor.getInitialPage().getItems();
+            assertNotNull(((StreamInfoItem) items.get(0)).getShortDescription());
+        }
+    }
 }