Skip to content

Commit

Permalink
Merge branch 'dev' of https://github.com/fuzhengyin/NewPipeExtractor …
Browse files Browse the repository at this point in the history 
…into dev
  • Loading branch information
@fuzhengyin
fuzhengyin committed Dec 6, 2021
2 parents 8f32465 + f6f2724 commit 0bd2035
Show file tree
Hide file tree
Showing 16 changed files with 189 additions and 78 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ allprojects {

ext {
nanojsonVersion = "1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751"
spotbugsVersion = "4.4.2"
spotbugsVersion = "4.5.0"
junitVersion = "4.13.2"
}
}
Expand Down
2 changes: 1 addition & 1 deletion extractor/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ dependencies {

testImplementation "junit:junit:$junitVersion"
testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
testImplementation 'com.google.code.gson:gson:2.8.8'
testImplementation 'com.google.code.gson:gson:2.8.9'
}
6 changes: 3 additions & 3 deletions extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ public static CommentsInfo getInfo(final String url) throws IOException, Extract
return getInfo(NewPipe.getServiceByUrl(url), url);
}

public static CommentsInfo getInfo(final StreamingService serviceByUrl, final String url)
public static CommentsInfo getInfo(final StreamingService service, final String url)
throws ExtractionException, IOException {
return getInfo(serviceByUrl.getCommentsExtractor(url));
return getInfo(service.getCommentsExtractor(url));
}

public static CommentsInfo getInfo(final CommentsExtractor commentsExtractor)
Expand Down Expand Up @@ -63,7 +63,7 @@ public static InfoItemsPage<CommentsInfoItem> getMoreItems(
final StreamingService service,
final CommentsInfo commentsInfo,
final Page page) throws IOException, ExtractionException {
if (null == commentsInfo.getCommentsExtractor()) {
if (commentsInfo.getCommentsExtractor() == null) {
commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl()));
commentsInfo.getCommentsExtractor().fetchPage();
}
Expand Down
30 changes: 26 additions & 4 deletions ...bi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {

private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)";
private static final String ID_PATTERN = "((accounts|a)|(video-channels|c))/([^/?&#]*)";
public static final String API_ENDPOINT = "/api/v1/";

public static PeertubeChannelLinkHandlerFactory getInstance() {
Expand All @@ -19,7 +19,7 @@ public static PeertubeChannelLinkHandlerFactory getInstance() {

@Override
public String getId(String url) throws ParsingException {
return Parser.matchGroup(ID_PATTERN, url, 0);
return fixId(Parser.matchGroup(ID_PATTERN, url, 0));
}

@Override
Expand All @@ -31,7 +31,7 @@ public String getUrl(String id, List<String> contentFilters, String searchFilter
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
throws ParsingException {
if (id.matches(ID_PATTERN)) {
return baseUrl + "/" + id;
return baseUrl + "/" + fixId(id);
} else {
// This is needed for compatibility with older versions were we didn't support video channels yet
return baseUrl + "/accounts/" + id;
Expand All @@ -40,6 +40,28 @@ public String getUrl(String id, List<String> contentFilter, String sortFilter, S

@Override
public boolean onAcceptUrl(String url) {
return url.contains("/accounts/") || url.contains("/video-channels/");
return url.contains("/accounts/") || url.contains("/a/")
|| url.contains("/video-channels/") || url.contains("/c/");
}

/**
* Fix id
*
* <p>
* a/:accountName and c/:channelName ids are supported
* by the PeerTube web client (>= v3.3.0)
* but not by the API.
* </p>
*
* @param id the id to fix
* @return the fixed id
*/
private String fixId(String id) {
if (id.startsWith("a/")) {
id = "accounts" + id.substring(1);
} else if (id.startsWith("c/")) {
id = "video-channels" + id.substring(1);
}
return id;
}
}
5 changes: 2 additions & 3 deletions ...i/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {

private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory();
private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)";
private static final String COMMENTS_ENDPOINT = "/api/v1/videos/%s/comment-threads";

public static PeertubeCommentsLinkHandlerFactory getInstance() {
Expand All @@ -20,12 +19,12 @@ public static PeertubeCommentsLinkHandlerFactory getInstance() {

@Override
public String getId(String url) throws ParsingException, IllegalArgumentException {
return Parser.matchGroup(ID_PATTERN, url, 2);
return PeertubeStreamLinkHandlerFactory.getInstance().getId(url); // the same id is needed
}

@Override
public boolean onAcceptUrl(final String url) throws FoundAdException {
return url.contains("/videos/");
return url.contains("/videos/") || url.contains("/w/");
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions ...i/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {

private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory();
private static final String ID_PATTERN = "/videos/watch/playlist/([^/?&#]*)";
private static final String ID_PATTERN = "(/videos/watch/playlist/|/w/p/)([^/?&#]*)";

public static PeertubePlaylistLinkHandlerFactory getInstance() {
return instance;
Expand All @@ -30,7 +30,7 @@ public String getUrl(String id, List<String> contentFilters, String sortFilter,

@Override
public String getId(String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
return Parser.matchGroup(ID_PATTERN, url, 2);
}

@Override
Expand Down
9 changes: 7 additions & 2 deletions ...abi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,13 @@
public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {

private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory();
private static final String ID_PATTERN = "/videos/(watch/|embed/)?([^/?&#]*)";
private static final String ID_PATTERN = "(/w/|(/videos/(watch/|embed/)?))(?!p/)([^/?&#]*)";
// we exclude p/ because /w/p/ is playlist, not video
public static final String VIDEO_API_ENDPOINT = "/api/v1/videos/";

// From PeerTube 3.3.0, the default path is /w/.
// We still use /videos/watch/ for compatibility reasons:
// /videos/watch/ is still accepted by >=3.3.0 but /w/ isn't by <3.3.0
private static final String VIDEO_PATH = "/videos/watch/";

private PeertubeStreamLinkHandlerFactory() {
Expand All @@ -32,7 +37,7 @@ public String getUrl(String id, String baseUrl) {

@Override
public String getId(String url) throws ParsingException, IllegalArgumentException {
return Parser.matchGroup(ID_PATTERN, url, 2);
return Parser.matchGroup(ID_PATTERN, url, 4);
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion ...c/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@

public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID =
"yoxLvaFlJ3V5LbNCt53Cwvw5KXKKxWfn"; // Updated on 01/10/21
"nGKlrpy2IotLQ0QGwBOmIgSFayis6H4e"; // Updated on 04/11/21
private static String clientId;
public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/";

Expand Down
4 changes: 4 additions & 0 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,10 @@ public static boolean isInvidioURL(@Nonnull final URL url) {
|| host.equalsIgnoreCase("y.com.cm");
}

public static boolean isY2ubeURL(@Nonnull final URL url) {
return url.getHost().equalsIgnoreCase("y2u.be");
}

/**
* Parses the duration string of the video expecting ":" or "." as separators
*
Expand Down
63 changes: 38 additions & 25 deletions ...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,19 +337,33 @@ public long getViewCount() throws ParsingException {
@Override
public long getLikeCount() throws ParsingException {
assertPageFetched();
String likesString = "";
String likesString = null;
try {
try {
likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
.getObject("sentimentBarRenderer").getString("tooltip").split("/")[0];
} catch (final NullPointerException e) {
likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
.getObject("sentimentBarRenderer").getString("tooltip");
if (likesString != null && likesString.contains("/")) {
likesString = likesString.split("/")[0];
} else {
likesString = getVideoPrimaryInfoRenderer()
.getObject("videoActions")
.getObject("menuRenderer")
.getArray("topLevelButtons")
.getObject(0)
.getObject("toggleButtonRenderer")
.getObject("defaultText")
.getObject("accessibility")
.getObject("accessibilityData")
.getString("label");
}

if (likesString == null) {
// If this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException(
"Ratings are enabled even though the like button is missing", e);
throw new ParsingException("Ratings are enabled even though the like button is missing");
}
return -1;
}

return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
} catch (final NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer",
Expand All @@ -366,29 +380,28 @@ public long getLikeCount() throws ParsingException {
public long getDislikeCount() throws ParsingException {
assertPageFetched();

String dislikesString = "";
try {
try {
dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
.getObject("sentimentBarRenderer").getString("tooltip").split("/")[1];
} catch (final NullPointerException e) {
// If this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException(
"Ratings are enabled even though the dislike button is missing", e);
String dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
.getObject("sentimentBarRenderer").getString("tooltip");
if (dislikesString != null && dislikesString.contains("/")) {
dislikesString = dislikesString.split("/")[1];
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
} else {
// Calculate dislike with average rating and like count
long likes = getLikeCount();
double averageRating = playerResponse.getObject("videoDetails").getDouble("averageRating");

if (likes != -1 && averageRating > 1) {
// If averageRating can't be gathered, it will be 0,
// but we also can't divide by 0 so we need > 1
return Math.round(likes * ((5 - averageRating) / (averageRating - 1)));
}
return -1;
}
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
} catch (final NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer",
nfe);
} catch (final Exception e) {
if (getAgeLimit() == NO_AGE_LIMIT) {
throw new ParsingException("Could not get dislike count", e);
}
return -1;
}
// Silently fail as YouTube is "gradually rolling out" removing dislike count
// https://blog.youtube/news-and-events/update-to-youtube/
return -1;
}

@Nonnull
Expand Down
3 changes: 2 additions & 1 deletion ...chabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public String getId(String urlString) throws ParsingException, IllegalArgumentEx

if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) ||
YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) ||
YoutubeParsingHelper.isInvidioURL(url))) {
YoutubeParsingHelper.isInvidioURL(url) || YoutubeParsingHelper.isY2ubeURL(url))) {
if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
throw new FoundAdException("Error found ad: " + urlString);
}
Expand Down Expand Up @@ -160,6 +160,7 @@ public String getId(String urlString) throws ParsingException, IllegalArgumentEx
return assertIsId(viewQueryValue);
}

case "Y2U.BE":
case "YOUTU.BE": {
String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {
Expand Down
49 changes: 30 additions & 19 deletions ...org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,44 +20,55 @@ public class PeertubeChannelLinkHandlerFactoryTest {

@BeforeClass
public static void setUp() {
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
PeerTube.setInstance(new PeertubeInstance("https://peertube.stream", "PeerTube on peertube.stream"));
linkHandler = PeertubeChannelLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}

@Test
public void acceptUrlTest() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/[email protected]"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/accounts/[email protected]/videos"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"));
assertTrue(linkHandler.acceptUrl("https://peertube.stream/accounts/[email protected]"));
assertTrue(linkHandler.acceptUrl("https://peertube.stream/a/[email protected]"));
assertTrue(linkHandler.acceptUrl("https://peertube.stream/api/v1/accounts/[email protected]/videos"));
assertTrue(linkHandler.acceptUrl("https://peertube.stream/video-channels/[email protected]/videos"));
assertTrue(linkHandler.acceptUrl("https://peertube.stream/c/[email protected]/videos"));
assertTrue(linkHandler.acceptUrl("https://peertube.stream/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"));
}

@Test
public void getId() throws ParsingException {
assertEquals("accounts/[email protected]",
linkHandler.fromUrl("https://peertube.mastodon.host/accounts/[email protected]").getId());
linkHandler.fromUrl("https://peertube.stream/accounts/[email protected]").getId());
assertEquals("accounts/[email protected]",
linkHandler.fromUrl("https://peertube.mastodon.host/accounts/[email protected]/videos").getId());
assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId());
linkHandler.fromUrl("https://peertube.stream/a/[email protected]").getId());
assertEquals("accounts/[email protected]",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/accounts/[email protected]").getId());
linkHandler.fromUrl("https://peertube.stream/accounts/[email protected]/videos").getId());
assertEquals("accounts/[email protected]",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/accounts/[email protected]/videos").getId());
assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getId());
linkHandler.fromUrl("https://peertube.stream/a/[email protected]/videos").getId());
assertEquals("accounts/[email protected]",
linkHandler.fromUrl("https://peertube.stream/api/v1/accounts/[email protected]").getId());
assertEquals("accounts/[email protected]",
linkHandler.fromUrl("https://peertube.stream/api/v1/accounts/[email protected]/videos").getId());

assertEquals("video-channels/[email protected]",
linkHandler.fromUrl("https://peertube.stream/video-channels/[email protected]/videos").getId());
assertEquals("video-channels/[email protected]",
linkHandler.fromUrl("https://peertube.stream/c/[email protected]/videos").getId());
assertEquals("video-channels/[email protected]",
linkHandler.fromUrl("https://peertube.stream/c/[email protected]/video-playlists").getId());
assertEquals("video-channels/[email protected]",
linkHandler.fromUrl("https://peertube.stream/api/v1/video-channels/[email protected]").getId());
}

@Test
public void getUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl());
assertEquals("https://peertube.mastodon.host/accounts/[email protected]",
assertEquals("https://peertube.stream/video-channels/[email protected]",
linkHandler.fromId("video-channels/[email protected]").getUrl());
assertEquals("https://peertube.stream/accounts/[email protected]",
linkHandler.fromId("accounts/[email protected]").getUrl());
assertEquals("https://peertube.mastodon.host/accounts/[email protected]",
assertEquals("https://peertube.stream/accounts/[email protected]",
linkHandler.fromId("[email protected]").getUrl());
assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl());
assertEquals("https://peertube.stream/video-channels/[email protected]",
linkHandler.fromUrl("https://peertube.stream/api/v1/video-channels/[email protected]").getUrl());
}
}
Loading

0 comments on commit 0bd2035

Please sign in to comment.