Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix YouTube Music search #484

Merged
merged 4 commits into from
Dec 25, 2020
Merged

Fix YouTube Music search #484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c9d9bd1
Fix typo in ListLinkHandlerFactory#getUrl
TobiGr Dec 17, 2020
b62144b
[YouTube] Music: Fix search extraction caused by changed JSON response
TobiGr Dec 22, 2020
3033c0b
[YouTube] Music Mix: Fix playlist not being accepted
TobiGr Dec 23, 2020
d9e2da5
Replace explicit string checks whether a playlist íd is a certain You…
TobiGr Dec 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public ListLinkHandler fromQuery(String id,
* however it should not be overridden by the actual implementation.
*
* @param id
* @return the url coresponding to id without any filters applied
* @return the url corresponding to id without any filters applied
*/
public String getUrl(String id) throws ParsingException {
return getUrl(id, new ArrayList<String>(0), "");
Expand Down
16 changes: 8 additions & 8 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,12 +205,12 @@ public static boolean isYoutubeMixId(final String playlistId) {

/**
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RDAMVM"
* Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
* @param playlistId
* @return Whether given id belongs to a YouTube Music Mix
*/
public static boolean isYoutubeMusicMixId(final String playlistId) {
return playlistId.startsWith("RDAMVM");
return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
}
/**
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
Expand All @@ -226,20 +226,20 @@ public static boolean isYoutubeChannelMixId(final String playlistId) {
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
*/
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
if (playlistId.startsWith("RDMM")) { //My Mix
if (playlistId.startsWith("RDMM")) { // My Mix
return playlistId.substring(4);

} else if (playlistId.startsWith("RDAMVM")) { //Music mix
} else if (isYoutubeMusicMixId(playlistId)) { // starts with "RDAMVM" or "RDCLAK"
return playlistId.substring(6);

} else if (playlistId.startsWith("RMCM")) { //Channel mix
//Channel mix are build with RMCM{channelId}, so videoId can't be determined
} else if (isYoutubeChannelMixId(playlistId)) { // starts with "RMCM"
// Channel mix are build with RMCM{channelId}, so videoId can't be determined
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);

} else if (playlistId.startsWith("RD")) { // Normal mix
} else if (isYoutubeMixId(playlistId)) { // normal mix, starts with "RD"
return playlistId.substring(2);

} else { //not a mix
} else { // not a mix
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
}
}
Expand Down
3 changes: 2 additions & 1 deletion extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {

@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())
&& !YoutubeParsingHelper.isYoutubeMusicMixId(linkHandler.getId())) {
return new YoutubeMixPlaylistExtractor(this, linkHandler);
} else {
return new YoutubePlaylistExtractor(this, linkHandler);
Expand Down
69 changes: 50 additions & 19 deletions ...org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;

Expand Down Expand Up @@ -258,16 +260,29 @@ private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, f
final TimeAgoParser timeAgoParser = getTimeAgoParser();

for (Object item : videos) {
final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer", null);
final JsonObject info = ((JsonObject) item)
.getObject("musicResponsiveListItemRenderer", null);
if (info != null) {
final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", EMPTY_STRING);
if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) {
continue; // no info about video URL available
}

final JsonObject flexColumnRenderer = info
.getArray("flexColumns")
.getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer");
final JsonArray descriptionElements = flexColumnRenderer
.getObject("text")
.getArray("runs");
final String searchType = getLinkHandler().getContentFilters().get(0);
if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) {
@Override
public String getUrl() throws ParsingException {
final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
if (!isNullOrEmpty(url)) {
return url;
final String id = info.getObject("playlistItemData").getString("videoId");
if (!isNullOrEmpty(id)) {
return "https://music.youtube.com/watch?v=" + id;
}
throw new ParsingException("Could not get url");
}
Expand All @@ -284,8 +299,9 @@ public String getName() throws ParsingException {

@Override
public long getDuration() throws ParsingException {
final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String duration = descriptionElements
.getObject(descriptionElements.size() - 1)
.getString("text");
if (!isNullOrEmpty(duration)) {
return YoutubeParsingHelper.parseDurationString(duration);
}
Expand All @@ -294,8 +310,7 @@ public long getDuration() throws ParsingException {

@Override
public String getUploaderName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String name = descriptionElements.getObject(0).getString("text");
if (!isNullOrEmpty(name)) {
return name;
}
Expand Down Expand Up @@ -346,8 +361,9 @@ public long getViewCount() throws ParsingException {
if (searchType.equals(MUSIC_SONGS)) {
return -1;
}
final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String viewCount = descriptionElements
.getObject(descriptionElements.size() - 3)
.getString("text");
if (!isNullOrEmpty(viewCount)) {
return Utils.mixedNumberWordToLong(viewCount);
}
Expand Down Expand Up @@ -451,9 +467,27 @@ public String getName() throws ParsingException {

@Override
public String getUrl() throws ParsingException {
final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
if (!isNullOrEmpty(url)) {
return url;
String playlistId = info.getObject("menu")
.getObject("menuRenderer")
.getArray("items")
.getObject(4)
.getObject("toggleMenuServiceItemRenderer")
.getObject("toggledServiceEndpoint")
.getObject("likeEndpoint")
.getObject("target")
.getString("playlistId");

if (isNullOrEmpty(playlistId)) {
playlistId = info.getObject("overlay")
.getObject("musicItemThumbnailOverlayRenderer")
.getObject("content")
.getObject("musicPlayButtonRenderer")
.getObject("playNavigationEndpoint")
.getObject("watchPlaylistEndpoint")
.getString("playlistId");
}
if (!isNullOrEmpty(playlistId)) {
return "https://music.youtube.com/playlist?list=" + playlistId;
}
throw new ParsingException("Could not get url");
}
Expand All @@ -462,11 +496,9 @@ public String getUrl() throws ParsingException {
public String getUploaderName() throws ParsingException {
final String name;
if (searchType.equals(MUSIC_ALBUMS)) {
name = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
name = descriptionElements.getObject(2).getString("text");
} else {
name = getTextFromObject(info.getArray("flexColumns").getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
name = descriptionElements.getObject(0).getString("text");
}
if (!isNullOrEmpty(name)) {
return name;
Expand All @@ -479,8 +511,7 @@ public long getStreamCount() throws ParsingException {
if (searchType.equals(MUSIC_ALBUMS)) {
return ITEM_COUNT_UNKNOWN;
}
final String count = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String count = descriptionElements.getObject(2).getString("text");
if (!isNullOrEmpty(count)) {
if (count.contains("100+")) {
return ITEM_COUNT_MORE_THAN_100;
Expand Down
5 changes: 0 additions & 5 deletions ...abi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,6 @@ public String getId(final String url) throws ParsingException {
"the list-ID given in the URL does not match the list pattern");
}

if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) {
throw new ContentNotSupportedException(
"YouTube Music Mix playlists are not yet supported");
}

if (YoutubeParsingHelper.isYoutubeChannelMixId(listID)
&& Utils.getQueryValue(urlObj, "v") == null) {
//Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId
Expand Down