From 72c9c538ef1c98d901ced8263bd60544b2575d99 Mon Sep 17 00:00:00 2001 From: Harold Date: Wed, 22 May 2024 19:32:42 +0800 Subject: [PATCH] youtube --- .../alist_tvbox/service/SettingService.java | 4 + .../alist_tvbox/youtube/YoutubeService.java | 12 +- .../kiulian/downloader/YoutubeDownloader.java | 115 +++++++++++++++ .../downloader/model/BrowseRequest.java | 13 ++ .../kiulian/downloader/parser/Parser.java | 46 ++++++ .../kiulian/downloader/parser/ParserImpl.java | 131 ++++++++++++++++++ web-ui/src/views/AccountsView.vue | 5 + web-ui/src/views/YouTubeView.vue | 41 ++++++ 8 files changed, 364 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/github/kiulian/downloader/YoutubeDownloader.java create mode 100644 src/main/java/com/github/kiulian/downloader/model/BrowseRequest.java create mode 100644 src/main/java/com/github/kiulian/downloader/parser/Parser.java create mode 100644 web-ui/src/views/YouTubeView.vue diff --git a/src/main/java/cn/har01d/alist_tvbox/service/SettingService.java b/src/main/java/cn/har01d/alist_tvbox/service/SettingService.java index 88fc5ba6c5..6ac956009a 100644 --- a/src/main/java/cn/har01d/alist_tvbox/service/SettingService.java +++ b/src/main/java/cn/har01d/alist_tvbox/service/SettingService.java @@ -113,6 +113,10 @@ public Setting get(String name) { return settingRepository.findById(name).orElse(null); } + public String getString(String name) { + return settingRepository.findById(name).map(Setting::getValue).orElse(null); + } + public Setting update(Setting setting) { if ("merge_site_source".equals(setting.getName())) { appProperties.setMerge("true".equals(setting.getValue())); diff --git a/src/main/java/cn/har01d/alist_tvbox/youtube/YoutubeService.java b/src/main/java/cn/har01d/alist_tvbox/youtube/YoutubeService.java index 8c993413df..daf3d4b286 100644 --- a/src/main/java/cn/har01d/alist_tvbox/youtube/YoutubeService.java +++ b/src/main/java/cn/har01d/alist_tvbox/youtube/YoutubeService.java @@ -3,6 +3,7 @@ import cn.har01d.alist_tvbox.config.AppProperties; import cn.har01d.alist_tvbox.model.Filter; import cn.har01d.alist_tvbox.model.FilterValue; +import cn.har01d.alist_tvbox.service.SettingService; import cn.har01d.alist_tvbox.service.SubscriptionService; import cn.har01d.alist_tvbox.tvbox.Category; import cn.har01d.alist_tvbox.tvbox.CategoryList; @@ -22,6 +23,7 @@ import com.github.kiulian.downloader.downloader.request.RequestVideoInfo; import com.github.kiulian.downloader.downloader.request.RequestVideoStreamDownload; import com.github.kiulian.downloader.downloader.response.Response; +import com.github.kiulian.downloader.model.BrowseRequest; import com.github.kiulian.downloader.model.Extension; import com.github.kiulian.downloader.model.playlist.PlaylistInfo; import com.github.kiulian.downloader.model.playlist.PlaylistVideoDetails; @@ -86,11 +88,11 @@ public class YoutubeService { .build(this::getVideoInfo); private final AppProperties appProperties; - private final SubscriptionService subscriptionService; + private final SettingService settingService; - public YoutubeService(AppProperties appProperties, SubscriptionService subscriptionService) { + public YoutubeService(AppProperties appProperties, SettingService settingService) { this.appProperties = appProperties; - this.subscriptionService = subscriptionService; + this.settingService = settingService; Config config = new Config.Builder().header("User-Agent", Constants.USER_AGENT).build(); try { @@ -165,6 +167,10 @@ public MovieList list(String text, String sort, String time, int page) { if (text.startsWith("playlist@")) { return getPlaylistVideo(text.substring(9)); } + if (text.startsWith("sub@")) { + downloader.browse(new BrowseRequest(settingService.getString("youtube_cookie"))); + return new MovieList(); + } return search(text, sort, time, page); } diff --git a/src/main/java/com/github/kiulian/downloader/YoutubeDownloader.java b/src/main/java/com/github/kiulian/downloader/YoutubeDownloader.java new file mode 100644 index 0000000000..618c3dc22c --- /dev/null +++ b/src/main/java/com/github/kiulian/downloader/YoutubeDownloader.java @@ -0,0 +1,115 @@ +package com.github.kiulian.downloader; + + +import com.github.kiulian.downloader.cipher.CachedCipherFactory; +import com.github.kiulian.downloader.downloader.Downloader; +import com.github.kiulian.downloader.downloader.DownloaderImpl; +import com.github.kiulian.downloader.downloader.request.RequestChannelUploads; +import com.github.kiulian.downloader.downloader.request.RequestPlaylistInfo; +import com.github.kiulian.downloader.downloader.request.RequestSearchContinuation; +import com.github.kiulian.downloader.downloader.request.RequestSearchResult; +import com.github.kiulian.downloader.downloader.request.RequestSearchable; +import com.github.kiulian.downloader.downloader.request.RequestSubtitlesInfo; +import com.github.kiulian.downloader.downloader.request.RequestVideoFileDownload; +import com.github.kiulian.downloader.downloader.request.RequestVideoInfo; +import com.github.kiulian.downloader.downloader.request.RequestVideoStreamDownload; +import com.github.kiulian.downloader.downloader.request.RequestWebpage; +import com.github.kiulian.downloader.downloader.response.Response; +import com.github.kiulian.downloader.downloader.response.ResponseImpl; +import com.github.kiulian.downloader.extractor.ExtractorImpl; +import com.github.kiulian.downloader.model.BrowseRequest; +import com.github.kiulian.downloader.model.playlist.PlaylistInfo; +import com.github.kiulian.downloader.model.search.SearchResult; +import com.github.kiulian.downloader.model.subtitles.SubtitlesInfo; +import com.github.kiulian.downloader.model.videos.VideoInfo; +import com.github.kiulian.downloader.parser.Parser; +import com.github.kiulian.downloader.parser.ParserImpl; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static com.github.kiulian.downloader.model.Utils.createOutDir; + +public class YoutubeDownloader { + + private final Config config; + private final Downloader downloader; + private final Parser parser; + + public YoutubeDownloader() { + this(Config.buildDefault()); + } + + public YoutubeDownloader(Config config) { + this.config = config; + this.downloader = new DownloaderImpl(config); + this.parser = new ParserImpl(config, downloader, new ExtractorImpl(downloader), new CachedCipherFactory(downloader)); + } + + public YoutubeDownloader(Config config, Downloader downloader) { + this(config, downloader, new ParserImpl(config, downloader, new ExtractorImpl(downloader), new CachedCipherFactory(downloader))); + } + + public YoutubeDownloader(Config config, Downloader downloader, Parser parser) { + this.config = config; + this.parser = parser; + this.downloader = downloader; + } + + public Config getConfig() { + return config; + } + + public Response getVideoInfo(RequestVideoInfo request) { + return parser.parseVideo(request); + } + + public Response> getSubtitlesInfo(RequestSubtitlesInfo request) { + return parser.parseSubtitlesInfo(request); + } + + public Response getChannelUploads(RequestChannelUploads request) { + return parser.parseChannelsUploads(request); + } + + public Response getPlaylistInfo(RequestPlaylistInfo request) { + return parser.parsePlaylist(request); + } + + public Response search(RequestSearchResult request) { + return parser.parseSearchResult(request); + } + + public Response searchContinuation(RequestSearchContinuation request) { + return parser.parseSearchContinuation(request); + } + + public Response search(RequestSearchable request) { + return parser.parseSearcheable(request); + } + + public void browse(BrowseRequest request) { + parser.browse(request); + } + + public Response downloadVideoFile(RequestVideoFileDownload request) { + File outDir = request.getOutputDirectory(); + try { + createOutDir(outDir); + } catch (IOException e) { + return ResponseImpl.error(e); + } + + return downloader.downloadVideoAsFile(request); + } + + public Response downloadVideoStream(RequestVideoStreamDownload request) { + return downloader.downloadVideoAsStream(request); + } + + public Response downloadSubtitle(RequestWebpage request) { + return downloader.downloadWebpage(request); + } + +} diff --git a/src/main/java/com/github/kiulian/downloader/model/BrowseRequest.java b/src/main/java/com/github/kiulian/downloader/model/BrowseRequest.java new file mode 100644 index 0000000000..ee59d4917a --- /dev/null +++ b/src/main/java/com/github/kiulian/downloader/model/BrowseRequest.java @@ -0,0 +1,13 @@ +package com.github.kiulian.downloader.model; + +public class BrowseRequest { + private final String cookie; + + public BrowseRequest(String cookie) { + this.cookie = cookie; + } + + public String getCookie() { + return cookie; + } +} diff --git a/src/main/java/com/github/kiulian/downloader/parser/Parser.java b/src/main/java/com/github/kiulian/downloader/parser/Parser.java new file mode 100644 index 0000000000..f1ac3df128 --- /dev/null +++ b/src/main/java/com/github/kiulian/downloader/parser/Parser.java @@ -0,0 +1,46 @@ +package com.github.kiulian.downloader.parser; + +import com.github.kiulian.downloader.downloader.request.RequestChannelUploads; +import com.github.kiulian.downloader.downloader.request.RequestPlaylistInfo; +import com.github.kiulian.downloader.downloader.request.RequestSearchContinuation; +import com.github.kiulian.downloader.downloader.request.RequestSearchResult; +import com.github.kiulian.downloader.downloader.request.RequestSearchable; +import com.github.kiulian.downloader.downloader.request.RequestSubtitlesInfo; +import com.github.kiulian.downloader.downloader.request.RequestVideoInfo; +import com.github.kiulian.downloader.downloader.response.Response; +import com.github.kiulian.downloader.model.BrowseRequest; +import com.github.kiulian.downloader.model.playlist.PlaylistInfo; +import com.github.kiulian.downloader.model.search.SearchResult; +import com.github.kiulian.downloader.model.subtitles.SubtitlesInfo; +import com.github.kiulian.downloader.model.videos.VideoInfo; + +import java.util.List; + +public interface Parser { + + /* Video */ + + Response parseVideo(RequestVideoInfo request); + + /* Playlist */ + + Response parsePlaylist(RequestPlaylistInfo request); + + /* Channel uploads */ + + Response parseChannelsUploads(RequestChannelUploads request); + + /* Subtitles */ + + Response> parseSubtitlesInfo(RequestSubtitlesInfo request); + + /* Search */ + + Response parseSearchResult(RequestSearchResult request); + + Response parseSearchContinuation(RequestSearchContinuation request); + + Response parseSearcheable(RequestSearchable request); + + void browse(BrowseRequest browseRequest); +} diff --git a/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java b/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java index 258676ff1f..172688581e 100644 --- a/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java +++ b/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java @@ -21,6 +21,7 @@ import com.github.kiulian.downloader.downloader.response.Response; import com.github.kiulian.downloader.downloader.response.ResponseImpl; import com.github.kiulian.downloader.extractor.Extractor; +import com.github.kiulian.downloader.model.BrowseRequest; import com.github.kiulian.downloader.model.playlist.PlaylistDetails; import com.github.kiulian.downloader.model.playlist.PlaylistInfo; import com.github.kiulian.downloader.model.playlist.PlaylistVideoDetails; @@ -46,10 +47,14 @@ import com.github.kiulian.downloader.model.videos.formats.Itag; import com.github.kiulian.downloader.model.videos.formats.VideoFormat; import com.github.kiulian.downloader.model.videos.formats.VideoWithAudioFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -61,6 +66,7 @@ import java.util.concurrent.Future; public class ParserImpl implements Parser { + private static final Logger logger = LoggerFactory.getLogger(ParserImpl.class); private static final String ANDROID_APIKEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; private final Config config; @@ -760,6 +766,131 @@ private SearchResult parseHtmlSearchResult(String url) throws YoutubeException { return parseSearchResult(estimatedCount, rootContents, continuation); } + @Override + public void browse(BrowseRequest browseRequest) { + String url = "https://www.youtube.com/youtubei/v1/browse?key=" + ANDROID_APIKEY + "&prettyPrint=false"; + + JSONObject body = new JSONObject() + .fluentPut("context", new JSONObject() + .fluentPut("client", new JSONObject() + .fluentPut("clientName", "WEB") + .fluentPut("clientVersion", "2.20201021.03.00")) + .fluentPut("user", new JSONObject() + .fluentPut("lockedSafetyMode", false)) + ) + .fluentPut("browseId", "FEsubscriptions"); + + RequestWebpage request = new RequestWebpage(url, "POST", body.toJSONString()) + .header("X-YouTube-Client-Name", "1") + .header("x-youtube-client-version", "2.20201021.03.00") + .header("x-origin", "https://www.youtube.com") + .header("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36") + .header("authorization", getSAPISIDHASH(browseRequest.getCookie())) + .header("cookie", browseRequest.getCookie()) + .header("Content-Type", "application/json"); + + Response response = downloader.downloadWebpage(request); + if (!response.ok()) { + throw new RuntimeException(String.format("Could not load url: %s, exception: %s", url, response.error().getMessage())); + } + String html = response.data(); + logger.info("{}", html); + + JSONObject jsonResponse = JSON.parseObject(html); + JSONObject content = jsonResponse.getJSONObject("contents") + .getJSONObject("twoColumnBrowseResultsRenderer") + .getJSONArray("tabs") + .getJSONObject(0) + .getJSONObject("tabRenderer") + .getJSONObject("content"); + + JSONArray rootContents; + if (content.containsKey("")) { + rootContents = content + .getJSONObject("richGridRenderer") + .getJSONArray("contents"); + } else { + rootContents = content + .getJSONObject("sectionListRenderer") + .getJSONArray("contents"); + } + // contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.richGridRenderer.contents[2].richItemRenderer.content.videoRenderer + + logger.info("size: {}", rootContents.size()); + for (int i = 0; i < rootContents.size(); i++) { + JSONObject item = rootContents.getJSONObject(i); + if (item.containsKey("itemSectionRenderer")) { + JSONObject shelfRenderer = item + .getJSONObject("itemSectionRenderer"). + getJSONArray("contents") + .getJSONObject(0) + .getJSONObject("shelfRenderer"); + + logger.info("author: {}", shelfRenderer.getJSONObject("title").getString("simpleText")); + JSONObject videoRenderer = shelfRenderer.getJSONObject("content") + .getJSONObject("expandedShelfContentsRenderer") + .getJSONArray("items") + .getJSONObject(0) + .getJSONObject("videoRenderer"); + String viewCount = videoRenderer.getJSONObject("viewCountText").getString("simpleText"); + String title = videoRenderer.getJSONObject("title").getJSONArray("runs").getJSONObject(0).getString("text"); + logger.info("video id: {} title: {} viewCount: {}", videoRenderer.getString("videoId"), title, viewCount); + } else { + JSONObject videoRenderer = item.getJSONObject("richItemRenderer") + .getJSONObject("content") + .getJSONObject("videoRenderer"); + String viewCount = videoRenderer.getJSONObject("viewCountText").getString("simpleText"); + String title = videoRenderer.getJSONObject("title").getJSONArray("runs").getJSONObject(0).getString("text"); + logger.info("video id: {} title: {} viewCount: {}", videoRenderer.getString("videoId"), title, viewCount); + // + } + } + } + + private String getSAPISIDHASH(String cookie) { + String time = String.valueOf(System.currentTimeMillis()); + String sid = getSAPISID(cookie); + String text = "SAPISIDHASH " + time + "_" + sha1(time + " " + sid + " https://www.youtube.com"); + logger.info("{} {}", sid, text); + return text; + } + + private String getSAPISID(String cookie) { + Map map = parseCookie(cookie); + if (map.containsKey("__Secure-3PAPISID")) { + return map.get("__Secure-3PAPISID"); + } + return map.get("SAPISID"); + } + + private Map parseCookie(String cookie) { + Map map = new HashMap<>(); + for (String item : cookie.split(";")) { + String[] parts = item.trim().split("="); + String key = parts[0].trim(); + String value = parts[1].trim(); + map.put(key, value); + } + return map; + } + + private String sha1(String text) { + try { + MessageDigest md = MessageDigest.getInstance("SHA-1"); + md.update(text.getBytes(StandardCharsets.UTF_8)); + byte[] digest = md.digest(); + + StringBuilder hexString = new StringBuilder(); + + for (byte b : digest) { + hexString.append(String.format("%02x", b)); + } + return hexString.toString(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private SearchResult parseSearchContinuation(SearchContinuation continuation, YoutubeCallback callback) throws YoutubeException { String url = "https://www.youtube.com/youtubei/v1/search?key=" + ANDROID_APIKEY + "&prettyPrint=false"; diff --git a/web-ui/src/views/AccountsView.vue b/web-ui/src/views/AccountsView.vue index ae0a47d342..d2c5450b88 100644 --- a/web-ui/src/views/AccountsView.vue +++ b/web-ui/src/views/AccountsView.vue @@ -225,6 +225,10 @@
+ +
+ + @@ -236,6 +240,7 @@ import {ElMessage} from "element-plus"; import {store} from "@/services/store"; import router from "@/router"; import PikPakView from '@/views/PikPakView.vue' +import YouTubeView from "@/views/YouTubeView.vue"; const iat = ref([0]) const exp = ref([0]) diff --git a/web-ui/src/views/YouTubeView.vue b/web-ui/src/views/YouTubeView.vue new file mode 100644 index 0000000000..9617a1a959 --- /dev/null +++ b/web-ui/src/views/YouTubeView.vue @@ -0,0 +1,41 @@ + + + + +