Skip to content

Commit 32f1a2e

Browse files
KaifTaufiqKaifTaufiq
authored
Youtube Extractor Fix with NewPipe (#2489)
1 parent cf084ac commit 32f1a2e

2 files changed

Lines changed: 68 additions & 229 deletions

File tree

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ lifecycleKtx = "2.9.4"
2929
material = "1.14.0-alpha08"
3030
media3 = "1.8.0"
3131
navigationKtx = "2.9.6"
32-
newpipeextractor = "v0.24.8"
32+
newpipeextractor = "v0.25.2"
3333
nextlibMedia3 = "1.8.0-0.9.0"
3434
nicehttp = "0.4.16"
3535
overlappingpanels = "0.1.5"

‎library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/YoutubeExtractor.kt‎

Lines changed: 67 additions & 228 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,13 @@
1-
// Made For cs-kraptor By @trup40, @kraptor123, @ByAyzen
21
package com.lagradost.cloudstream3.extractors
32

4-
import com.fasterxml.jackson.annotation.JsonProperty
53
import com.lagradost.cloudstream3.SubtitleFile
6-
import com.lagradost.cloudstream3.app
4+
import com.lagradost.cloudstream3.newAudioFile
75
import com.lagradost.cloudstream3.newSubtitleFile
8-
import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
96
import com.lagradost.cloudstream3.utils.ExtractorApi
107
import com.lagradost.cloudstream3.utils.ExtractorLink
11-
import com.lagradost.cloudstream3.utils.ExtractorLinkType
12-
import com.lagradost.cloudstream3.utils.HlsPlaylistParser
13-
import com.lagradost.cloudstream3.utils.SubtitleHelper
148
import com.lagradost.cloudstream3.utils.newExtractorLink
15-
import okhttp3.MediaType.Companion.toMediaType
16-
import okhttp3.RequestBody.Companion.toRequestBody
17-
import java.net.URLDecoder
18-
9+
import com.lagradost.cloudstream3.utils.ExtractorLinkType
10+
import org.schabi.newpipe.extractor.stream.StreamInfo
1911

2012
class YoutubeShortLinkExtractor : YoutubeExtractor() {
2113
override val mainUrl = "https://youtu.be"
@@ -30,90 +22,10 @@ class YoutubeNoCookieExtractor : YoutubeExtractor() {
3022
}
3123

3224
open class YoutubeExtractor : ExtractorApi() {
25+
3326
override val mainUrl = "https://www.youtube.com"
34-
override val requiresReferer = false
3527
override val name = "YouTube"
36-
private val youtubeUrl = "https://www.youtube.com"
37-
38-
companion object {
39-
private const val USER_AGENT =
40-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15"
41-
private val HEADERS = mapOf(
42-
"User-Agent" to USER_AGENT,
43-
"Accept-Language" to "en-US,en;q=0.5"
44-
)
45-
}
46-
47-
48-
private fun extractYtCfg(html: String): String? {
49-
val regex = Regex("""ytcfg\.set\(\s*(\{.*?\})\s*\)\s*;""")
50-
val match = regex.find(html)
51-
return match?.groupValues?.getOrNull(1)
52-
}
53-
54-
data class PageConfig(
55-
@JsonProperty("INNERTUBE_API_KEY")
56-
val apiKey: String,
57-
@JsonProperty("INNERTUBE_CLIENT_VERSION")
58-
val clientVersion: String = "2.20240725.01.00",
59-
@JsonProperty("VISITOR_DATA")
60-
val visitorData: String = ""
61-
)
62-
63-
private suspend fun getPageConfig(videoId: String): PageConfig? =
64-
tryParseJson(extractYtCfg(app.get("$mainUrl/watch?v=$videoId", headers = HEADERS).text))
65-
66-
fun extractYouTubeId(url: String): String {
67-
return when {
68-
url.contains("oembed") && url.contains("url=") -> {
69-
val encodedUrl = url.substringAfter("url=").substringBefore("&")
70-
val decodedUrl = URLDecoder.decode(encodedUrl, "UTF-8")
71-
extractYouTubeId(decodedUrl)
72-
}
73-
74-
url.contains("attribution_link") && url.contains("u=") -> {
75-
val encodedUrl = url.substringAfter("u=").substringBefore("&")
76-
val decodedUrl = URLDecoder.decode(encodedUrl, "UTF-8")
77-
extractYouTubeId(decodedUrl)
78-
}
79-
80-
url.contains("watch?v=") -> url.substringAfter("watch?v=").substringBefore("&")
81-
.substringBefore("#")
82-
83-
url.contains("&v=") -> url.substringAfter("&v=").substringBefore("&")
84-
.substringBefore("#")
85-
86-
url.contains("youtu.be/") -> url.substringAfter("youtu.be/").substringBefore("?")
87-
.substringBefore("#").substringBefore("&")
88-
89-
url.contains("/embed/") -> url.substringAfter("/embed/").substringBefore("?")
90-
.substringBefore("#")
91-
92-
url.contains("/v/") -> url.substringAfter("/v/").substringBefore("?")
93-
.substringBefore("#")
94-
95-
url.contains("/e/") -> url.substringAfter("/e/").substringBefore("?")
96-
.substringBefore("#")
97-
98-
url.contains("/shorts/") -> url.substringAfter("/shorts/").substringBefore("?")
99-
.substringBefore("#")
100-
101-
url.contains("/live/") -> url.substringAfter("/live/").substringBefore("?")
102-
.substringBefore("#")
103-
104-
url.contains("/watch/") -> url.substringAfter("/watch/").substringBefore("?")
105-
.substringBefore("#")
106-
107-
url.contains("watch%3Fv%3D") -> url.substringAfter("watch%3Fv%3D")
108-
.substringBefore("%26").substringBefore("#")
109-
110-
url.contains("v%3D") -> url.substringAfter("v%3D").substringBefore("%26")
111-
.substringBefore("#")
112-
113-
else -> error("No Id Found")
114-
}
115-
}
116-
28+
override val requiresReferer = false
11729

11830
override suspend fun getUrl(
11931
url: String,
@@ -122,162 +34,89 @@ open class YoutubeExtractor : ExtractorApi() {
12234
callback: (ExtractorLink) -> Unit
12335
) {
12436
val videoId = extractYouTubeId(url)
125-
val config = getPageConfig(videoId) ?: return
126-
127-
val jsonBody = """
128-
{
129-
"context": {
130-
"client": {
131-
"hl": "en",
132-
"gl": "US",
133-
"clientName": "WEB",
134-
"clientVersion": "${config.clientVersion}",
135-
"visitorData": "${config.visitorData}",
136-
"platform": "DESKTOP",
137-
"userAgent": "$USER_AGENT"
138-
}
139-
},
140-
"videoId": "$videoId",
141-
"playbackContext": {
142-
"contentPlaybackContext": {
143-
"html5Preference": "HTML5_PREF_WANTS"
144-
}
145-
}
146-
}
147-
""".toRequestBody("application/json; charset=utf-8".toMediaType())
37+
val watchUrl = "$mainUrl/watch?v=$videoId"
14838

149-
val response =
150-
app.post(
151-
"$youtubeUrl/youtubei/v1/player?key=${config.apiKey}",
152-
headers = HEADERS,
153-
requestBody = jsonBody
154-
).parsed<Root>()
39+
val info = StreamInfo.getInfo(watchUrl)
15540

156-
val captionTracks = response.captions?.playerCaptionsTracklistRenderer?.captionTracks
41+
val isLive = info.streamType?.name.equals("LIVE_STREAM")
15742

158-
if (captionTracks != null) {
159-
for (caption in captionTracks) {
160-
subtitleCallback.invoke(
161-
newSubtitleFile(
162-
lang =caption.name.simpleText,
163-
url ="${caption.baseUrl}&fmt=ttml" // The default format is not supported
164-
) { headers = HEADERS })
165-
}
43+
if( isLive && info.hlsUrl != null ) {
44+
callback(
45+
newExtractorLink(
46+
source = name,
47+
name = "YouTube Live",
48+
url = info.hlsUrl
49+
) {
50+
type = ExtractorLinkType.M3U8
51+
}
52+
)
53+
} else {
54+
processVideo(info, subtitleCallback, callback)
16655
}
56+
}
16757

168-
val hlsUrl = response.streamingData.hlsManifestUrl
169-
val getHls = app.get(hlsUrl, headers = HEADERS).text
170-
val playlist = HlsPlaylistParser.parse(hlsUrl, getHls) ?: return
171-
172-
var variantIndex = 0
173-
for (tag in playlist.tags) {
174-
val trimmedTag = tag.trim()
175-
if (!trimmedTag.startsWith("#EXT-X-STREAM-INF")) {
176-
continue
177-
}
178-
val variant = playlist.variants.getOrNull(variantIndex++) ?: continue
58+
private suspend fun processVideo(
59+
info: StreamInfo,
60+
subtitleCallback: (SubtitleFile) -> Unit,
61+
callback: (ExtractorLink) -> Unit
62+
): Boolean {
17963

180-
val audioId = trimmedTag.split(",")
181-
.find { it.trim().startsWith("YT-EXT-AUDIO-CONTENT-ID=") }
182-
?.split("=")
183-
?.get(1)
184-
?.trim('"') ?: ""
64+
val videoStreams = info.videoOnlyStreams.orEmpty()
18565

186-
val langString =
187-
SubtitleHelper.fromTagToEnglishLanguageName(
188-
audioId.substringBefore(".")
189-
) ?: SubtitleHelper.fromTagToEnglishLanguageName(
190-
audioId.substringBefore("-")
191-
) ?: audioId
66+
if (videoStreams.isEmpty()) return false
19267

193-
val url = variant.url.toString()
68+
val audioStreams = info.audioStreams.orEmpty()
19469

195-
if (url.isBlank()) {
196-
continue
197-
}
70+
videoStreams.forEach { video ->
19871

199-
callback.invoke(
72+
callback(
20073
newExtractorLink(
201-
source = this.name,
202-
name = "Youtube${if (langString.isNotBlank()) " $langString" else ""}",
203-
url = url,
204-
type = ExtractorLinkType.M3U8
74+
source = name,
75+
name = "YouTube ${normalizeCodec(video.codec)}",
76+
url = video.content
20577
) {
206-
this.referer = "${mainUrl}/"
207-
this.quality = variant.format.height
78+
quality = video.height
79+
audioTracks = audioStreams.map { newAudioFile(it.content) }
20880
}
20981
)
21082
}
211-
}
21283

21384

214-
private data class Root(
215-
// val responseContext: ResponseContext,
216-
// val playabilityStatus: PlayabilityStatus,
217-
@JsonProperty("streamingData")
218-
val streamingData: StreamingData,
219-
// val playbackTracking: PlaybackTracking,
220-
@JsonProperty("captions")
221-
val captions: Captions?,
222-
// val videoDetails: VideoDetails,
223-
// val annotations: List<Annotation>,
224-
// val playerConfig: PlayerConfig,
225-
// val storyboards: Storyboards,
226-
// val microformat: Microformat,
227-
// val cards: Cards,
228-
// val trackingParams: String,
229-
// val endscreen: Endscreen,
230-
// val paidContentOverlay: PaidContentOverlay,
231-
// val adPlacements: List<AdPlacement>,
232-
// val adBreakHeartbeatParams: String,
233-
// val frameworkUpdates: FrameworkUpdates,
234-
)
85+
info.subtitles.forEach { subtitle ->
86+
subtitleCallback(
87+
newSubtitleFile(
88+
lang = subtitle.displayLanguageName
89+
?: subtitle.languageTag
90+
?: "Unknown",
91+
url = subtitle.content
92+
)
93+
)
94+
}
23595

236-
private data class StreamingData(
237-
//val expiresInSeconds: String,
238-
//val formats: List<Format>,
239-
//val adaptiveFormats: List<AdaptiveFormat>,
240-
@JsonProperty("hlsManifestUrl")
241-
val hlsManifestUrl: String,
242-
//val serverAbrStreamingUrl: String,
243-
)
96+
return true
97+
}
24498

245-
private data class Captions(
246-
@JsonProperty("playerCaptionsTracklistRenderer")
247-
val playerCaptionsTracklistRenderer: PlayerCaptionsTracklistRenderer?,
248-
)
99+
// ---------------- HELPERS ----------------
249100

250-
private data class PlayerCaptionsTracklistRenderer(
251-
@JsonProperty("captionTracks")
252-
val captionTracks: List<CaptionTrack>?,
253-
//val audioTracks: List<AudioTrack>,
254-
//val translationLanguages: List<TranslationLanguage>,
255-
//@JsonProperty("defaultAudioTrackIndex")
256-
//val defaultAudioTrackIndex: Long,
257-
)
101+
private fun extractYouTubeId(url: String): String {
102+
val regex = Regex(
103+
"(?:youtu\\.be/|youtube(?:-nocookie)?\\.com/(?:.*v=|v/|u/\\w/|embed/|shorts/|live/))([\\w-]{11})"
104+
)
105+
return regex.find(url)?.groupValues?.get(1)
106+
?: throw IllegalArgumentException("Invalid YouTube URL: $url")
107+
}
258108

259-
private data class CaptionTrack(
260-
@JsonProperty("baseUrl")
261-
val baseUrl: String,
262-
@JsonProperty("name")
263-
val name: Name,
264-
//val vssId: String,
265-
//val languageCode: String,
266-
//val kind: String?,
267-
//val isTranslatable: Boolean,
268-
//val trackName: String,
269-
)
109+
private fun normalizeCodec(codec: String?): String {
110+
if (codec.isNullOrBlank()) return ""
270111

271-
private data class Name(
272-
@JsonProperty("simpleText")
273-
val simpleText: String,
274-
)
112+
val c = codec.lowercase()
275113

276-
// data class AudioTrack(
277-
// val captionTrackIndices: List<Long>,
278-
// val defaultCaptionTrackIndex: Long,
279-
// val hasDefaultTrack: Boolean,
280-
// val audioTrackId: String,
281-
// val captionsInitialState: String,
282-
// )
114+
return when {
115+
c.startsWith("av01") -> "AV1"
116+
c.startsWith("vp9") -> "VP9"
117+
c.startsWith("avc1") || c.startsWith("h264") -> "H264"
118+
c.startsWith("hev1") || c.startsWith("hvc1") || c.startsWith("hevc") -> "H265"
119+
else -> codec.substringBefore('.').uppercase()
120+
}
121+
}
283122
}

0 commit comments

Comments
 (0)