Skip to content

Commit f794f3a

Browse files
KaifTaufiqKaifTaufiq
committed
Youtube Extractor Updated with NewPipe
1 parent 137a7ff commit f794f3a

1 file changed

Lines changed: 73 additions & 234 deletions

File tree

‎library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/YoutubeExtractor.kt‎

Lines changed: 73 additions & 234 deletions
Original file line numberDiff line numberDiff line change
@@ -1,119 +1,39 @@
1-
// Made For cs-kraptor By @trup40, @kraptor123, @ByAyzen
21
package com.lagradost.cloudstream3.extractors
32

4-
import com.fasterxml.jackson.annotation.JsonProperty
53
import com.lagradost.cloudstream3.SubtitleFile
6-
import com.lagradost.cloudstream3.app
4+
import com.lagradost.cloudstream3.newAudioFile
75
import com.lagradost.cloudstream3.newSubtitleFile
8-
import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
96
import com.lagradost.cloudstream3.utils.ExtractorApi
107
import com.lagradost.cloudstream3.utils.ExtractorLink
11-
import com.lagradost.cloudstream3.utils.ExtractorLinkType
12-
import com.lagradost.cloudstream3.utils.HlsPlaylistParser
13-
import com.lagradost.cloudstream3.utils.SubtitleHelper
148
import com.lagradost.cloudstream3.utils.newExtractorLink
15-
import okhttp3.MediaType.Companion.toMediaType
16-
import okhttp3.RequestBody.Companion.toRequestBody
17-
import java.net.URLDecoder
9+
import org.schabi.newpipe.extractor.services.youtube.YoutubeService
10+
import org.schabi.newpipe.extractor.stream.StreamInfo
1811

19-
20-
class YoutubeShortLinkExtractor : YoutubeExtractor() {
12+
class YoutubeShortLinkExtractor(
13+
maxResolution: Int? = null
14+
) : YoutubeExtractor(maxResolution) {
2115
override val mainUrl = "https://youtu.be"
2216
}
2317

24-
class YoutubeMobileExtractor : YoutubeExtractor() {
18+
class YoutubeMobileExtractor(
19+
maxResolution: Int? = null
20+
) : YoutubeExtractor(maxResolution) {
2521
override val mainUrl = "https://m.youtube.com"
2622
}
2723

28-
class YoutubeNoCookieExtractor : YoutubeExtractor() {
24+
class YoutubeNoCookieExtractor(
25+
maxResolution: Int? = null
26+
) : YoutubeExtractor(maxResolution) {
2927
override val mainUrl = "https://www.youtube-nocookie.com"
3028
}
3129

32-
open class YoutubeExtractor : ExtractorApi() {
30+
open class YoutubeExtractor(
31+
private val maxResolution: Int? = null
32+
) : ExtractorApi() {
33+
3334
override val mainUrl = "https://www.youtube.com"
34-
override val requiresReferer = false
3535
override val name = "YouTube"
36-
private val youtubeUrl = "https://www.youtube.com"
37-
38-
companion object {
39-
private const val USER_AGENT =
40-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15"
41-
private val HEADERS = mapOf(
42-
"User-Agent" to USER_AGENT,
43-
"Accept-Language" to "en-US,en;q=0.5"
44-
)
45-
}
46-
47-
48-
private fun extractYtCfg(html: String): String? {
49-
val regex = Regex("""ytcfg\.set\(\s*(\{.*?\})\s*\)\s*;""")
50-
val match = regex.find(html)
51-
return match?.groupValues?.getOrNull(1)
52-
}
53-
54-
data class PageConfig(
55-
@JsonProperty("INNERTUBE_API_KEY")
56-
val apiKey: String,
57-
@JsonProperty("INNERTUBE_CLIENT_VERSION")
58-
val clientVersion: String = "2.20240725.01.00",
59-
@JsonProperty("VISITOR_DATA")
60-
val visitorData: String = ""
61-
)
62-
63-
private suspend fun getPageConfig(videoId: String): PageConfig? =
64-
tryParseJson(extractYtCfg(app.get("$mainUrl/watch?v=$videoId", headers = HEADERS).text))
65-
66-
fun extractYouTubeId(url: String): String {
67-
return when {
68-
url.contains("oembed") && url.contains("url=") -> {
69-
val encodedUrl = url.substringAfter("url=").substringBefore("&")
70-
val decodedUrl = URLDecoder.decode(encodedUrl, "UTF-8")
71-
extractYouTubeId(decodedUrl)
72-
}
73-
74-
url.contains("attribution_link") && url.contains("u=") -> {
75-
val encodedUrl = url.substringAfter("u=").substringBefore("&")
76-
val decodedUrl = URLDecoder.decode(encodedUrl, "UTF-8")
77-
extractYouTubeId(decodedUrl)
78-
}
79-
80-
url.contains("watch?v=") -> url.substringAfter("watch?v=").substringBefore("&")
81-
.substringBefore("#")
82-
83-
url.contains("&v=") -> url.substringAfter("&v=").substringBefore("&")
84-
.substringBefore("#")
85-
86-
url.contains("youtu.be/") -> url.substringAfter("youtu.be/").substringBefore("?")
87-
.substringBefore("#").substringBefore("&")
88-
89-
url.contains("/embed/") -> url.substringAfter("/embed/").substringBefore("?")
90-
.substringBefore("#")
91-
92-
url.contains("/v/") -> url.substringAfter("/v/").substringBefore("?")
93-
.substringBefore("#")
94-
95-
url.contains("/e/") -> url.substringAfter("/e/").substringBefore("?")
96-
.substringBefore("#")
97-
98-
url.contains("/shorts/") -> url.substringAfter("/shorts/").substringBefore("?")
99-
.substringBefore("#")
100-
101-
url.contains("/live/") -> url.substringAfter("/live/").substringBefore("?")
102-
.substringBefore("#")
103-
104-
url.contains("/watch/") -> url.substringAfter("/watch/").substringBefore("?")
105-
.substringBefore("#")
106-
107-
url.contains("watch%3Fv%3D") -> url.substringAfter("watch%3Fv%3D")
108-
.substringBefore("%26").substringBefore("#")
109-
110-
url.contains("v%3D") -> url.substringAfter("v%3D").substringBefore("%26")
111-
.substringBefore("#")
112-
113-
else -> error("No Id Found")
114-
}
115-
}
116-
36+
override val requiresReferer = false
11737

11838
override suspend fun getUrl(
11939
url: String,
@@ -122,162 +42,81 @@ open class YoutubeExtractor : ExtractorApi() {
12242
callback: (ExtractorLink) -> Unit
12343
) {
12444
val videoId = extractYouTubeId(url)
125-
val config = getPageConfig(videoId) ?: return
45+
val watchUrl = "$mainUrl/watch?v=$videoId"
12646

127-
val jsonBody = """
128-
{
129-
"context": {
130-
"client": {
131-
"hl": "en",
132-
"gl": "US",
133-
"clientName": "WEB",
134-
"clientVersion": "${config.clientVersion}",
135-
"visitorData": "${config.visitorData}",
136-
"platform": "DESKTOP",
137-
"userAgent": "$USER_AGENT"
138-
}
139-
},
140-
"videoId": "$videoId",
141-
"playbackContext": {
142-
"contentPlaybackContext": {
143-
"html5Preference": "HTML5_PREF_WANTS"
144-
}
145-
}
146-
}
147-
""".toRequestBody("application/json; charset=utf-8".toMediaType())
148-
149-
val response =
150-
app.post(
151-
"$youtubeUrl/youtubei/v1/player?key=${config.apiKey}",
152-
headers = HEADERS,
153-
requestBody = jsonBody
154-
).parsed<Root>()
155-
156-
val captionTracks = response.captions?.playerCaptionsTracklistRenderer?.captionTracks
47+
val streamInfo = StreamInfo.getInfo(YoutubeService(0), watchUrl)
15748

158-
if (captionTracks != null) {
159-
for (caption in captionTracks) {
160-
subtitleCallback.invoke(
161-
newSubtitleFile(
162-
lang =caption.name.simpleText,
163-
url ="${caption.baseUrl}&fmt=ttml" // The default format is not supported
164-
) { headers = HEADERS })
165-
}
166-
}
167-
168-
val hlsUrl = response.streamingData.hlsManifestUrl
169-
val getHls = app.get(hlsUrl, headers = HEADERS).text
170-
val playlist = HlsPlaylistParser.parse(hlsUrl, getHls) ?: return
49+
processStreams(streamInfo, subtitleCallback, callback)
50+
}
17151

172-
var variantIndex = 0
173-
for (tag in playlist.tags) {
174-
val trimmedTag = tag.trim()
175-
if (!trimmedTag.startsWith("#EXT-X-STREAM-INF")) {
176-
continue
177-
}
178-
val variant = playlist.variants.getOrNull(variantIndex++) ?: continue
52+
private suspend fun processStreams(
53+
info: StreamInfo,
54+
subtitleCallback: (SubtitleFile) -> Unit,
55+
callback: (ExtractorLink) -> Unit
56+
): Boolean {
17957

180-
val audioId = trimmedTag.split(",")
181-
.find { it.trim().startsWith("YT-EXT-AUDIO-CONTENT-ID=") }
182-
?.split("=")
183-
?.get(1)
184-
?.trim('"') ?: ""
58+
val videoStreams = info.videoOnlyStreams
59+
?.filterByResolution(maxResolution)
60+
?: emptyList()
18561

186-
val langString =
187-
SubtitleHelper.fromTagToEnglishLanguageName(
188-
audioId.substringBefore(".")
189-
) ?: SubtitleHelper.fromTagToEnglishLanguageName(
190-
audioId.substringBefore("-")
191-
) ?: audioId
62+
if (videoStreams.isEmpty()) return false
19263

193-
val url = variant.url.toString()
64+
val audioStreams = info.audioStreams.orEmpty()
19465

195-
if (url.isBlank()) {
196-
continue
197-
}
66+
videoStreams.forEach { video ->
19867

199-
callback.invoke(
68+
callback(
20069
newExtractorLink(
201-
source = this.name,
202-
name = "Youtube${if (langString.isNotBlank()) " $langString" else ""}",
203-
url = url,
204-
type = ExtractorLinkType.M3U8
70+
source = name,
71+
name = "YouTube ${normalizeCodec(video.codec)}",
72+
url = video.content
20573
) {
206-
this.referer = "${mainUrl}/"
207-
this.quality = variant.format.height
74+
quality = video.height
75+
audioTracks = audioStreams.map { newAudioFile(it.content) }
20876
}
20977
)
21078
}
211-
}
21279

21380

214-
private data class Root(
215-
// val responseContext: ResponseContext,
216-
// val playabilityStatus: PlayabilityStatus,
217-
@JsonProperty("streamingData")
218-
val streamingData: StreamingData,
219-
// val playbackTracking: PlaybackTracking,
220-
@JsonProperty("captions")
221-
val captions: Captions?,
222-
// val videoDetails: VideoDetails,
223-
// val annotations: List<Annotation>,
224-
// val playerConfig: PlayerConfig,
225-
// val storyboards: Storyboards,
226-
// val microformat: Microformat,
227-
// val cards: Cards,
228-
// val trackingParams: String,
229-
// val endscreen: Endscreen,
230-
// val paidContentOverlay: PaidContentOverlay,
231-
// val adPlacements: List<AdPlacement>,
232-
// val adBreakHeartbeatParams: String,
233-
// val frameworkUpdates: FrameworkUpdates,
234-
)
81+
info.subtitles.forEach { subtitle ->
82+
subtitleCallback(
83+
newSubtitleFile(
84+
lang = subtitle.displayLanguageName
85+
?: subtitle.languageTag
86+
?: "Unknown",
87+
url = subtitle.content
88+
)
89+
)
90+
}
91+
92+
return true
93+
}
23594

236-
private data class StreamingData(
237-
//val expiresInSeconds: String,
238-
//val formats: List<Format>,
239-
//val adaptiveFormats: List<AdaptiveFormat>,
240-
@JsonProperty("hlsManifestUrl")
241-
val hlsManifestUrl: String,
242-
//val serverAbrStreamingUrl: String,
243-
)
95+
// ---------------- HELPERS ----------------
24496

245-
private data class Captions(
246-
@JsonProperty("playerCaptionsTracklistRenderer")
247-
val playerCaptionsTracklistRenderer: PlayerCaptionsTracklistRenderer?,
248-
)
97+
private fun extractYouTubeId(url: String): String {
98+
val regex = Regex(
99+
"(?:youtu\\.be/|youtube(?:-nocookie)?\\.com/(?:.*v=|v/|u/\\w/|embed/|shorts/|live/))([\\w-]{11})"
100+
)
101+
return regex.find(url)?.groupValues?.get(1)
102+
?: throw IllegalArgumentException("Invalid YouTube URL: $url")
103+
}
249104

250-
private data class PlayerCaptionsTracklistRenderer(
251-
@JsonProperty("captionTracks")
252-
val captionTracks: List<CaptionTrack>?,
253-
//val audioTracks: List<AudioTrack>,
254-
//val translationLanguages: List<TranslationLanguage>,
255-
//@JsonProperty("defaultAudioTrackIndex")
256-
//val defaultAudioTrackIndex: Long,
257-
)
105+
private fun List<org.schabi.newpipe.extractor.stream.VideoStream>.filterByResolution(
106+
max: Int?
107+
) = if (max == null) this else filter { it.height <= max }
258108

259-
private data class CaptionTrack(
260-
@JsonProperty("baseUrl")
261-
val baseUrl: String,
262-
@JsonProperty("name")
263-
val name: Name,
264-
//val vssId: String,
265-
//val languageCode: String,
266-
//val kind: String?,
267-
//val isTranslatable: Boolean,
268-
//val trackName: String,
269-
)
109+
private fun normalizeCodec(codec: String?): String {
110+
if (codec.isNullOrBlank()) return ""
270111

271-
private data class Name(
272-
@JsonProperty("simpleText")
273-
val simpleText: String,
274-
)
112+
val c = codec.lowercase()
275113

276-
// data class AudioTrack(
277-
// val captionTrackIndices: List<Long>,
278-
// val defaultCaptionTrackIndex: Long,
279-
// val hasDefaultTrack: Boolean,
280-
// val audioTrackId: String,
281-
// val captionsInitialState: String,
282-
// )
114+
return when {
115+
c.startsWith("av01") -> "AV1"
116+
c.startsWith("vp9") -> "VP9"
117+
c.startsWith("avc1") || c.startsWith("h264") -> "H264"
118+
c.startsWith("hev1") || c.startsWith("hvc1") || c.startsWith("hevc") -> "H265"
119+
else -> codec.substringBefore('.').uppercase()
120+
}
121+
}
283122
}

0 commit comments

Comments
 (0)