refactor: simpify project structure and enhance downloader

.env.test

@@ -1,5 +1,5 @@
 PORT=8888
-LOG_LEVEL=DEBUG
+LOG_LEVEL=INFO
 FASTAPI_RELOAD=false
 
 LOCALSTORE_USE_CWD=true

src/nonebot_plugin_parser/download/__init__.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 from functools import partial
 from contextlib import contextmanager
+from urllib.parse import urljoin
 
 import aiofiles
 from httpx import HTTPError, AsyncClient
@@ -17,7 +18,7 @@
 from ..utils import merge_av, safe_unlink, generate_file_name
 from ..config import pconfig
 from ..constants import COMMON_HEADER, DOWNLOAD_TIMEOUT
-from ..exception import DownloadException, ZeroSizeException, SizeLimitException
+from ..exception import IgnoreException, DownloadException
 
 
 class StreamDownloader:
@@ -41,7 +42,7 @@ def rich_progress(self, desc: str, total: int | None = None):
             yield partial(progress.update, task_id)
 
     @auto_task
-    async def streamd(
+    async def download_file(
         self,
         url: str,
         *,
@@ -67,11 +68,11 @@ async def streamd(
 
                 if content_length == 0:
                     logger.warning(f"媒体 url: {url}, 大小为 0, 取消下载")
-                    raise ZeroSizeException
+                    raise IgnoreException
 
                 if (file_size := content_length / 1024 / 1024) > pconfig.max_size:
-                    logger.warning(f"媒体 url: {url} 大小 {file_size:.2f} MB 超过 {pconfig.max_size} MB, 取消下载")
-                    raise SizeLimitException
+                    logger.warning(f"媒体 url: {url} 大小 {file_size:.2f} MB, 超过 {pconfig.max_size} MB, 取消下载")
+                    raise IgnoreException
 
                 with self.rich_progress(file_name, content_length) as update_progress:
                     async with aiofiles.open(file_path, "wb") as file:
@@ -97,7 +98,7 @@ async def download_video(
         """download video file by url with stream"""
         if video_name is None:
             video_name = generate_file_name(url, ".mp4")
-        return await self.streamd(url, file_name=video_name, ext_headers=ext_headers, chunk_size=1024 * 1024)
+        return await self.download_file(url, file_name=video_name, ext_headers=ext_headers, chunk_size=1024 * 1024)
 
     @auto_task
     async def download_audio(
@@ -110,7 +111,7 @@ async def download_audio(
         """download audio file by url with stream"""
         if audio_name is None:
             audio_name = generate_file_name(url, ".mp3")
-        return await self.streamd(url, file_name=audio_name, ext_headers=ext_headers)
+        return await self.download_file(url, file_name=audio_name, ext_headers=ext_headers)
 
     @auto_task
     async def download_img(
@@ -123,7 +124,7 @@ async def download_img(
         """download image file by url with stream"""
         if img_name is None:
             img_name = generate_file_name(url, ".jpg")
-        return await self.streamd(url, file_name=img_name, ext_headers=ext_headers)
+        return await self.download_file(url, file_name=img_name, ext_headers=ext_headers)
 
     @auto_task
     async def download_av_and_merge(
@@ -155,6 +156,54 @@ async def download_imgs_without_raise(
         )
         return [p for p in paths_or_errs if isinstance(p, Path)]
 
+    @auto_task
+    async def download_m3u8(
+        self,
+        m3u8_url: str,
+        *,
+        video_name: str | None = None,
+        ext_headers: dict[str, str] | None = None,
+    ) -> Path:
+        """download m3u8 file by url with stream"""
+        if video_name is None:
+            video_name = generate_file_name(m3u8_url, ".mp4")
+
+        video_path = pconfig.cache_dir / video_name
+
+        try:
+            async with aiofiles.open(video_path, "wb") as f:
+                total_size = 0
+                with self.rich_progress(desc=video_name) as update_progress:
+                    for url in await self._get_m3u8_slices(m3u8_url):
+                        async with self.client.stream("GET", url, headers=ext_headers) as response:
+                            async for chunk in response.aiter_bytes(chunk_size=1024 * 1024):
+                                await f.write(chunk)
+                                total_size += len(chunk)
+                                update_progress(advance=len(chunk), total=total_size)
+        except HTTPError:
+            await safe_unlink(video_path)
+            logger.exception("m3u8 视频下载失败")
+            raise DownloadException("m3u8 视频下载失败")
+
+        return video_path
+
+    async def _get_m3u8_slices(self, m3u8_url: str):
+        """获取 m3u8 分片"""
+
+        response = await self.client.get(m3u8_url)
+        response.raise_for_status()
+
+        slices_text = response.text
+        slices: list[str] = []
+
+        for line in slices_text.splitlines():
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            slices.append(urljoin(m3u8_url, line))
+
+        return slices
+
 
 DOWNLOADER: StreamDownloader = StreamDownloader()
 

src/nonebot_plugin_parser/download/ytdlp.py

@@ -5,11 +5,12 @@
 
 import yt_dlp
 from msgspec import Struct, convert
+from nonebot import logger
 
 from .task import auto_task
 from ..utils import LimitedSizeDict, generate_file_name
 from ..config import pconfig
-from ..exception import ParseException, DurationLimitException
+from ..exception import ParseException, IgnoreException
 
 
 class VideoInfo(Struct):
@@ -36,8 +37,6 @@ def author_name(self) -> str:
 
 
 class YtdlpDownloader:
-    """YtdlpDownloader class"""
-
     def __init__(self):
         if TYPE_CHECKING:
             from yt_dlp import _Params
@@ -55,15 +54,8 @@ def __init__(self):
             self._extract_base_opts["proxy"] = proxy
 
     async def extract_video_info(self, url: str, cookiefile: Path | None = None) -> VideoInfo:
-        """get video info by url
-
-        Args:
-            url (str): url address
-            cookiefile (Path | None ): cookie file path. Defaults to None.
+        """Get video info by yt-dlp"""
 
-        Returns:
-            dict[str, str]: video info
-        """
         video_info = self._video_info_mapping.get(url, None)
         if video_info:
             return video_info
@@ -83,19 +75,13 @@ async def extract_video_info(self, url: str, cookiefile: Path | None = None) ->
 
     @auto_task
     async def download_video(self, url: str, cookiefile: Path | None = None) -> Path:
-        """download video by yt-dlp
+        """Download video by yt-dlp"""
 
-        Args:
-            url (str): url address
-            cookiefile (Path | None): cookie file path. Defaults to None.
-
-        Returns:
-            Path: video file path
-        """
         video_info = await self.extract_video_info(url, cookiefile)
         duration = video_info.duration
         if duration > pconfig.duration_maximum:
-            raise DurationLimitException
+            logger.warning(f"视频时长 {duration} 秒, 超过 {pconfig.duration_maximum} 秒, 取消下载")
+            raise IgnoreException
 
         video_path = pconfig.cache_dir / generate_file_name(url, ".mp4")
         if video_path.exists():
@@ -127,15 +113,8 @@ async def download_video(self, url: str, cookiefile: Path | None = None) -> Path
 
     @auto_task
     async def download_audio(self, url: str, cookiefile: Path | None = None) -> Path:
-        """download audio by yt-dlp
-
-        Args:
-            url (str): url address
-            cookiefile (Path | None): cookie file path. Defaults to None.
+        """Download audio by yt-dlp"""
 
-        Returns:
-            Path: audio file path
-        """
         file_name = generate_file_name(url)
         audio_path = pconfig.cache_dir / f"{file_name}.flac"
         if audio_path.exists():

src/nonebot_plugin_parser/exception.py

@@ -1,46 +1,22 @@
 class ParseException(Exception):
-    """异常基类"""
-
     def __init__(self, message: str):
         super().__init__(message)
         self.message = message
 
 
-class TipException(ParseException):
-    """提示异常"""
-
-    pass
-
-
 class DownloadException(ParseException):
     """下载异常"""
 
     def __init__(self, message: str | None = None):
         super().__init__(message or "媒体下载失败")
 
 
-class DownloadLimitException(DownloadException):
-    """下载超过限制异常"""
-
-    pass
-
-
-class SizeLimitException(DownloadLimitException):
-    """下载大小超过限制异常"""
-
-    def __init__(self):
-        super().__init__("媒体大小超过配置限制，取消下载")
-
-
-class DurationLimitException(DownloadLimitException):
-    """下载时长超过限制异常"""
-
-    def __init__(self):
-        super().__init__("媒体时长超过配置限制，取消下载")
+class IgnoreException(ParseException):
+    """可忽略异常"""
 
+    def __init__(self, message: str | None = None):
+        super().__init__(message or "可忽略异常")
 
-class ZeroSizeException(DownloadException):
-    """下载大小为 0 异常"""
 
-    def __init__(self):
-        super().__init__("媒体大小为 0, 取消下载")
+class TipException(ParseException):
+    """提示异常"""

src/nonebot_plugin_parser/matchers/__init__.py

@@ -11,7 +11,6 @@
 from ..helper import UniHelper, UniMessage
 from ..parsers import BaseParser, ParseResult, BilibiliParser
 from ..renders import get_renderer
-from ..download import DOWNLOADER
 
 
 def _get_enabled_parser_classes() -> list[type[BaseParser]]:
@@ -104,7 +103,7 @@ async def _(message: Message = CommandArg()):
     if not audio_url:
         await UniMessage("未找到可下载的音频").finish()
 
-    audio_path = await DOWNLOADER.download_audio(
+    audio_path = await parser.downloader.download_audio(
         audio_url, audio_name=f"{bvid}-{page_idx}.mp3", ext_headers=parser.headers
     )
     await UniMessage(UniHelper.record_seg(audio_path)).send()

src/nonebot_plugin_parser/matchers/filter.py

@@ -37,7 +37,6 @@ def get_group_key(session: Session) -> str:
     return f"{session.scope}_{session.scene_path}"
 
 
-# Rule
 def is_enabled(session: Session = UniSession()) -> bool:
     """判断当前会话是否在关闭解析的名单中"""
     if session.scene.is_private:

src/nonebot_plugin_parser/matchers/rule.py

@@ -72,14 +72,7 @@ def _searched(state: T_State) -> SearchResult | None:
 
 
 def _extract_url(hyper: Hyper) -> str | None:
-    """处理 JSON 类型的消息段，提取 URL
-
-    Args:
-        json_seg: JSON 类型的消息段
-
-    Returns:
-        Optional[str]: 提取的 URL, 如果提取失败则返回 None
-    """
+    """处理 JSON 类型的消息段，提取 URL"""
     data = hyper.data
     raw_str: str | None = data.get("raw")