From 3cea08c3e35dbfeceed9f05bd533e1e42dd3bfab Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 29 May 2026 10:34:51 -0500 Subject: [PATCH] feat: added path templating and slskd --- .gitignore | 3 +- Dockerfile | 6 +- go.mod | 7 +-- go.sum | 2 + sample.env | 2 + src/config/config.go | 2 + src/downloader/downloader.go | 104 +++++++++++++++++++++++++++++------ src/downloader/slskd.go | 2 +- src/downloader/youtube.go | 54 ++++++++++++++---- 9 files changed, 145 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 49cffa89..28a16e55 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,6 @@ explo src/web/dist/ src/web/frontend/node_modules/ /cache -data/ +/data/ +/.data/ .zed/ diff --git a/Dockerfile b/Dockerfile index f170c940..f65e5c07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN npm ci COPY src/web/frontend/ ./ RUN VITE_VERSION=${VERSION} npm run build -FROM --platform=$BUILDPLATFORM golang:1.24-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder # Set the working directory WORKDIR /app @@ -31,7 +31,7 @@ RUN apk add --no-cache \ yt-dlp \ tzdata \ shadow \ - su-exec + su-exec # Install ytmusicapi in the container RUN pip install --no-cache-dir ytmusicapi @@ -52,4 +52,4 @@ ENV WEB_ADDR=":7288" EXPOSE 7288 -CMD ["/start.sh"] \ No newline at end of file +CMD ["/start.sh"] diff --git a/go.mod b/go.mod index 5b3bc9e0..b045a151 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module explo -go 1.24.0 - -toolchain go1.24.3 +go 1.25.0 require ( github.com/go-co-op/gocron/v2 v2.21.1 @@ -12,6 +10,7 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/wader/goutubedl v0.0.0-20250417150709-083444e4ab87 golang.org/x/crypto v0.46.0 + golang.org/x/net v0.48.0 golang.org/x/sync v0.19.0 golang.org/x/text v0.32.0 maunium.net/go/mautrix v0.26.0 @@ -43,8 +42,8 @@ require ( github.com/u2takey/go-utils v0.3.1 // indirect go.mau.fi/util v0.9.3 // indirect golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect - golang.org/x/net v0.48.0 // indirect golang.org/x/sys v0.39.0 // indirect + golang.org/x/time v0.15.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) diff --git a/go.sum b/go.sum index 363c6d62..7a155d3e 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sample.env b/sample.env index 40cf2dae..e478bc42 100644 --- a/sample.env +++ b/sample.env @@ -43,6 +43,8 @@ LIBRARY_NAME= # KEEP_PERMISSIONS=true # Comma-separated list (no spaces) of download services, in priority order (default: youtube) # DOWNLOAD_SERVICES=youtube +# Path templating, Options are Artist, Album, TrackName, TrackNumber, File, Ext (eg. "{{Artist}}/{{Album}}/{{File}}") +# PATH_TEMPLATING="" # Directory for writing .m3u playlists (required only for MPD) # PLAYLIST_DIR=/path/to/playlist/folder/ diff --git a/src/config/config.go b/src/config/config.go index c60cb433..9a2100f6 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -90,6 +90,7 @@ type SubsonicConfig struct { type DownloadConfig struct { DownloadDir string `env:"DOWNLOAD_DIR" env-default:"/data/"` + PathTemplate string `env:"PATH_TEMPLATE"` Youtube Youtube YoutubeMusic YoutubeMusic Slskd Slskd @@ -117,6 +118,7 @@ type Youtube struct { CookiesPath string `env:"COOKIES_PATH" env-default:"./cookies.txt"` Filters Filters CoversDir string + PathTemplate string `env:"PATH_TEMPLATE"` } type YoutubeMusic struct { diff --git a/src/downloader/downloader.go b/src/downloader/downloader.go index 0235c165..8dc4d65b 100644 --- a/src/downloader/downloader.go +++ b/src/downloader/downloader.go @@ -1,19 +1,23 @@ package downloader import ( + "context" "fmt" "io" "log/slog" "os" "path" "path/filepath" + "strconv" "strings" - - "golang.org/x/sync/errgroup" + "time" cfg "explo/src/config" "explo/src/models" "explo/src/util" + + "golang.org/x/sync/errgroup" + "golang.org/x/time/rate" ) type DownloadClient struct { @@ -52,6 +56,7 @@ func (c *DownloadClient) StartDownload(tracks *[]*models.Track) { if c.Cfg.ExcludeLocal { // remove locally found tracks, so they can't be added to playlist filterLocalTracks(tracks, true) } + if c.needsDownloadDir() { if err := os.MkdirAll(c.Cfg.DownloadDir, 0755); err != nil { slog.Error(err.Error()) @@ -61,40 +66,56 @@ func (c *DownloadClient) StartDownload(tracks *[]*models.Track) { for _, d := range c.Downloaders { var g errgroup.Group - g.SetLimit(1) + g.SetLimit(3) + + limiter := rate.NewLimiter(rate.Every(time.Second), 1) for _, track := range *tracks { if track.Present { continue } + track := track + g.Go(func() error { + ctx := context.Background() + + if err := limiter.Wait(ctx); err != nil { + return err + } if err := d.QueryTrack(track); err != nil { slog.Warn(err.Error()) return nil } + + if err := limiter.Wait(ctx); err != nil { + return err + } + if err := d.GetTrack(track); err != nil { slog.Warn(err.Error()) return nil } + return nil }) } + if err := g.Wait(); err != nil { + slog.Warn(err.Error()) return } if m, ok := d.(Monitor); ok { - err := c.MonitorDownloads(*tracks, m) - if err != nil { + if err := c.MonitorDownloads(*tracks, m); err != nil { slog.Warn(err.Error()) } } } + filterLocalTracks(tracks, false) } - func (c *DownloadClient) needsDownloadDir() bool { for _, svc := range c.Cfg.Services { if svc == "youtube" || svc == "youtube-music" { @@ -181,7 +202,48 @@ func containsLower(str string, substr string) bool { ) } -// Move download from the source dir to the dest dir (download dir) +func sanitize(s string) string { + replacer := strings.NewReplacer( + "/", "-", + "\\", "-", + ":", "-", + "*", "", + "?", "", + "\"", "", + "<", "", + ">", "", + "|", "", + ) + + return strings.TrimSpace(replacer.Replace(s)) +} + +func buildTrackPath(template string, track *models.Track) string { + replacements := map[string]string{ + "Artist": sanitize(track.MainArtist), + "Album": sanitize(track.Album), + "AlbumName": sanitize(track.Album), + "TrackName": sanitize(track.CleanTitle), + "TrackNumber": fmt.Sprintf("%02d", track.TrackNumber), + "DiscNumber": fmt.Sprintf("%02d", track.DiscNumber), + "Year": strconv.Itoa(track.OriginalYear), + "File": sanitize(track.File), + "ext": strings.TrimPrefix(filepath.Ext(track.File), "."), + } + + result := template + + for key, value := range replacements { + result = strings.ReplaceAll( + result, + "{{"+key+"}}", + value, + ) + } + + return filepath.Clean(result) +} + func (c *DownloadClient) MoveDownload(srcDir, destDir, trackPath string, track *models.Track) error { trackDir := filepath.Join(srcDir, trackPath) srcFile := filepath.Join(trackDir, track.File) @@ -197,23 +259,34 @@ func (c *DownloadClient) MoveDownload(srcDir, destDir, trackPath string, track * defer func() { if cerr := in.Close(); cerr != nil { - slog.Error(fmt.Sprintf("failed to close source file: %s", err.Error())) + slog.Error(fmt.Sprintf("failed to close source file: %s", cerr.Error())) } }() - if err = os.MkdirAll(destDir, os.ModePerm); err != nil { - return fmt.Errorf("couldn't make download directory: %s", err.Error()) + var dstFile string + + if c.Cfg.PathTemplate != "" { + relativePath := buildTrackPath(c.Cfg.PathTemplate, track) + dstFile = filepath.Join(destDir, relativePath) + } else { + if err = os.MkdirAll(destDir, os.ModePerm); err != nil { + return fmt.Errorf("couldn't make download directory: %s", err.Error()) + } + + dstFile = filepath.Join(destDir, track.File) + } + if err = os.MkdirAll(filepath.Dir(dstFile), os.ModePerm); err != nil { + return fmt.Errorf("couldn't make destination directory: %s", err.Error()) } - dstFile := filepath.Join(destDir, track.File) out, err := os.Create(dstFile) if err != nil { return fmt.Errorf("couldn't create destination file: %s", err.Error()) } defer func() { - if err = out.Close(); err != nil { - slog.Error(fmt.Sprintf("failed to close destination file: %s", err.Error())) + if cerr := out.Close(); cerr != nil { + slog.Error(fmt.Sprintf("failed to close destination file: %s", cerr.Error())) } }() @@ -225,7 +298,6 @@ func (c *DownloadClient) MoveDownload(srcDir, destDir, trackPath string, track * return fmt.Errorf("sync failed: %s", err.Error()) } - // Keep permissions, unless specified otherwise in .env (some systems don't support chmod) if c.Cfg.KeepPermissions { info, err := os.Stat(srcFile) if err != nil { @@ -236,12 +308,10 @@ func (c *DownloadClient) MoveDownload(srcDir, destDir, trackPath string, track * } } - // Remove only the moved file, not the directory if err = os.Remove(srcFile); err != nil { return fmt.Errorf("failed to delete original file: %s", err.Error()) } - // to avoid removing additional downloads check if directory is empty before removing isEmpty, err := isDirEmpty(trackDir) if err != nil { return fmt.Errorf("couldn't check if directory is empty: %s", err.Error()) @@ -250,9 +320,11 @@ func (c *DownloadClient) MoveDownload(srcDir, destDir, trackPath string, track * return fmt.Errorf("failed to remove empty directory: %s", err.Error()) } } + return nil } + func isDirEmpty(path string) (bool, error) { f, err := os.Open(path) if err != nil { diff --git a/src/downloader/slskd.go b/src/downloader/slskd.go index f764017d..f1785632 100644 --- a/src/downloader/slskd.go +++ b/src/downloader/slskd.go @@ -475,4 +475,4 @@ func normalize(state string) string{ } } return state -} \ No newline at end of file +} diff --git a/src/downloader/youtube.go b/src/downloader/youtube.go index 15de358e..1942971f 100644 --- a/src/downloader/youtube.go +++ b/src/downloader/youtube.go @@ -49,6 +49,7 @@ type Youtube struct { HttpClient *util.HttpClient Cfg cfg.Youtube gouTubeOpts goutubedl.Options + Sleep int } func NewYoutube(cfg cfg.Youtube, discovery, downloadDir string, httpClient *util.HttpClient) *Youtube { // init downloader cfg for youtube @@ -205,22 +206,51 @@ func saveVideo(c Youtube, track models.Track, stream *goutubedl.DownloadResult) metadata := util.BuildffmpegMetadata(track) + outputPath := filepath.Join(c.DownloadDir, track.File) + + if c.Cfg.PathTemplate != "" { + outputPath = filepath.Join( + c.DownloadDir, + buildTrackPath(c.Cfg.PathTemplate, &track), + ) + } + + if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil { + slog.Error("failed to create output directory", "context", err.Error()) + return false + } + var cmd *ffmpeg.Stream if c.Cfg.EmbedCoverArt { - coversDir := c.Cfg.CoversDir - util.DownloadCover(track.CoverURL, coversDir) - coverPath := filepath.Join(coversDir, track.MusicBrainzAlbumID+".jpg") - cmd = ffmpeg.Output([]*ffmpeg.Stream{ffmpeg.Input(input), ffmpeg.Input(coverPath)}, filepath.Join(c.DownloadDir, track.File), ffmpeg.KwArgs{ - "metadata": metadata, - "loglevel": "error", - }).OverWriteOutput().ErrorToStdOut() + coversDir := c.Cfg.CoversDir + util.DownloadCover(track.CoverURL, coversDir) + + coverPath := filepath.Join( + coversDir, + track.MusicBrainzAlbumID+".jpg", + ) + + cmd = ffmpeg.Output( + []*ffmpeg.Stream{ + ffmpeg.Input(input), + ffmpeg.Input(coverPath), + }, + outputPath, + ffmpeg.KwArgs{ + "metadata": metadata, + "loglevel": "error", + }, + ).OverWriteOutput().ErrorToStdOut() } else { - cmd = ffmpeg.Input(input).Output(filepath.Join(c.DownloadDir, track.File), ffmpeg.KwArgs{ - "map": "0:a", - "metadata": metadata, - "loglevel": "error", - }).OverWriteOutput().ErrorToStdOut() + cmd = ffmpeg.Input(input).Output( + outputPath, + ffmpeg.KwArgs{ + "map": "0:a", + "metadata": metadata, + "loglevel": "error", + }, + ).OverWriteOutput().ErrorToStdOut() } if c.Cfg.FfmpegPath != "" {