GitHub/tiddl
1
0
Fork 0
mirror of https://github.com/oskvr37/tiddl.git synced 2026-06-13 04:05:08 +03:00
Code Issues Packages Projects Releases Wiki Activity

✨ Threaded Track Download & Videos Support (#85)

Browse Source
This commit is contained in:
Oskar Dudziński
2025-02-09 00:04:07 +01:00
committed by GitHub
parent 0604c9fd71
commit 3a14939f15
12 changed files with 269 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -48,11 +48,11 @@ tiddl auth login
## Download resource
You can download track / album / artist / playlist / (video coming soon)
<!-- TODO: remove coming soon after adding video download -->
You can download track / video / album / artist / playlist
```bash
tiddl url https://listen.tidal.com/track/103805726 download
tiddl url https://listen.tidal.com/video/25747442 download
tiddl url https://listen.tidal.com/album/103805723 download
tiddl url https://listen.tidal.com/artist/25022 download
tiddl url https://listen.tidal.com/playlist/84974059-76af-406a-aede-ece2b78fa372 download
examples/concurrent_download_rich.py
-2
View File
@@ -52,8 +52,6 @@ progress = Progress(
def handleItemDownload(item: Union[Track, Video]):
# TODO: check if item is already downloaded
if isinstance(item, Track):
track_stream = api.getTrackStream(item.id, quality=QUALITY)
urls, extension = parseTrackStream(track_stream)
tiddl/api.py
+10 -4
View File
@@ -4,7 +4,12 @@ from pathlib import Path
from typing import Any, Literal, Type, TypeVar
from pydantic import BaseModel
from requests_cache import CachedSession, EXPIRE_IMMEDIATELY, NEVER_EXPIRE
from requests_cache import (
CachedSession,
EXPIRE_IMMEDIATELY,
NEVER_EXPIRE,
DO_NOT_CACHE,
)
from tiddl.models.api import (
Album,
@@ -28,6 +33,7 @@ from tiddl.exceptions import ApiError
from tiddl.config import HOME_PATH
DEBUG = False
T = TypeVar("T", bound=BaseModel)
logger = logging.getLogger(__name__)
@@ -209,7 +215,7 @@ class TidalApi:
def getSession(self):
return self.fetch(
SessionResponse, "sessions", expire_after=EXPIRE_IMMEDIATELY
SessionResponse, "sessions", expire_after=DO_NOT_CACHE
)
def getTrack(self, track_id: str | int):
@@ -226,7 +232,7 @@ class TidalApi:
"playbackmode": "STREAM",
"assetpresentation": "FULL",
},
expire_after=3600,
expire_after=DO_NOT_CACHE,
)
def getVideo(self, video_id: str | int):
@@ -243,5 +249,5 @@ class TidalApi:
"playbackmode": "STREAM",
"assetpresentation": "FULL",
},
expire_after=3600,
expire_after=DO_NOT_CACHE,
)
tiddl/cli/__init__.py
-2
View File
@@ -7,7 +7,6 @@ from .ctx import ContextObj, passContext, Context
from .auth import AuthGroup
from .download import UrlGroup, FavGroup, SearchGroup, FileGroup
from .config import ConfigCommand
from .download_concurrent import DownloadCommand
from tiddl.config import HOME_PATH
@@ -59,7 +58,6 @@ cli.add_command(UrlGroup)
cli.add_command(FavGroup)
cli.add_command(SearchGroup)
cli.add_command(FileGroup)
cli.add_command(DownloadCommand)
if __name__ == "__main__":
cli()
tiddl/cli/download/__init__.py
+217 -102
View File
@@ -1,6 +1,31 @@
import logging
import click
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from requests import Session
from rich.progress import (
BarColumn,
Progress,
TextColumn,
)
from tiddl.download import parseTrackStream, parseVideoStream
from tiddl.exceptions import ApiError, AuthError
from tiddl.metadata import Cover, addMetadata, addVideoMetadata
from tiddl.models.api import AlbumItemsCredits
from tiddl.models.constants import ARG_TO_QUALITY, TrackArg
from tiddl.models.resource import Track, Video, Album
from tiddl.utils import (
TidalResource,
formatResource,
convertFileExtension,
trackExists,
)
from typing import List, Literal, Union
from .fav import FavGroup
from .file import FileGroup
from .search import SearchGroup
@@ -8,43 +33,35 @@ from .url import UrlGroup
from ..ctx import Context, passContext
from typing import List, Literal
from tiddl.download import downloadTrackStream
from tiddl.utils import (
formatTrack,
trackExists,
TidalResource,
convertFileExtension,
)
from tiddl.metadata import addMetadata, Cover
from tiddl.exceptions import ApiError, AuthError
from tiddl.models.constants import TrackArg, ARG_TO_QUALITY
from tiddl.models.resource import Track, Album
from tiddl.models.api import PlaylistItems, AlbumItemsCredits
SinglesFilter = Literal["none", "only", "include"]
@click.command("download")
@click.option(
"--quality", "-q", "quality", type=click.Choice(TrackArg.__args__)
"--quality", "-q", "QUALITY", type=click.Choice(TrackArg.__args__)
)
@click.option(
"--output", "-o", "template", type=str, help="Format track file template."
"--output", "-o", "TEMPLATE", type=str, help="Format track file template."
)
@click.option(
"--threads",
"-t",
"THREADS_COUNT",
type=int,
help="Number of threads to use in concurrent download; use with caution.",
)
@click.option(
"--noskip",
"-ns",
"noskip",
"DO_NOT_SKIP",
is_flag=True,
default=False,
help="Dont skip downloaded tracks.",
help="Do not skip already downloaded tracks.",
)
@click.option(
"--singles",
"-s",
"singles_filter",
"SINGLES_FILTER",
type=click.Choice(SinglesFilter.__args__),
default="none",
help="Defines how to treat artist EPs and singles.",
@@ -52,92 +69,179 @@ SinglesFilter = Literal["none", "only", "include"]
@passContext
def DownloadCommand(
ctx: Context,
quality: TrackArg | None,
template: str | None,
noskip: bool,
singles_filter: SinglesFilter = "none",
QUALITY: TrackArg | None,
TEMPLATE: str | None,
THREADS_COUNT: int,
DO_NOT_SKIP: bool,
SINGLES_FILTER: SinglesFilter,
):
"""Download the tracks"""
"""Download resources"""
# TODO: pretty print
logging.debug(
(QUALITY, TEMPLATE, THREADS_COUNT, DO_NOT_SKIP, SINGLES_FILTER)
)
DOWNLOAD_QUALITY = ARG_TO_QUALITY[
QUALITY or ctx.obj.config.download.quality
]
api = ctx.obj.getApi()
def downloadTrack(
track: Track,
file_name: str,
progress = Progress(
TextColumn("{task.description}"),
BarColumn(bar_width=40),
console=ctx.obj.console,
transient=True,
auto_refresh=True,
)
def handleItemDownload(
item: Union[Track, Video],
path: Path,
cover_data=b"",
credits: List[AlbumItemsCredits.ItemWithCredits.CreditsEntry] = [],
):
if not track.allowStreaming:
logging.warning(f"Track {file_name} does not allow streaming")
return
if isinstance(item, Track):
track_stream = api.getTrackStream(item.id, quality=DOWNLOAD_QUALITY)
logging.info(
f"★ Track '{item.title}' "
f"{(str(track_stream.bitDepth) + ' bit') if track_stream.bitDepth else ''} "
f"{str(track_stream.sampleRate) + ' kHz' if track_stream.sampleRate else ''}"
)
download_quality = ARG_TO_QUALITY[
quality or ctx.obj.config.download.quality
]
urls, extension = parseTrackStream(track_stream)
elif isinstance(item, Video):
video_stream = api.getVideoStream(item.id)
logging.info(
f"★ Video '{item.title}' {video_stream.videoQuality} quality"
)
# .suffix is needed because the Path.with_suffix method will replace any content after dot
# for example: 'album/01. title' becomes 'album/01.m4a'
path = ctx.obj.config.download.path / f"{file_name}.suffix"
urls = parseVideoStream(video_stream)
extension = ".ts"
else:
raise TypeError(
f"Invalid item type: expected an instance of Track or Video, "
f"received an instance of {type(item).__name__}. "
)
if not noskip and trackExists(
track.audioQuality, download_quality, path
):
logging.info(f"Skipping track {file_name}")
return
task_id = progress.add_task(
description=f"{type(item).__name__}: {item.title}",
start=True,
visible=True,
total=len(urls),
)
logging.info(f"Downloading track {file_name}")
with Session() as s:
stream_data = b""
track_stream = api.getTrackStream(track.id, download_quality)
for url in urls:
req = s.get(url)
stream_data, file_extension = downloadTrackStream(track_stream)
assert req.status_code == 200, (
f"Could not download stream data for: "
f"{type(item).__name__} '{item.title}', "
f"status code: {req.status_code}"
)
full_path = path.with_suffix(file_extension)
full_path.parent.mkdir(parents=True, exist_ok=True)
stream_data += req.content
progress.advance(task_id)
with full_path.open("wb") as f:
path = path.with_suffix(extension)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("wb") as f:
f.write(stream_data)
# extract flac from m4a container
if isinstance(item, Track):
if track_stream.audioQuality == "HI_RES_LOSSLESS":
path = convertFileExtension(
source_file=path,
extension=".flac",
remove_source=True,
is_video=False,
copy_audio=True, # extract flac from m4a container
)
if track_stream.audioQuality == "HI_RES_LOSSLESS":
full_path = convertFileExtension(
full_path, ".flac", remove_source=True, copy_audio=True
if not cover_data and item.album.cover:
cover_data = Cover(item.album.cover).content
try:
addMetadata(path, item, cover_data, credits)
except Exception as e:
logging.error(f"Can not add metadata to: {path}, {e}")
elif isinstance(item, Video):
path = convertFileExtension(
source_file=path,
extension=".mp4",
remove_source=True,
is_video=True,
copy_audio=True,
)
if not cover_data and track.album.cover:
cover_data = Cover(track.album.cover).content
try:
addVideoMetadata(path, item)
except Exception as e:
logging.error(f"Can not add metadata to: {path}, {e}")
try:
addMetadata(
full_path, track, cover_data=cover_data, credits=credits
progress.remove_task(task_id)
logging.info(f"✔ '{item.title}'")
pool = ThreadPoolExecutor(
max_workers=THREADS_COUNT or ctx.obj.config.download.threads
)
def submitItem(
item: Union[Track, Video],
filename: str,
cover_data=b"",
credits: List[AlbumItemsCredits.ItemWithCredits.CreditsEntry] = [],
):
if not item.allowStreaming:
logging.warning(
f"✖ {type(item).__name__} '{item.title}' does not allow streaming"
)
except Exception as e:
logging.error(f"Cant set metadata to {file_name}: {e}")
return
path = ctx.obj.config.download.path / f"{filename}.*"
if not DO_NOT_SKIP: # check if item is already downloaded
if isinstance(item, Track):
if trackExists(item.audioQuality, DOWNLOAD_QUALITY, path):
logging.warning(f"Track '{item.title}' skipped")
return
elif isinstance(item, Video):
if path.with_suffix(".mp4").exists():
logging.warning(f"Video '{item.title}' skipped")
return
pool.submit(
handleItemDownload,
item=item,
path=path,
cover_data=cover_data,
credits=credits,
)
def downloadAlbum(album: Album):
logging.info(f"Album {album.title}")
logging.info(f"★ Album '{album.title}'")
cover_data = Cover(album.cover).content if album.cover else b""
offset = 0
while True:
album_items = api.getAlbumItemsCredits(album.id, offset=offset)
for item in album_items.items:
if isinstance(item.item, Track):
track = item.item
filename = formatResource(
template=TEMPLATE or ctx.obj.config.template.album,
resource=item.item,
album_artist=album.artist.name,
)
file_name = formatTrack(
template=template or ctx.obj.config.template.album,
track=track,
album_artist=album.artist.name,
)
downloadTrack(
track=track,
file_name=file_name,
cover_data=cover_data,
credits=item.credits,
)
submitItem(item.item, filename, cover_data, item.credits)
if (
album_items.limit + album_items.offset
@@ -147,26 +251,34 @@ def DownloadCommand(
offset += album_items.limit
def handleResource(resource: TidalResource):
def handleResource(resource: TidalResource) -> None:
logging.debug(f"Handling Resource '{resource}'")
match resource.type:
case "track":
track = api.getTrack(resource.id)
file_name = formatTrack(
template=template or ctx.obj.config.template.track,
track=track,
filename = formatResource(
TEMPLATE or ctx.obj.config.template.track, track
)
downloadTrack(
track=track,
file_name=file_name,
submitItem(track, filename)
case "video":
video = api.getVideo(resource.id)
filename = formatResource(
TEMPLATE or ctx.obj.config.template.video, video
)
submitItem(video, filename)
case "album":
album = api.getAlbum(resource.id)
downloadAlbum(album)
case "artist":
artist = api.getArtist(resource.id)
logging.info(f"★ Artist '{artist.name}'")
def getAllAlbums(singles: bool):
offset = 0
@@ -189,16 +301,15 @@ def DownloadCommand(
offset += artist_albums.limit
if singles_filter == "include":
if SINGLES_FILTER == "include":
getAllAlbums(False)
getAllAlbums(True)
else:
getAllAlbums(singles_filter == "only")
getAllAlbums(SINGLES_FILTER == "only")
case "playlist":
playlist = api.getPlaylist(resource.id)
logging.info(f"Playlist {playlist.title}")
logging.info(f"★ Playlist '{playlist.title}'")
offset = 0
while True:
@@ -207,21 +318,15 @@ def DownloadCommand(
)
for item in playlist_items.items:
if isinstance(
item.item,
PlaylistItems.PlaylistTrackItem.PlaylistTrack,
):
track = item.item
filename = formatResource(
template=TEMPLATE
or ctx.obj.config.template.playlist,
resource=item.item,
playlist_title=playlist.title,
playlist_index=item.item.index // 100000,
)
file_name = formatTrack(
template=template
or ctx.obj.config.template.playlist,
track=track,
playlist_title=playlist.title,
playlist_index=track.index // 100000,
)
downloadTrack(track=item.item, file_name=file_name)
submitItem(item.item, filename)
if (
playlist_items.limit + playlist_items.offset
@@ -231,16 +336,26 @@ def DownloadCommand(
offset += playlist_items.limit
progress.start()
# TODO: make sure every resource is unique
for resource in ctx.obj.resources:
try:
handleResource(resource)
except AuthError as e:
logging.error(e)
break
except ApiError as e:
logging.error(e)
except AuthError as e:
logging.error(e)
return
# session does not have streaming privileges
if e.sub_status == 4006:
break
pool.shutdown(wait=True)
progress.stop()
UrlGroup.add_command(DownloadCommand)
tiddl/cli/download/fav.py
+1 -1
View File
@@ -3,7 +3,7 @@ import click
from tiddl.utils import TidalResource, ResourceTypeLiteral
from ..ctx import Context, passContext
ResourceTypeList: list[ResourceTypeLiteral] = ["track", "album", "artist", "playlist"]
ResourceTypeList: list[ResourceTypeLiteral] = ["track", "video", "album", "artist", "playlist"]
@click.group("fav")
tiddl/cli/download/search.py
+6 -1
View File
@@ -23,6 +23,10 @@ def SearchGroup(ctx: Context, query: str):
# it's not that big deal as we refetch one resource at most,
# but it should be redesigned
if not search.topHit:
click.echo(f"No search results for '{query}'")
return
value = search.topHit.value
icon = click.style("\u2bcc", "magenta")
@@ -39,6 +43,7 @@ def SearchGroup(ctx: Context, query: str):
resource = TidalResource(type="playlist", id=str(value.uuid))
click.echo(f"{icon} Playlist {value.title}")
elif isinstance(value, Video):
click.echo(f"{icon} Video {value.title} (currently not supported)")
resource = TidalResource(type="video", id=str(value.id))
click.echo(f"{icon} Video {value.title}")
ctx.obj.resources.append(resource)
tiddl/cli/download/url.py
+1 -1
View File
@@ -21,7 +21,7 @@ def UrlGroup(ctx: Context, url: TidalResource):
Get Tidal URL.
It can be Tidal link or `resource_type/resource_id` format.
The resource can be a track, album, playlist or artist.
The resource can be a track, video, album, playlist or artist.
"""
ctx.obj.resources.append(url)
tiddl/cli/download_concurrent/__init__.py
-100
View File
@@ -1,100 +0,0 @@
import logging
import click
from ..ctx import Context, passContext
from typing import List, Literal
from concurrent.futures import ThreadPoolExecutor
from rich.console import Console
from rich.logging import RichHandler
from rich.progress import (
BarColumn,
Progress,
TextColumn,
)
from tiddl.download import downloadTrackStream
from tiddl.utils import (
formatTrack,
trackExists,
TidalResource,
convertFileExtension,
)
from tiddl.metadata import addMetadata, Cover
from tiddl.exceptions import ApiError, AuthError
from tiddl.models.constants import TrackArg, ARG_TO_QUALITY
from tiddl.models.resource import Track, Album
from tiddl.models.api import PlaylistItems, AlbumItemsCredits
SinglesFilter = Literal["none", "only", "include"]
@click.command("download")
@click.option(
"--quality", "-q", "QUALITY", type=click.Choice(TrackArg.__args__)
)
@click.option(
"--output", "-o", "TEMPLATE", type=str, help="Format track file template."
)
@click.option(
"--threads",
"-t",
"THREADS_COUNT",
type=int,
help="Number of threads to use in concurrent download; use with caution.",
default=1,
)
@click.option(
"--noskip",
"-ns",
"DO_NOT_SKIP",
is_flag=True,
default=False,
help="Do not skip already downloaded tracks.",
)
@click.option(
"--singles",
"-s",
"SINGLES_FILTER",
type=click.Choice(SinglesFilter.__args__),
default="none",
help="Defines how to treat artist EPs and singles.",
)
@passContext
def DownloadCommand(
ctx: Context,
QUALITY: TrackArg | None,
TEMPLATE: str | None,
THREADS_COUNT: int,
DO_NOT_SKIP: bool,
SINGLES_FILTER: SinglesFilter,
):
"""Download resources"""
logging.debug(
(QUALITY, TEMPLATE, THREADS_COUNT, DO_NOT_SKIP, SINGLES_FILTER)
)
api = ctx.obj.getApi()
def handleResource(resource: TidalResource) -> None:
pass
failed_resources: list[TidalResource] = []
for resource in ctx.obj.resources:
try:
handleResource(resource)
except ApiError as e:
# TODO: handle rate limit
logging.error(e)
failed_resources.append(resource)
except AuthError as e:
logging.error(e)
return
# TODO: do something with `failed_resources`
tiddl/config.py
+1 -1
View File
@@ -20,7 +20,7 @@ class TemplateConfig(BaseModel):
class DownloadConfig(BaseModel):
quality: TrackArg = "high"
path: Path = Path.home() / "Music" / "Tiddl"
threads: int = 1
threads: int = 4
class AuthConfig(BaseModel):
tiddl/metadata.py
+30 -1
View File
@@ -9,7 +9,7 @@ from mutagen.flac import Picture
from mutagen.mp4 import MP4 as MutagenMP4
from mutagen.mp4 import MP4Cover
from tiddl.models.resource import Track
from tiddl.models.resource import Track, Video
from tiddl.models.api import AlbumItemsCredits
from typing import List
@@ -105,7 +105,36 @@ def addMetadata(
logger.error(f"Failed to add metadata to {track_path}: {e}")
def addVideoMetadata(path: Path, video: Video):
metadata = MutagenEasyMP4(path)
metadata.update(
{
"title": video.title,
"albumartist": video.artist.name if video.artist else "",
"artist": ";".join(
[artist.name.strip() for artist in video.artists]
),
"album": video.album.title if video.album else "",
"date": str(video.streamStartDate) if video.streamStartDate else "",
}
)
if video.trackNumber:
metadata["tracknumber"] = str(video.trackNumber)
if video.volumeNumber:
metadata["discnumber"] = str(video.volumeNumber)
try:
metadata.save(path)
except Exception as e:
logger.error(f"Failed to add metadata to {path}: {e}")
class Cover:
# TODO: cache covers
def __init__(self, uid: str, size=1280) -> None:
if size > 1280:
logger.warning(
tiddl/models/api.py
+1 -1
View File
@@ -168,4 +168,4 @@ class Search(BaseModel):
playlists: Playlists
tracks: Tracks
videos: Videos
topHit: TopHit
topHit: Optional[TopHit] = None