diff --git a/README.md b/README.md index 2980e93..5ad5b57 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,11 @@ tiddl auth login ## Download resource -You can download track / album / artist / playlist / (video coming soon) - +You can download track / video / album / artist / playlist ```bash tiddl url https://listen.tidal.com/track/103805726 download +tiddl url https://listen.tidal.com/video/25747442 download tiddl url https://listen.tidal.com/album/103805723 download tiddl url https://listen.tidal.com/artist/25022 download tiddl url https://listen.tidal.com/playlist/84974059-76af-406a-aede-ece2b78fa372 download diff --git a/examples/concurrent_download_rich.py b/examples/concurrent_download_rich.py index a1ec780..511a26f 100644 --- a/examples/concurrent_download_rich.py +++ b/examples/concurrent_download_rich.py @@ -52,8 +52,6 @@ progress = Progress( def handleItemDownload(item: Union[Track, Video]): - # TODO: check if item is already downloaded - if isinstance(item, Track): track_stream = api.getTrackStream(item.id, quality=QUALITY) urls, extension = parseTrackStream(track_stream) diff --git a/tiddl/api.py b/tiddl/api.py index 3144d74..1e0df58 100644 --- a/tiddl/api.py +++ b/tiddl/api.py @@ -4,7 +4,12 @@ from pathlib import Path from typing import Any, Literal, Type, TypeVar from pydantic import BaseModel -from requests_cache import CachedSession, EXPIRE_IMMEDIATELY, NEVER_EXPIRE +from requests_cache import ( + CachedSession, + EXPIRE_IMMEDIATELY, + NEVER_EXPIRE, + DO_NOT_CACHE, +) from tiddl.models.api import ( Album, @@ -28,6 +33,7 @@ from tiddl.exceptions import ApiError from tiddl.config import HOME_PATH DEBUG = False + T = TypeVar("T", bound=BaseModel) logger = logging.getLogger(__name__) @@ -209,7 +215,7 @@ class TidalApi: def getSession(self): return self.fetch( - SessionResponse, "sessions", expire_after=EXPIRE_IMMEDIATELY + SessionResponse, "sessions", expire_after=DO_NOT_CACHE ) def getTrack(self, track_id: str | int): @@ -226,7 +232,7 @@ class TidalApi: "playbackmode": "STREAM", "assetpresentation": "FULL", }, - expire_after=3600, + expire_after=DO_NOT_CACHE, ) def getVideo(self, video_id: str | int): @@ -243,5 +249,5 @@ class TidalApi: "playbackmode": "STREAM", "assetpresentation": "FULL", }, - expire_after=3600, + expire_after=DO_NOT_CACHE, ) diff --git a/tiddl/cli/__init__.py b/tiddl/cli/__init__.py index 9b3651b..181add1 100644 --- a/tiddl/cli/__init__.py +++ b/tiddl/cli/__init__.py @@ -7,7 +7,6 @@ from .ctx import ContextObj, passContext, Context from .auth import AuthGroup from .download import UrlGroup, FavGroup, SearchGroup, FileGroup from .config import ConfigCommand -from .download_concurrent import DownloadCommand from tiddl.config import HOME_PATH @@ -59,7 +58,6 @@ cli.add_command(UrlGroup) cli.add_command(FavGroup) cli.add_command(SearchGroup) cli.add_command(FileGroup) -cli.add_command(DownloadCommand) if __name__ == "__main__": cli() diff --git a/tiddl/cli/download/__init__.py b/tiddl/cli/download/__init__.py index 5d3ba0b..d6f423d 100644 --- a/tiddl/cli/download/__init__.py +++ b/tiddl/cli/download/__init__.py @@ -1,6 +1,31 @@ import logging import click +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from requests import Session + +from rich.progress import ( + BarColumn, + Progress, + TextColumn, +) + +from tiddl.download import parseTrackStream, parseVideoStream +from tiddl.exceptions import ApiError, AuthError +from tiddl.metadata import Cover, addMetadata, addVideoMetadata +from tiddl.models.api import AlbumItemsCredits +from tiddl.models.constants import ARG_TO_QUALITY, TrackArg +from tiddl.models.resource import Track, Video, Album +from tiddl.utils import ( + TidalResource, + formatResource, + convertFileExtension, + trackExists, +) + +from typing import List, Literal, Union + from .fav import FavGroup from .file import FileGroup from .search import SearchGroup @@ -8,43 +33,35 @@ from .url import UrlGroup from ..ctx import Context, passContext -from typing import List, Literal - -from tiddl.download import downloadTrackStream -from tiddl.utils import ( - formatTrack, - trackExists, - TidalResource, - convertFileExtension, -) -from tiddl.metadata import addMetadata, Cover -from tiddl.exceptions import ApiError, AuthError -from tiddl.models.constants import TrackArg, ARG_TO_QUALITY -from tiddl.models.resource import Track, Album -from tiddl.models.api import PlaylistItems, AlbumItemsCredits - SinglesFilter = Literal["none", "only", "include"] @click.command("download") @click.option( - "--quality", "-q", "quality", type=click.Choice(TrackArg.__args__) + "--quality", "-q", "QUALITY", type=click.Choice(TrackArg.__args__) ) @click.option( - "--output", "-o", "template", type=str, help="Format track file template." + "--output", "-o", "TEMPLATE", type=str, help="Format track file template." +) +@click.option( + "--threads", + "-t", + "THREADS_COUNT", + type=int, + help="Number of threads to use in concurrent download; use with caution.", ) @click.option( "--noskip", "-ns", - "noskip", + "DO_NOT_SKIP", is_flag=True, default=False, - help="Dont skip downloaded tracks.", + help="Do not skip already downloaded tracks.", ) @click.option( "--singles", "-s", - "singles_filter", + "SINGLES_FILTER", type=click.Choice(SinglesFilter.__args__), default="none", help="Defines how to treat artist EPs and singles.", @@ -52,92 +69,179 @@ SinglesFilter = Literal["none", "only", "include"] @passContext def DownloadCommand( ctx: Context, - quality: TrackArg | None, - template: str | None, - noskip: bool, - singles_filter: SinglesFilter = "none", + QUALITY: TrackArg | None, + TEMPLATE: str | None, + THREADS_COUNT: int, + DO_NOT_SKIP: bool, + SINGLES_FILTER: SinglesFilter, ): - """Download the tracks""" + """Download resources""" + + # TODO: pretty print + logging.debug( + (QUALITY, TEMPLATE, THREADS_COUNT, DO_NOT_SKIP, SINGLES_FILTER) + ) + + DOWNLOAD_QUALITY = ARG_TO_QUALITY[ + QUALITY or ctx.obj.config.download.quality + ] api = ctx.obj.getApi() - def downloadTrack( - track: Track, - file_name: str, + progress = Progress( + TextColumn("{task.description}"), + BarColumn(bar_width=40), + console=ctx.obj.console, + transient=True, + auto_refresh=True, + ) + + def handleItemDownload( + item: Union[Track, Video], + path: Path, cover_data=b"", credits: List[AlbumItemsCredits.ItemWithCredits.CreditsEntry] = [], ): - if not track.allowStreaming: - logging.warning(f"Track {file_name} does not allow streaming") - return + if isinstance(item, Track): + track_stream = api.getTrackStream(item.id, quality=DOWNLOAD_QUALITY) + logging.info( + f"★ Track '{item.title}' " + f"{(str(track_stream.bitDepth) + ' bit') if track_stream.bitDepth else ''} " + f"{str(track_stream.sampleRate) + ' kHz' if track_stream.sampleRate else ''}" + ) - download_quality = ARG_TO_QUALITY[ - quality or ctx.obj.config.download.quality - ] + urls, extension = parseTrackStream(track_stream) + elif isinstance(item, Video): + video_stream = api.getVideoStream(item.id) + logging.info( + f"★ Video '{item.title}' {video_stream.videoQuality} quality" + ) - # .suffix is needed because the Path.with_suffix method will replace any content after dot - # for example: 'album/01. title' becomes 'album/01.m4a' - path = ctx.obj.config.download.path / f"{file_name}.suffix" + urls = parseVideoStream(video_stream) + extension = ".ts" + else: + raise TypeError( + f"Invalid item type: expected an instance of Track or Video, " + f"received an instance of {type(item).__name__}. " + ) - if not noskip and trackExists( - track.audioQuality, download_quality, path - ): - logging.info(f"Skipping track {file_name}") - return + task_id = progress.add_task( + description=f"{type(item).__name__}: {item.title}", + start=True, + visible=True, + total=len(urls), + ) - logging.info(f"Downloading track {file_name}") + with Session() as s: + stream_data = b"" - track_stream = api.getTrackStream(track.id, download_quality) + for url in urls: + req = s.get(url) - stream_data, file_extension = downloadTrackStream(track_stream) + assert req.status_code == 200, ( + f"Could not download stream data for: " + f"{type(item).__name__} '{item.title}', " + f"status code: {req.status_code}" + ) - full_path = path.with_suffix(file_extension) - full_path.parent.mkdir(parents=True, exist_ok=True) + stream_data += req.content + progress.advance(task_id) - with full_path.open("wb") as f: + path = path.with_suffix(extension) + path.parent.mkdir(parents=True, exist_ok=True) + + with path.open("wb") as f: f.write(stream_data) - # extract flac from m4a container + if isinstance(item, Track): + if track_stream.audioQuality == "HI_RES_LOSSLESS": + path = convertFileExtension( + source_file=path, + extension=".flac", + remove_source=True, + is_video=False, + copy_audio=True, # extract flac from m4a container + ) - if track_stream.audioQuality == "HI_RES_LOSSLESS": - full_path = convertFileExtension( - full_path, ".flac", remove_source=True, copy_audio=True + if not cover_data and item.album.cover: + cover_data = Cover(item.album.cover).content + + try: + addMetadata(path, item, cover_data, credits) + except Exception as e: + logging.error(f"Can not add metadata to: {path}, {e}") + + elif isinstance(item, Video): + path = convertFileExtension( + source_file=path, + extension=".mp4", + remove_source=True, + is_video=True, + copy_audio=True, ) - if not cover_data and track.album.cover: - cover_data = Cover(track.album.cover).content + try: + addVideoMetadata(path, item) + except Exception as e: + logging.error(f"Can not add metadata to: {path}, {e}") - try: - addMetadata( - full_path, track, cover_data=cover_data, credits=credits + progress.remove_task(task_id) + logging.info(f"✔ '{item.title}'") + + pool = ThreadPoolExecutor( + max_workers=THREADS_COUNT or ctx.obj.config.download.threads + ) + + def submitItem( + item: Union[Track, Video], + filename: str, + cover_data=b"", + credits: List[AlbumItemsCredits.ItemWithCredits.CreditsEntry] = [], + ): + if not item.allowStreaming: + logging.warning( + f"✖ {type(item).__name__} '{item.title}' does not allow streaming" ) - except Exception as e: - logging.error(f"Cant set metadata to {file_name}: {e}") + return + + path = ctx.obj.config.download.path / f"{filename}.*" + + if not DO_NOT_SKIP: # check if item is already downloaded + if isinstance(item, Track): + if trackExists(item.audioQuality, DOWNLOAD_QUALITY, path): + logging.warning(f"Track '{item.title}' skipped") + return + elif isinstance(item, Video): + if path.with_suffix(".mp4").exists(): + logging.warning(f"Video '{item.title}' skipped") + return + + pool.submit( + handleItemDownload, + item=item, + path=path, + cover_data=cover_data, + credits=credits, + ) def downloadAlbum(album: Album): - logging.info(f"Album {album.title}") + logging.info(f"★ Album '{album.title}'") + cover_data = Cover(album.cover).content if album.cover else b"" offset = 0 while True: album_items = api.getAlbumItemsCredits(album.id, offset=offset) + for item in album_items.items: - if isinstance(item.item, Track): - track = item.item + filename = formatResource( + template=TEMPLATE or ctx.obj.config.template.album, + resource=item.item, + album_artist=album.artist.name, + ) - file_name = formatTrack( - template=template or ctx.obj.config.template.album, - track=track, - album_artist=album.artist.name, - ) - - downloadTrack( - track=track, - file_name=file_name, - cover_data=cover_data, - credits=item.credits, - ) + submitItem(item.item, filename, cover_data, item.credits) if ( album_items.limit + album_items.offset @@ -147,26 +251,34 @@ def DownloadCommand( offset += album_items.limit - def handleResource(resource: TidalResource): + def handleResource(resource: TidalResource) -> None: + logging.debug(f"Handling Resource '{resource}'") + match resource.type: case "track": track = api.getTrack(resource.id) - file_name = formatTrack( - template=template or ctx.obj.config.template.track, - track=track, + filename = formatResource( + TEMPLATE or ctx.obj.config.template.track, track ) - downloadTrack( - track=track, - file_name=file_name, + submitItem(track, filename) + + case "video": + video = api.getVideo(resource.id) + filename = formatResource( + TEMPLATE or ctx.obj.config.template.video, video ) + submitItem(video, filename) + case "album": album = api.getAlbum(resource.id) downloadAlbum(album) case "artist": + artist = api.getArtist(resource.id) + logging.info(f"★ Artist '{artist.name}'") def getAllAlbums(singles: bool): offset = 0 @@ -189,16 +301,15 @@ def DownloadCommand( offset += artist_albums.limit - if singles_filter == "include": + if SINGLES_FILTER == "include": getAllAlbums(False) getAllAlbums(True) else: - getAllAlbums(singles_filter == "only") + getAllAlbums(SINGLES_FILTER == "only") case "playlist": playlist = api.getPlaylist(resource.id) - logging.info(f"Playlist {playlist.title}") - + logging.info(f"★ Playlist '{playlist.title}'") offset = 0 while True: @@ -207,21 +318,15 @@ def DownloadCommand( ) for item in playlist_items.items: - if isinstance( - item.item, - PlaylistItems.PlaylistTrackItem.PlaylistTrack, - ): - track = item.item + filename = formatResource( + template=TEMPLATE + or ctx.obj.config.template.playlist, + resource=item.item, + playlist_title=playlist.title, + playlist_index=item.item.index // 100000, + ) - file_name = formatTrack( - template=template - or ctx.obj.config.template.playlist, - track=track, - playlist_title=playlist.title, - playlist_index=track.index // 100000, - ) - - downloadTrack(track=item.item, file_name=file_name) + submitItem(item.item, filename) if ( playlist_items.limit + playlist_items.offset @@ -231,16 +336,26 @@ def DownloadCommand( offset += playlist_items.limit + progress.start() + + # TODO: make sure every resource is unique for resource in ctx.obj.resources: try: handleResource(resource) + except AuthError as e: + logging.error(e) + break + except ApiError as e: logging.error(e) - except AuthError as e: - logging.error(e) - return + # session does not have streaming privileges + if e.sub_status == 4006: + break + + pool.shutdown(wait=True) + progress.stop() UrlGroup.add_command(DownloadCommand) diff --git a/tiddl/cli/download/fav.py b/tiddl/cli/download/fav.py index 5a38259..bb76e74 100644 --- a/tiddl/cli/download/fav.py +++ b/tiddl/cli/download/fav.py @@ -3,7 +3,7 @@ import click from tiddl.utils import TidalResource, ResourceTypeLiteral from ..ctx import Context, passContext -ResourceTypeList: list[ResourceTypeLiteral] = ["track", "album", "artist", "playlist"] +ResourceTypeList: list[ResourceTypeLiteral] = ["track", "video", "album", "artist", "playlist"] @click.group("fav") diff --git a/tiddl/cli/download/search.py b/tiddl/cli/download/search.py index db8c82a..01abae8 100644 --- a/tiddl/cli/download/search.py +++ b/tiddl/cli/download/search.py @@ -23,6 +23,10 @@ def SearchGroup(ctx: Context, query: str): # it's not that big deal as we refetch one resource at most, # but it should be redesigned + if not search.topHit: + click.echo(f"No search results for '{query}'") + return + value = search.topHit.value icon = click.style("\u2bcc", "magenta") @@ -39,6 +43,7 @@ def SearchGroup(ctx: Context, query: str): resource = TidalResource(type="playlist", id=str(value.uuid)) click.echo(f"{icon} Playlist {value.title}") elif isinstance(value, Video): - click.echo(f"{icon} Video {value.title} (currently not supported)") + resource = TidalResource(type="video", id=str(value.id)) + click.echo(f"{icon} Video {value.title}") ctx.obj.resources.append(resource) diff --git a/tiddl/cli/download/url.py b/tiddl/cli/download/url.py index 265987d..a485fe8 100644 --- a/tiddl/cli/download/url.py +++ b/tiddl/cli/download/url.py @@ -21,7 +21,7 @@ def UrlGroup(ctx: Context, url: TidalResource): Get Tidal URL. It can be Tidal link or `resource_type/resource_id` format. - The resource can be a track, album, playlist or artist. + The resource can be a track, video, album, playlist or artist. """ ctx.obj.resources.append(url) diff --git a/tiddl/cli/download_concurrent/__init__.py b/tiddl/cli/download_concurrent/__init__.py deleted file mode 100644 index 1d551cb..0000000 --- a/tiddl/cli/download_concurrent/__init__.py +++ /dev/null @@ -1,100 +0,0 @@ -import logging -import click - -from ..ctx import Context, passContext - -from typing import List, Literal - -from concurrent.futures import ThreadPoolExecutor - -from rich.console import Console -from rich.logging import RichHandler -from rich.progress import ( - BarColumn, - Progress, - TextColumn, -) - -from tiddl.download import downloadTrackStream -from tiddl.utils import ( - formatTrack, - trackExists, - TidalResource, - convertFileExtension, -) -from tiddl.metadata import addMetadata, Cover -from tiddl.exceptions import ApiError, AuthError -from tiddl.models.constants import TrackArg, ARG_TO_QUALITY -from tiddl.models.resource import Track, Album -from tiddl.models.api import PlaylistItems, AlbumItemsCredits - -SinglesFilter = Literal["none", "only", "include"] - - -@click.command("download") -@click.option( - "--quality", "-q", "QUALITY", type=click.Choice(TrackArg.__args__) -) -@click.option( - "--output", "-o", "TEMPLATE", type=str, help="Format track file template." -) -@click.option( - "--threads", - "-t", - "THREADS_COUNT", - type=int, - help="Number of threads to use in concurrent download; use with caution.", - default=1, -) -@click.option( - "--noskip", - "-ns", - "DO_NOT_SKIP", - is_flag=True, - default=False, - help="Do not skip already downloaded tracks.", -) -@click.option( - "--singles", - "-s", - "SINGLES_FILTER", - type=click.Choice(SinglesFilter.__args__), - default="none", - help="Defines how to treat artist EPs and singles.", -) -@passContext -def DownloadCommand( - ctx: Context, - QUALITY: TrackArg | None, - TEMPLATE: str | None, - THREADS_COUNT: int, - DO_NOT_SKIP: bool, - SINGLES_FILTER: SinglesFilter, -): - """Download resources""" - - logging.debug( - (QUALITY, TEMPLATE, THREADS_COUNT, DO_NOT_SKIP, SINGLES_FILTER) - ) - - api = ctx.obj.getApi() - - def handleResource(resource: TidalResource) -> None: - pass - - failed_resources: list[TidalResource] = [] - - for resource in ctx.obj.resources: - try: - handleResource(resource) - - except ApiError as e: - # TODO: handle rate limit - logging.error(e) - failed_resources.append(resource) - - except AuthError as e: - logging.error(e) - return - - # TODO: do something with `failed_resources` diff --git a/tiddl/config.py b/tiddl/config.py index c5300ce..6de03f6 100644 --- a/tiddl/config.py +++ b/tiddl/config.py @@ -20,7 +20,7 @@ class TemplateConfig(BaseModel): class DownloadConfig(BaseModel): quality: TrackArg = "high" path: Path = Path.home() / "Music" / "Tiddl" - threads: int = 1 + threads: int = 4 class AuthConfig(BaseModel): diff --git a/tiddl/metadata.py b/tiddl/metadata.py index 1489945..c0d9c5f 100644 --- a/tiddl/metadata.py +++ b/tiddl/metadata.py @@ -9,7 +9,7 @@ from mutagen.flac import Picture from mutagen.mp4 import MP4 as MutagenMP4 from mutagen.mp4 import MP4Cover -from tiddl.models.resource import Track +from tiddl.models.resource import Track, Video from tiddl.models.api import AlbumItemsCredits from typing import List @@ -105,7 +105,36 @@ def addMetadata( logger.error(f"Failed to add metadata to {track_path}: {e}") +def addVideoMetadata(path: Path, video: Video): + metadata = MutagenEasyMP4(path) + + metadata.update( + { + "title": video.title, + "albumartist": video.artist.name if video.artist else "", + "artist": ";".join( + [artist.name.strip() for artist in video.artists] + ), + "album": video.album.title if video.album else "", + "date": str(video.streamStartDate) if video.streamStartDate else "", + } + ) + + if video.trackNumber: + metadata["tracknumber"] = str(video.trackNumber) + + if video.volumeNumber: + metadata["discnumber"] = str(video.volumeNumber) + + try: + metadata.save(path) + except Exception as e: + logger.error(f"Failed to add metadata to {path}: {e}") + + class Cover: + # TODO: cache covers + def __init__(self, uid: str, size=1280) -> None: if size > 1280: logger.warning( diff --git a/tiddl/models/api.py b/tiddl/models/api.py index 8c7478b..7f310ef 100644 --- a/tiddl/models/api.py +++ b/tiddl/models/api.py @@ -168,4 +168,4 @@ class Search(BaseModel): playlists: Playlists tracks: Tracks videos: Videos - topHit: TopHit + topHit: Optional[TopHit] = None