Bump version to 2.6.3

Playlist Parsing

README.md

@@ -129,6 +129,7 @@ Config file values can be overridden using command-line arguments.
 | `--exclude-tags` / `exclude_tags`                               | Comma-separated tags to exclude.                                             | `null`                                         |
 | `--cover-size` / `cover_size`                                   | Cover size.                                                                  | `1200`                                         |
 | `--truncate` / `truncate`                                       | Maximum length of the file/folder names.                                     | `null`                                         |
+| `--database-path` / `database_path`                             | Path to the downloaded media database file.                                  | `null`                                         |
 | `--codec-song` / `codec_song`                                   | Song codec.                                                                  | `aac-legacy`                                   |
 | `--synced-lyrics-format` / `synced_lyrics_format`               | Synced lyrics format.                                                        | `lrc`                                          |
 | `--codec-music-video` / `codec_music_video`                     | Comma-separated music video codec priority.                                  | `h264,h265`                                    |
@@ -245,6 +246,11 @@ The following variables can be used in the template folders/files and/or in the
 - `png`: Lossless format.
 - `raw`: Raw cover without processing (requires `save_cover` to save separately).
 
+### Database path
+
+You can specify any path for storing a database file of downloaded media.
+This is useful if you want to avoid waiting for Gamdl to fetch metadata for checking if a media item has already been downloaded.
+
 ## Embedding
 
 Gamdl can be used as a library in Python scripts. Here's a basic example of downloading a song by its ID:
@@ -266,5 +272,7 @@ downloader = Downloader(
 downloader.set_cdm()
 downloader_song = DownloaderSong(downloader=downloader)
 
-downloader_song.download(media_id="1624945512")
+for download_info in downloader_song.download(media_id="1624945512"):
+    # Process download_info as needed
+    pass
 ```

gamdl/__init__.py

@@ -5,4 +5,4 @@ from .downloader_post import DownloaderPost
 from .downloader_song import DownloaderSong
 from .itunes_api import ItunesApi
 
-__version__ = "2.6.2"
+__version__ = "2.6.3"

gamdl/cli.py

@@ -319,6 +319,12 @@ def load_config_file(
     default=downloader_sig.parameters["truncate"].default,
     help="Maximum length of the file/folder names.",
 )
+@click.option(
+    "--database-path",
+    type=Path,
+    default=downloader_sig.parameters["database_path"].default,
+    help="Path to the downloaded media database file.",
+)
 # DownloaderSong specific options
 @click.option(
     "--codec-song",
@@ -401,6 +407,7 @@ def main(
     exclude_tags: list[str],
     cover_size: int,
     truncate: int,
+    database_path: Path,
     codec_song: SongCodec,
     synced_lyrics_format: SyncedLyricsFormat,
     codec_music_video: list[MusicVideoCodec],
@@ -471,6 +478,7 @@ def main(
         exclude_tags,
         cover_size,
         truncate,
+        database_path,
         log_level in ("WARNING", "ERROR"),
     )
 
@@ -605,23 +613,26 @@ def main(
                     continue
 
                 if media_metadata["type"] in {"songs", "library-songs"}:
-                    downloader_song.download(
+                    for _ in downloader_song.download(
                         media_metadata=media_metadata,
                         playlist_attributes=download_queue.playlist_attributes,
                         playlist_track=download_index,
-                    )
+                    ):
+                        pass
 
                 if media_metadata["type"] in {"music-videos", "library-music-videos"}:
-                    downloader_music_video.download(
+                    for _ in downloader_music_video.download(
                         media_metadata=media_metadata,
                         playlist_attributes=download_queue.playlist_attributes,
                         playlist_track=download_index,
-                    )
+                    ):
+                        pass
 
                 if media_metadata["type"] == "uploaded-videos":
-                    downloader_post.download(
+                    for _ in downloader_post.download(
                         media_metadata=media_metadata,
-                    )
+                    ):
+                        pass
             except KeyboardInterrupt:
                 exit(0)
             except (

gamdl/database.py

@@ -0,0 +1,50 @@
+import sqlite3
+from pathlib import Path
+
+
+class Database:
+    INITIAL_QUERY = """
+        CREATE TABLE IF NOT EXISTS media (
+            media_id TEXT PRIMARY KEY,
+            media_path TEXT NOT NULL
+        )
+    """
+    ADD_MEDIA_QUERY = """
+        INSERT OR REPLACE INTO media (media_id, media_path) VALUES (?, ?)
+    """
+    GET_MEDIA_QUERY = """
+        SELECT media_path FROM media WHERE media_id = ?
+    """
+
+    def __init__(self, file_path: Path):
+        self.file_path = file_path
+        self._initialize_db()
+
+    def _initialize_db(self):
+        self.file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with sqlite3.connect(self.file_path) as conn:
+            conn.execute(self.INITIAL_QUERY)
+            conn.commit()
+
+    def add_media(self, media_id: str, media_path: Path):
+        with sqlite3.connect(self.file_path) as conn:
+            conn.execute(
+                self.ADD_MEDIA_QUERY,
+                (
+                    media_id,
+                    str(media_path.absolute()),
+                ),
+            )
+            conn.commit()
+
+    def get_media(self, media_id: str) -> Path | None:
+        with sqlite3.connect(self.file_path) as conn:
+            cursor = conn.execute(
+                self.GET_MEDIA_QUERY,
+                (media_id,),
+            )
+            result = cursor.fetchone()
+            if result:
+                return Path(result[0])
+            return None

gamdl/downloader.py

@@ -23,6 +23,7 @@ from pywidevine import PSSH, Cdm, Device
 from yt_dlp import YoutubeDL
 
 from .apple_music_api import AppleMusicApi
+from .database import Database
 from .enums import CoverFormat, DownloadMode, MediaFileFormat, RemuxMode
 from .hardcoded_wvd import HARDCODED_WVD
 from .itunes_api import ItunesApi
@@ -47,7 +48,7 @@ class Downloader:
         r"/(?P<storefront>[a-z]{2})"
         r"/(?P<type>artist|album|playlist|song|music-video|post)"
         r"(?:/(?P<slug>[^\s/]+))?"
-        r"/(?P<id>[0-9]+|pl\.[0-9a-z]{32}|pl\.u-[a-zA-Z0-9]{15})"
+        r"/(?P<id>[0-9]+|pl\.[0-9a-z]{32}|pl\.u-[a-zA-Z0-9]{10})"
         r"(?:\?i=(?P<sub_id>[0-9]+))?"
         r")|("
         r"(?:/(?P<library_storefront>[a-z]{2}))?"
@@ -90,6 +91,7 @@ class Downloader:
         exclude_tags: list[str] = None,
         cover_size: int = 1200,
         truncate: int = None,
+        database_path: Path = None,
         silent: bool = False,
         skip_processing: bool = False,
     ):
@@ -121,12 +123,14 @@ class Downloader:
         self.exclude_tags = exclude_tags
         self.cover_size = cover_size
         self.truncate = truncate
+        self.database_path = database_path
         self.silent = silent
         self.skip_processing = skip_processing
         self._set_temp_path()
         self._set_exclude_tags()
         self._set_binaries_path_full()
         self._set_truncate()
+        self._set_database()
         self._set_subprocess_additional_args()
 
     def _set_temp_path(self):
@@ -146,6 +150,12 @@ class Downloader:
         if self.truncate is not None:
             self.truncate = None if self.truncate < 4 else self.truncate
 
+    def _set_database(self):
+        if self.database_path is not None:
+            self.database = Database(self.database_path)
+        else:
+            self.database = None
+
     def _set_subprocess_additional_args(self):
         if self.silent:
             self.subprocess_additional_args = {
@@ -347,6 +357,18 @@ class Downloader:
     ) -> bool:
         return bool(media_metadata["attributes"].get("playParams"))
 
+    def get_database_final_path(self, media_id: str) -> Path | None:
+        if self.database is None:
+            return
+
+        final_path_database = self.database.get_media(media_id)
+        if (
+            final_path_database is not None
+            and final_path_database.exists()
+            and not self.overwrite
+        ):
+            return final_path_database
+
     def get_playlist_tags(
         self,
         playlist_attributes: dict,
@@ -673,13 +695,32 @@ class Downloader:
             encoding="utf8",
         )
 
-    def cleanup_temp_path(self, override_skip_processing_check: bool = False) -> None:
-        if self.skip_processing and not override_skip_processing_check:
-            return
-
+    def cleanup_temp_path(self) -> None:
         if self.temp_path_generated.exists():
             shutil.rmtree(self.temp_path_generated)
 
+    def _final_processing_wrapper(
+        self,
+        func,
+        *args,
+        **kwargs,
+    ) -> typing.Generator[DownloadInfo, None, None]:
+        exception = None
+        download_info = None
+        try:
+            for download_info in func(*args, **kwargs):
+                yield download_info
+        except Exception as e:
+            exception = e
+        finally:
+            if download_info is not None and isinstance(download_info, DownloadInfo):
+                self._final_processing(
+                    download_info,
+                )
+
+            if exception is not None:
+                raise exception
+
     def _final_processing(
         self,
         download_info: DownloadInfo,
@@ -687,11 +728,20 @@ class Downloader:
         if self.skip_processing:
             return
 
-        colored_media_id = color_text(download_info.media_id, colorama.Style.DIM)
+        if download_info.media_id:
+            colored_media_id = color_text(
+                download_info.media_id,
+                colorama.Style.DIM,
+            )
+        else:
+            colored_media_id = color_text(
+                "Unknown",
+                colorama.Style.DIM,
+            )
 
         if download_info.staged_path:
             logger.debug(
-                f"[{colored_media_id}] Applying tags to {download_info.staged_path}"
+                f'[{colored_media_id}] Applying tags to "{download_info.staged_path}"'
             )
             self.apply_tags(
                 download_info.staged_path,
@@ -707,6 +757,15 @@ class Downloader:
             )
             logger.info(f"[{colored_media_id}] Download completed successfully")
 
+            if self.database is not None:
+                logger.debug(
+                    f'[{colored_media_id}] Adding entry to database at "{self.database_path}"'
+                )
+                self.database.add_media(
+                    download_info.media_id,
+                    download_info.final_path,
+                )
+
         if (
             download_info.cover_path and not self.save_cover
         ) or not download_info.cover_path:
@@ -742,6 +801,7 @@ class Downloader:
                 download_info.synced_lyrics_path,
                 download_info.lyrics.synced,
             )
+
         if download_info.playlist_tags and self.save_playlist:
             playlist_file_path = self.get_playlist_file_path(
                 download_info.playlist_tags
@@ -754,3 +814,5 @@ class Downloader:
                 download_info.final_path,
                 download_info.playlist_tags.playlist_track,
             )
+
+        self.cleanup_temp_path()

gamdl/downloader_music_video.py

@@ -430,24 +430,22 @@ class DownloaderMusicVideo:
             self.downloader.get_cover_file_extension(cover_format)
         )
 
+    import typing
+
     def download(
         self,
         media_id: str = None,
         media_metadata: dict = None,
         playlist_attributes: dict = None,
         playlist_track: int = None,
-    ) -> DownloadInfo:
-        try:
-            download_info = self._download(
-                media_id,
-                media_metadata,
-                playlist_attributes,
-                playlist_track,
-            )
-            self.downloader._final_processing(download_info)
-        finally:
-            self.downloader.cleanup_temp_path()
-        return download_info
+    ) -> typing.Generator[DownloadInfo, None, None]:
+        yield from self.downloader._final_processing_wrapper(
+            self._download,
+            media_id,
+            media_metadata,
+            playlist_attributes,
+            playlist_track,
+        )
 
     def _download(
         self,
@@ -455,8 +453,9 @@ class DownloaderMusicVideo:
         media_metadata: dict = None,
         playlist_attributes: dict = None,
         playlist_track: int = None,
-    ) -> DownloadInfo:
+    ) -> typing.Generator[DownloadInfo, None, None]:
         download_info = DownloadInfo()
+        yield download_info
 
         if playlist_track is None and playlist_attributes:
             raise ValueError(
@@ -474,20 +473,24 @@ class DownloaderMusicVideo:
         if not media_id and not media_metadata:
             raise ValueError("Either media_id or media_metadata must be provided")
 
-        if not media_metadata:
-            logger.debug(
-                f"[{color_text(media_id, colorama.Style.DIM)}] "
-                "Getting Music Video metadata"
-            )
-            media_metadata = self.downloader.apple_music_api.get_music_video(media_id)
-        download_info.media_metadata = media_metadata
-
-        if not media_id:
+        if media_metadata:
             media_id = self.downloader.get_media_id_of_library_media(media_metadata)
         download_info.media_id = media_id
         colored_media_id = color_text(media_id, colorama.Style.DIM)
 
+        database_final_path = self.downloader.get_database_final_path(media_id)
+        if database_final_path:
+            download_info.final_path = database_final_path
+            yield download_info
+            raise MediaFileAlreadyExistsException(database_final_path)
+
+        if not media_metadata:
+            logger.debug(f"[{colored_media_id}] Getting Music Video metadata")
+            media_metadata = self.downloader.apple_music_api.get_music_video(media_id)
+        download_info.media_metadata = media_metadata
+
         if not self.downloader.is_media_streamable(media_metadata):
+            yield download_info
             raise MediaNotStreamableException()
 
         alt_media_id = self.get_music_video_id_alt(media_metadata) or media_id
@@ -515,8 +518,11 @@ class DownloaderMusicVideo:
             webplayback = self.downloader.apple_music_api.get_webplayback(media_id)
             logger.debug(f"[{colored_media_id}] Getting stream info")
             stream_info = self.get_stream_info_from_webplayback(webplayback)
+
         if not stream_info:
+            yield download_info
             raise MediaFormatNotAvailableException()
+
         download_info.stream_info = stream_info
 
         final_path = self.downloader.get_final_path(
@@ -537,6 +543,7 @@ class DownloaderMusicVideo:
         download_info.cover_path = cover_path
 
         if final_path.exists() and not self.downloader.overwrite:
+            yield download_info
             raise MediaFileAlreadyExistsException(final_path)
 
         logger.debug(f"[{colored_media_id}] Getting decryption key")
@@ -590,6 +597,7 @@ class DownloaderMusicVideo:
         )
 
         logger.debug(
+            f"[{colored_media_id}] "
             "Decrypting video/audio to "
             f'{decrypted_path_video}"/"{decrypted_path_audio}" '
             f'and remuxing to "{staged_path}"'
@@ -604,4 +612,4 @@ class DownloaderMusicVideo:
         )
         download_info.staged_path = staged_path
 
-        return download_info
+        yield download_info

gamdl/downloader_post.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import typing
 from pathlib import Path
 
 import colorama
@@ -87,41 +88,42 @@ class DownloaderPost:
         self,
         media_id: str = None,
         media_metadata: dict = None,
-    ) -> DownloadInfo:
-        try:
-            download_info = self._download(
-                media_id,
-                media_metadata,
-            )
-            self.downloader._final_processing(download_info)
-        finally:
-            self.downloader.cleanup_temp_path()
-        return download_info
+    ) -> typing.Generator[DownloadInfo, None, None]:
+        yield from self.downloader._final_processing_wrapper(
+            self._download,
+            media_id,
+            media_metadata,
+        )
 
     def _download(
         self,
         media_id: str = None,
         media_metadata: dict = None,
-    ) -> DownloadInfo:
+    ) -> typing.Generator[DownloadInfo, None, None]:
         download_info = DownloadInfo()
+        yield download_info
 
         if not media_id and not media_metadata:
             raise ValueError("Either media_id or media_metadata must be provided")
 
-        if not media_metadata:
-            logger.debug(
-                f"[{color_text(media_id, colorama.Style.DIM)}] "
-                "Getting Post Video metadata"
-            )
-            media_metadata = self.downloader.apple_music_api.get_post(media_id)
-        download_info.media_metadata = media_metadata
-
-        if not media_id:
+        if media_metadata:
             media_id = media_metadata["id"]
         download_info.media_id = media_id
         colored_media_id = color_text(media_id, colorama.Style.DIM)
 
+        database_final_path = self.downloader.get_database_final_path(media_id)
+        if database_final_path:
+            download_info.final_path = database_final_path
+            yield download_info
+            raise MediaFileAlreadyExistsException(database_final_path)
+
+        if not media_metadata:
+            logger.debug(f"[{colored_media_id}] Getting Post Video metadata")
+            media_metadata = self.downloader.apple_music_api.get_post(media_id)
+        download_info.media_metadata = media_metadata
+
         if not self.downloader.is_media_streamable(media_metadata):
+            yield download_info
             raise MediaNotStreamableException()
 
         tags = self.get_tags(media_metadata)
@@ -134,6 +136,7 @@ class DownloaderPost:
         download_info.final_path = final_path
 
         if final_path.exists() and not self.downloader.overwrite:
+            yield download_info
             raise MediaFileAlreadyExistsException(final_path)
 
         cover_url = self.downloader.get_cover_url(media_metadata)
@@ -162,4 +165,4 @@ class DownloaderPost:
         )
         download_info.staged_path = staged_path
 
-        return download_info
+        yield download_info

gamdl/downloader_song.py

@@ -6,6 +6,7 @@ import json
 import logging
 import re
 import subprocess
+import typing
 from pathlib import Path
 from xml.dom import minidom
 from xml.etree import ElementTree
@@ -593,18 +594,14 @@ class DownloaderSong:
         media_metadata: dict = None,
         playlist_attributes: dict = None,
         playlist_track: int = None,
-    ) -> DownloadInfo:
-        try:
-            download_info = self._download(
-                media_id,
-                media_metadata,
-                playlist_attributes,
-                playlist_track,
-            )
-            self.downloader._final_processing(download_info)
-        finally:
-            self.downloader.cleanup_temp_path()
-        return download_info
+    ) -> typing.Generator[DownloadInfo, None, None]:
+        yield from self.downloader._final_processing_wrapper(
+            self._download,
+            media_id,
+            media_metadata,
+            playlist_attributes,
+            playlist_track,
+        )
 
     def _download(
         self,
@@ -612,8 +609,9 @@ class DownloaderSong:
         media_metadata: dict = None,
         playlist_attributes: dict = None,
         playlist_track: int = None,
-    ) -> DownloadInfo:
+    ) -> typing.Generator[DownloadInfo, None, None]:
         download_info = DownloadInfo()
+        yield download_info
 
         if playlist_track is None and playlist_attributes:
             raise ValueError(
@@ -631,18 +629,22 @@ class DownloaderSong:
         if not media_id and not media_metadata:
             raise ValueError("Either media_id or media_metadata must be provided")
 
-        if not media_metadata:
-            logger.debug(
-                f"[{color_text(media_id, colorama.Style.DIM)}] Getting Song metadata"
-            )
-            media_metadata = self.downloader.apple_music_api.get_song(media_id)
-        download_info.media_metadata = media_metadata
-
-        if not media_id:
+        if media_metadata:
             media_id = self.downloader.get_media_id_of_library_media(media_metadata)
         download_info.media_id = media_id
         colored_media_id = color_text(media_id, colorama.Style.DIM)
 
+        database_final_path = self.downloader.get_database_final_path(media_id)
+        if database_final_path:
+            download_info.final_path = database_final_path
+            yield download_info
+            raise MediaFileAlreadyExistsException(database_final_path)
+
+        if not media_metadata:
+            logger.debug(f"[{colored_media_id}] Getting Song metadata")
+            media_metadata = self.downloader.apple_music_api.get_song(media_id)
+        download_info.media_metadata = media_metadata
+
         if not self.downloader.is_media_streamable(media_metadata):
             raise MediaNotStreamableException()
 
@@ -672,7 +674,8 @@ class DownloaderSong:
             logger.info(
                 f"[{colored_media_id}] Downloading synced lyrics only, skipping song download"
             )
-            return download_info
+            yield download_info
+            return
 
         cover_url = self.downloader.get_cover_url(media_metadata)
         cover_format = self.downloader.get_cover_format(cover_url)
@@ -685,6 +688,7 @@ class DownloaderSong:
         download_info.cover_path = cover_path
 
         if final_path.exists() and not self.downloader.overwrite:
+            yield download_info
             raise MediaFileAlreadyExistsException(final_path)
 
         logger.debug(f"[{colored_media_id}] Getting stream info")
@@ -699,8 +703,11 @@ class DownloaderSong:
             download_info.decryption_key = decryption_key
         else:
             stream_info = self.get_stream_info(media_metadata)
+
             if not stream_info or not stream_info.audio_track.widevine_pssh:
+                yield download_info
                 raise MediaFormatNotAvailableException()
+
             logger.debug(f"[{colored_media_id}] Getting decryption key")
             decryption_key = self.get_decryption_key(
                 stream_info,
@@ -745,4 +752,4 @@ class DownloaderSong:
         )
         download_info.staged_path = staged_path
 
-        return download_info
+        yield download_info