#!/home/lucas/scripts/.venv/bin/python import argparse import json import logging import re import sqlite3 import subprocess import requests import sys from typing import Any import tempfile from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from mutagen import File from mutagen.id3 import ( COMM, ID3, TALB, TCOM, TCON, TIT1, TIT2, TPE1, TPE2, ) from mutagen.mp4 import MP4, MP4Tags from mutagen.oggopus import OggOpus from serverlibrary import ( AUDIO_RE_SUFFIXES, AUDIO_SUFFIXES, CustomLogger, blue_tag, green_tag, red_tag, score_match_jaccard, setup_logging, words, parser_add_loglevel ) ID3_FRAME_CLASSES = { "TIT2": TIT2, "TALB": TALB, "TPE1": TPE1, "TPE2": TPE2, "TCON": TCON, "TCOM": TCOM, "TIT1": TIT1, } logging.setLoggerClass(CustomLogger) logger: CustomLogger = logging.getLogger(__name__) # type: ignore[assignment] MP4_MAP = { "artist": "\xa9ART", "album": "\xa9alb", "title": "\xa9nam", "genre": "\xa9gen", "comment": "\xa9cmt", "composer": "\xa9wrt", "albumartist": "aART", "grouping": "\xa9grp", "description": "desc", } ID3_MAP = { "artist": "TPE1", "album": "TALB", "title": "TIT2", "genre": "TCON", "composer": "TCOM", "albumartist": "TPE2", "grouping": "TIT1", } metadata_fields = [ "artist", "album", "composer", "comment", "genre", "grouping", "albumartist", "description", "title", ] SEP = r"\s*[-_:,.]+\s*" series_patterns = [ rf"^(.+?){SEP}(?:Book|Band|Volume|Vol)\s+(\d+)(?:{SEP}(.+))$", rf"^(.+?){SEP}(?:Book|Band|Volume|Vol)\s+(\d+)$", rf"^(\d+){SEP}(.+)$", rf"^(.+?){SEP}(\d+)$", ] def ffmpeg_escape(path: Path) -> str: return str(path.resolve()).replace("'", r"'\''") def clean_genre(genre): if not genre: return "" genre_old = genre # genre = genre.lower() genre = re.sub(r"(?i)^audio\s*books?$", "", genre) genre = re.sub(r"(?i)^audiobooks?;?$", "", genre) genre = re.sub(r"(?i)\b(hörb(uch|ücher)|hoerb(uch|uecher))\b", "", genre) genre = re.sub(r"(?i)^(hörspiele?|hoerspiele?)$", "", genre) genre = re.sub(r"(?i)^spoken\s*word$", "", genre) genre = re.sub(r"(?i)gesprochene\s*inhalte$", "", genre) genre = re.sub(r"(?i)^speech$", "", genre) genre = re.sub(r"Fanasie", "Fantasie", genre) genre = re.sub(r"Mit Videospiel-?", "Videospielbezug", genre) genre = re.sub(r"(?i)nonfiction", "NonFiction", genre) genre = re.sub(r"(?i)^speech$", "", genre) genre = re.sub(r"(?i)^vocal$", "", genre) genre = re.sub(r"(?i)^retail$", "", genre) genre = re.sub(r"(?i)^other$", "", genre) genre = re.sub(r"(?i)^misc$", "", genre) genre = re.sub(r"(?i)^(sci[- ]?fi|sf)$", "Science Fiction", genre) genre = re.sub(r"(?i)^\(?science fiction\)?$", "Science Fiction", genre) genre = re.sub(r"Thrillers", "Thriller", genre) genre = re.sub(r"(?i)\bchildren'?s audiobooks?\b", "", genre) genre = re.sub(r"^-\s*|\s*-$", "", genre) genre = re.sub(r"(?i)Engelske\s*\sLydbånd", "", genre) genre = re.sub(r"^FIC028000.*$", "", genre) genre = re.sub(r"^800.*$", "", genre) genre = re.sub(r"^Fantastic Beasts.*$", "", genre) genre = re.sub(r"^and thei.*$", "", genre) genre = re.sub(r"^those you.*$", "", genre) # Normalize separators genre = re.sub(r"\s*[,:/]\s*", ";", genre) genre = re.sub(r"\s*;\s*", "; ", genre) # Trim punctuation / whitespace genre = re.sub(r"^[\s,;:\-]+", "", genre) genre = re.sub(r"[\s,;\-]+$", "", genre) genre = genre.strip() genre = genre.lstrip() if genre != genre_old: logger.debug(f"Update genre from {genre_old} to {genre}") return genre def ffmetadata_escape(value: str) -> str: return ( value.replace("\\", "\\\\") .replace("=", "\\=") .replace(";", "\\;") .replace("#", "\\#") .replace("\n", "\\\n") ) def get_chapters(media: Path) -> list[dict]: result = subprocess.run( [ "ffprobe", "-v", "error", "-show_chapters", "-of", "json", str(media), ], capture_output=True, text=True, check=True, ) data = json.loads(result.stdout) return data.get("chapters", []) def _to_str(value) -> str: if value is None: return "" if hasattr(value, "text"): value = value.text if isinstance(value, (list, tuple)): return "; ".join(str(v) for v in value) return str(value) def get_metadata_all(file: Path) -> dict[str, str]: metadata = dict.fromkeys(metadata_fields, "") try: audio = File(file) if audio is None or audio.tags is None: return metadata # MP4/M4A/M4B if isinstance(audio.tags, MP4Tags): for field, tag in MP4_MAP.items(): value = audio.tags.get(tag) if value: metadata[field] = _to_str(value) # ID3 (MP3 or mislabeled .m4b) elif isinstance(audio.tags, ID3): for field, frame_id in ID3_MAP.items(): frame = audio.tags.get(frame_id) if frame: metadata[field] = _to_str(frame) comments = audio.tags.getall("COMM") if comments: metadata["comment"] = _to_str(comments[0]) # Optional fallback if not metadata["description"]: metadata["description"] = metadata["comment"] # FLAC / Opus / Vorbis else: for field in metadata_fields: value = audio.tags.get(field) if value: metadata[field] = _to_str(value) except Exception: logger.exception(f"Error getting metadata for {file}") return metadata def write_metadata(file, fields, values, dry_run=False): changes = { field.lower().replace(" ", ""): str(value) for field, value in zip(fields, values) } logger.debug(f"Writing metadata for {file}: {changes}") if dry_run: return try: suffix = file.suffix.lower() # MP3 / ID3 if suffix == ".mp3": tags = ID3(file) for field, value in changes.items(): if field == "comment": tags.delall("COMM") if value: tags.add( COMM( encoding=3, lang="eng", desc="", text=[value], ) ) continue frame_id = ID3_MAP.get(field) if not frame_id: continue tags.delall(frame_id) if value: frame_cls = ID3_FRAME_CLASSES[frame_id] tags.add(frame_cls(encoding=3, text=[value])) tags.save(file) # M4A / M4B / MP4 elif suffix in {".m4a", ".m4b", ".mp4"}: tags = MP4(file) for field, value in changes.items(): mp4_key = MP4_MAP.get(field) if not mp4_key: logger.debug(f"Unsupported MP4 tag: {field}") continue if value: tags[mp4_key] = [value] else: tags.pop(mp4_key, None) tags.save() # Opus elif suffix == ".opus": tags = OggOpus(file) for field, value in changes.items(): if value: tags[field] = [value] else: tags.pop(field, None) tags.save() else: logger.warning(f"Unsupported file type: {file}") except Exception: logger.exception(f"Error writing metadata for {file}") def get_duration_ms(media: Path) -> int: duration = subprocess.check_output( [ "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", str(media), ], text=True, ).strip() return int(float(duration) * 1000) def normalize_name(name: str, series: str) -> str: # Remove anything in (), [], {} name_out = "" # name = re.sub( # rf"^(?:{re.escape(series)}){SEP}|{SEP}(?:{re.escape(series)})$", # "", name, flags=re.IGNORECASE, ).strip() name = re.sub(r"\s*[(\[{].*?[)\]}]\s*", " ", name).strip() name = re.sub(r"\s+", " ", name) for pattern in series_patterns: match = re.match(pattern, name, re.IGNORECASE) if match: if pattern.startswith(r"^(\d+)"): name_out = match.group(2).strip() else: name_out = match.group(1).strip() break else: name_out = name pattern = rf"^(.+?){SEP}(?:Book|Band|Volume)\s+(\d+){SEP}(.+)$" match = re.match(pattern, name, re.IGNORECASE) if match: name_out1 = match.group(1).strip() name_out3 = match.group(3).strip() if score_match_jaccard(words(name_out1), words(series)) > score_match_jaccard( words(name_out3), words(series) ): return name_out3 else: return name_out1 return name_out def cleanup(text: str) -> str: # Collapse repeated punctuation groups like ", ,", ". .", "! !", "? ?" while True: new_text = re.sub(r"([.,!?;:-])(?:\s+\1)+", r"\1", text) if new_text == text: break text = new_text # Optional: normalize multiple spaces text = re.sub(r"\s{2,}", " ", text) return text.strip() def get_empty_keys(data: dict) -> str: empty_keys = [str(key) for key, value in data.items() if not value] return f"Empty: {', '.join(empty_keys)}" if empty_keys else "" def get_codec_bitrate(file: Path | None) -> tuple[str, str]: if not file: raise ValueError info = json.loads( subprocess.check_output( [ "ffprobe", "-v", "error", "-select_streams", "a:0", "-show_entries", "stream=codec_name,bit_rate", "-of", "json", str(file), ], text=True, ) ) stream = info["streams"][0] bitrate = int(stream.get("bit_rate") or 128000) bitrate_k = f"{bitrate // 1000}k" return stream["codec_name"].lower(), bitrate_k # codec, bits/s class Audiobook: def __init__(self, book_path: Path, library_root_path: Path) -> None: self.action_required = False self.changed = None # Metadata self.metadata = {} self.metadata_original = {} self.artist = "" self.genre = "" self.composer = "" self.name = "" self.comment = "" self.description = "" self.series_and_num = "" self.series = "" self.series_num = "" self.title: str = "" self.title_list: list = [] # File Paths self.files: list[Path] = [] self.files_extra = [] self.bookPath = book_path self.library_root_path = library_root_path self.firstFile: Path | None = None self.pdf_file = None self.epub_file = None # Files self.singleFile = False self.fileConunt = 0 self.size = 0 def init_book(self) -> None: self.files = sorted( ( file for file in self.bookPath.iterdir() if file.is_file() and file.suffix.lower() in AUDIO_SUFFIXES ), key=self.extract_track_number, ) epubs = list(self.bookPath.glob("*.epub")) self.epub_file = epubs[0] if len(epubs) == 1 else None pdfs = list(self.bookPath.glob("*.pdf")) self.pdf_file = pdfs[0] if len(pdfs) == 1 else None self.files_extra = [ file for file in self.bookPath.glob("*") if file.suffix.lower() not in AUDIO_SUFFIXES ] self.firstFile = self.get_first_file() self.size = sum( file.stat().st_size for pattern in AUDIO_RE_SUFFIXES for file in self.bookPath.glob(pattern) ) self.fileConunt = len(self.files) self.singleFile = self.fileConunt == 1 # Metadata self.load_metadata() logger.info( f"Found {self.bookPath.name}, {self.fileConunt} files, {get_empty_keys(self.metadata)}" ) self.changed = False def __repr__(self): return ( f"Audiobook(" f"name={self.name!r}, " f"artist={self.artist!r}, " f"composer={self.composer!r}, " f"path={str(self.bookPath)!r})," f"size={self.size!r}, " f"fileConunt={self.fileConunt!r}, " ) def get_first_file(self) -> Path: return self.files[0] def write_fields_from_metadata(self) -> None: self.name = self.metadata["album"] self.title = self.metadata["title"] self.composer = self.metadata["composer"] self.comment = self.metadata["comment"] self.genre = self.metadata["genre"] if "#" in self.metadata["grouping"]: self.series, self.series_num = self.metadata["grouping"].rsplit("#", 1) self.series = self.series.strip() self.series_and_num = self.metadata["grouping"] if self.metadata.get("artist"): self.artist = self.metadata["artist"].strip() elif self.metadata.get("albumartist"): self.artist = self.metadata.get("albumartist") self.description = self.metadata["description"] def write_metadata_from_fields(self): self.metadata["album"] = self.name self.metadata["title"] = self.title self.metadata["composer"] = self.composer self.metadata["genre"] = self.genre self.metadata["grouping"] = self.series_and_num self.metadata["comment"] = self.comment self.metadata["description"] = self.description if self.artist: self.metadata["artist"] = self.artist self.metadata["albumartist"] = self.artist def extract_track_number(self, file: Path) -> int: stem = file.stem title = self.name # Remove title if it appears at the beginning if title: escaped_title = re.escape(title) stem = re.sub( rf"^{escaped_title}(?:\s*[-_:]\s*)?", "", stem, flags=re.IGNORECASE, ).strip() patterns = [ # 0099_Apollo, 01 Title, 001 Chapter, 02GLCK~1.WMA r"^(\d+)(?=\D|$)", # CD1-01, CD 1 - 01, CD1 - 01. r"CD\s*(\d+)\s*-\s*(\d{1,3})", # 018-01, 018 - 01, 138-01, 0001-0999 r"^(\d+)\s*-\s*(\d+)(?:\D|$)", # Track 01 r"Track\s+(\d+)$", # 01 - Kapitel 01 r"^(\d+)\s*-", # Rubinrot 101, 102, 201, 202 r".*?(\d)(\d{2})$", # A Thousand Ships - 007 r".*?-\s*(\d+)\s*$", ] for pattern in patterns: m = re.search(pattern, stem, re.IGNORECASE) if not m: continue groups = m.groups() if len(groups) == 1: return int(groups[0]) if len(groups) == 2 and "CD" in pattern: disc = int(groups[0]) track = int(groups[1]) return (disc - 1) * 1000 + track if len(groups) == 2 and stem.startswith(groups[0]): disc = int(groups[0]) track = int(groups[1]) if track < 100: return disc * 1000 + track if len(groups) == 2 and len(groups[0]) == 1: disc = int(groups[0]) track = int(groups[1]) return disc * 1000 + track return int(groups[-1]) return 0 def load_metadata(self): self.metadata = get_metadata_all(self.get_first_file()) self.metadata_original = self.metadata.copy() self.write_fields_from_metadata() logger.trace(f"Load metadata {self.metadata}") def fix_artist_dir(self): rel_path = self.bookPath.relative_to(self.library_root_path).parts if not self.artist: logger.trace(f"No artist tag for {self.name}") return if len(rel_path) < 2: desired_path, desired_name = self.generate_dir_from_metadata() self.move(desired_path, desired_name) logger.debug(f"Artist not visible in folder structure: {rel_path}") def fix_artist_from_comment(self): if not isinstance(self.comment, str): logger.trace(f"No comment for {self.name}") return m = re.search( r"(?:" r"author(?:s)?:?|" r"writer(?:s)?:?|" r"co[- ]?author(?:s)?:?|" r"co[- ]?writer(?:s)?:?|" r"creator(?:s)?:?|" r"contributor(?:s)?:?|" r"composer(?:s)?:?|" r"lyricist(?:s)?:?|" r"songwriter(?:s)?:?|" r"written\s+by:?|" r"story\s+by:?|" r"autorin(?:nen)?:?|" r"autor\(en\):?|" r"autor(?:en)?:?|" r"text(?:e)?\s+von:?|" r"geschrieben\s+von:?|" r"verfasst\s+von:?|" r"auteur(?:s)?:?|" r"autore(?:i)?:?|" r"autor(?:es)?:?|" r"escritor(?:es|as)?:?" r")\s*:?\s*([^\n.,;]+)", self.comment, re.IGNORECASE, ) if not m: logger.trace(f"No artist found in comment for {self.bookPath}") return if len(m.group(1).strip()) < 5: logger.debug(f"Match group {m.group(1)} found but discarded") return logger.trace(f"Match groups: {m.group()}, Book: {self.name}") self.changed = True artist = re.sub(r"\s*\([^)]*\)", "", m.group(1).strip()).strip() logger.trace(f"Remove {artist} from comment: {self.comment}") self.comment = (self.comment[: m.start()] + self.comment[m.end() :]).strip( " .,;\n" ) self.comment = cleanup(self.comment) if not self.artist: self.artist = artist self.changed = True elif self.artist != artist: logger.warning( f"{red_tag('Action Required')} artist {artist} found not matching {self.artist} for {self.bookPath}" ) self.action_required = True else: logger.debug(f"Artist {artist} matches file metadata for {self.name}") def fix_artist_from_path(self, force_artist_from_path): rel_path = self.bookPath.relative_to(self.library_root_path).parts if len(rel_path) < 2: logger.trace(f"Artist not visible in folder structure: {rel_path}") return if self.artist == rel_path[0]: return if not self.artist or force_artist_from_path: self.changed = True self.artist = rel_path[0] logger.info( f"[Updated] artist {self.artist} found in path for {self.bookPath}" ) def fix_composer_from_narrator(self): if not self.metadata.get("narrator"): logger.trace(f"No Narrator found in metadata for {self.name}") return if not self.composer or len(self.composer) < 5: self.composer = self.metadata["narrator"] def fix_composer_from_comment(self): m = re.search( r"(?:read\s+by:?|" r"spoken\s+by:?|" r"erzähler:?|" r"narrated\s+by:?|" r"gesprochen\s+von:?|" r"gelesen\s+von)\s+([^\n.,;]+)", self.comment, re.IGNORECASE, ) if not m: logger.trace(f"No composer found in comment for {self.name}") return self.changed = True self.comment = (self.comment[: m.start()] + self.comment[m.end() :]).strip( " .,;\n" ) self.comment = cleanup(self.comment) if not self.composer or len(self.composer) < 5: logger.info(f"[Updated] composer found in comment for {self.name}") self.composer = str(m.group(1).strip()) def fix_composer_from_path(self): rel_path = self.bookPath.relative_to(self.library_root_path) m = re.search( r"(?:read\s+by:?|" r"spoken\s+by:?|" r"narrated\s+by:?|" r"gesprochen\s+von:?|" r"gelesen\s+von)\s+([^\n.,;]+)", str(rel_path), re.IGNORECASE, ) if not m: logger.trace(f"No composer found in path for {self.bookPath}") return if not self.composer or len(self.composer) < 5: self.changed = True logger.info(f"[Updated] composer found in Path for {self.bookPath}") self.composer = str(m.group(1).strip()) def fix_name(self): if not self.name: self.name = normalize_name(self.bookPath.name, self.series) self.changed = True logger.info(f"[Updated] name {self.name} found in path for {self.bookPath}") return name = normalize_name(self.name, self.series) if name != self.name: logger.info(f"[Updated] name by cleaning {self.name} to {name}") self.name = name self.changed = True def fix_series(self): if self.series_and_num: logger.trace(f"{self.name} already has series: {self.series_and_num}") rel_path = self.bookPath.relative_to(self.library_root_path).parts number = "" if len(rel_path) <= 2: logger.trace(f"No series found for {self.name}") return for i, pattern in enumerate(series_patterns): match = re.match(pattern, rel_path[2], re.IGNORECASE) if not match: continue if i == 0: number = match.group(1) elif i == 1: number = match.group(2) else: number = match.group(2) or match.group(3) break number_str = "" and_number = "" if not number: logger.trace(f"No series number found for {self.name}") else: number_str = " #" + number and_number = " and number" self.series_num = number if self.series_and_num != rel_path[1] + number_str: self.series_and_num = rel_path[1] + number_str self.series = rel_path[1] self.changed = True logger.info( f"[Updated] series{and_number} found for {self.name} to {self.series_and_num}" ) return def fix_genre(self): self.genre = clean_genre(self.genre) self.changed = True def move(self, desired_path: Path, desired_name_stem: str, dry_run=False): logger.trace(f"Moving {self.bookPath} to {desired_path}") desired_path.mkdir(parents=True, exist_ok=True) padding = len(str(len(self.files))) once = True for number, file in enumerate(self.files, start=1): if self.singleFile: new_name = desired_path / f"{desired_name_stem}{file.suffix.lower()}" else: new_name = ( desired_path / f"{number:0{padding}d} - {desired_name_stem}{file.suffix}" ) if file == new_name: continue if once: once = False logger.debug(f"Moving {file} to {new_name}") else: logger.trace(f"Moving {file} to {new_name}") if dry_run: continue self.files[number - 1] = file.rename(new_name) for file in self.files_extra: if file.suffix.lower() in {".cue", ".log"} or file in { self.epub_file, self.pdf_file, }: filename = f"{desired_name_stem}{file.suffix.lower()}" else: filename = file.name new_file = desired_path / filename if file == new_file: continue logger.trace(f"Moving {file} to {new_file}") if dry_run: continue new_file = file.rename(new_file) if file == self.epub_file: self.epub_file = new_file elif file == self.pdf_file: self.pdf_file = new_file if self.bookPath != desired_path: logger.debug(f"Moving {self.bookPath} to {desired_path}") remaining = list(self.bookPath.iterdir()) if not remaining and not dry_run: self.bookPath.rmdir() else: logger.warning( f"{red_tag('Action Required')} old path not empty after move: " f"{self.bookPath}: {remaining}" ) self.bookPath = desired_path self.files = sorted(self.files, key=self.extract_track_number) self.firstFile = self.get_first_file() def generate_dir_from_metadata(self): composer = "" name = self.name if self.composer: composer = f" (by {self.composer})" name = self.name + composer artist = "" if self.artist: artist = re.split(r"[,;]", self.artist)[0].strip().replace("/", "_") desired_path = self.library_root_path / artist / name if self.series_and_num: series = self.series_and_num number = "" if "#" in self.series_and_num: series = self.series_and_num.split("#") number = f"{series[1]} - " series = series[0].rstrip() desired_path = ( self.library_root_path / artist / series.replace("/", "_") / f"{number}{name}".replace("/", "_") ) desired_name_stem = ( f"{self.artist} - {self.name}{composer}{self.bookPath.suffix}" ) desired_name_stem = desired_name_stem.replace("/", "_") return desired_path, desired_name_stem def create_m4b_from_book(self, creat_m4b_no_delete: bool = False) -> None: output_file_final = self.bookPath / f"{self.bookPath.name}.m4b" if self.action_required: logger.warning("Action required, resolve before creating m4b file") return if output_file_final.exists(): logger.info(f"{red_tag('SKIP')} {output_file_final}") logger.info("Book already present") return logger.info(f"{blue_tag('create')} {output_file_final}") with tempfile.TemporaryDirectory(dir=self.library_root_path.parent) as tmp: tmpdir = Path(tmp) output_file_tmp = tmpdir / f"{self.bookPath.name}.m4b" concat_file = tmpdir / "concat.txt" metadata_file = tmpdir / "chapters.txt" # Inspect all inputs before deciding how to create chapters. file_info = [] has_existing_chapters = False for media in self.files: chapters = get_chapters(media) duration_ms = get_duration_ms(media) file_info.append( { "media": media, "chapters": chapters, "duration_ms": duration_ms, } ) if chapters: has_existing_chapters = True if has_existing_chapters: logger.debug( "Existing chapter tracks found; preserving chapter metadata" ) else: logger.debug( "No existing chapters found; generating chapters from filenames" ) current_ms = 0 with ( concat_file.open("w", encoding="utf-8") as cf, metadata_file.open("w", encoding="utf-8") as mf, ): mf.write(";FFMETADATA1\n") for key, value in self.metadata.items(): if key == "title": mf.write( f"{ffmetadata_escape(key)}={ffmetadata_escape(self.metadata['album'])}\n" ) else: mf.write( f"{ffmetadata_escape(key)}={ffmetadata_escape(value)}\n" ) for info in file_info: media = info["media"] media_name = info["media"].name # Remove title media_name = media_name.replace(self.name, "") # Clean separators and whitespace media_name = re.sub(r"[\s._-]+", " ", media_name).strip() # Remove leading/trailing separators again if needed media_name = re.sub(r"^[\s._-]+|[\s._-]+$", "", media_name) info["media"].with_name(media_name) chapters = info["chapters"] duration_ms = info["duration_ms"] cf.write(f"file '{ffmpeg_escape(media)}'\n") if has_existing_chapters: # Preserve actual chapter boundaries and titles. for chapter in chapters: start_ms = round(float(chapter["start_time"]) * 1000) end_ms = round(float(chapter["end_time"]) * 1000) tags = chapter.get("tags", {}) title = tags.get("title", media.stem) mf.write("[CHAPTER]\n") mf.write("TIMEBASE=1/1000\n") mf.write(f"START={current_ms + start_ms}\n") mf.write(f"END={current_ms + end_ms}\n") mf.write(f"title={ffmetadata_escape(title)}\n") else: # No input contains chapters: one chapter per file. mf.write("[CHAPTER]\n") mf.write("TIMEBASE=1/1000\n") mf.write(f"START={current_ms}\n") mf.write(f"END={current_ms + duration_ms}\n") mf.write(f"title={ffmetadata_escape(media.stem)}\n") current_ms += duration_ms codec, bitrate_k = get_codec_bitrate(self.firstFile) codec_args = ["-c:a", "copy"] if codec == "mp3": codec_args = ["-c:a", "aac", "-b:a", bitrate_k] elif codec == "flac": codec_args = ["-c:a", "alac", "-b:a", bitrate_k] elif codec not in {"aac", "alac"}: codec_args = ["-c:a", "aac", "-b:a", bitrate_k] cmd = [ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(concat_file), "-i", str(metadata_file), "-i", str(self.firstFile), # audio from concat "-map", "0:a", # cover image from first media "-map", "2:v?", # Metadata and chapters from our generated FFMETADATA file "-map_metadata", "1", "-map_chapters", "1", *codec_args, "-c:v", "copy", "-disposition:v:0", "attached_pic", str(output_file_tmp), ] result = subprocess.run( cmd, capture_output=True, text=True, check=True, ) logger.trace(result.stdout) if result.returncode == 0: output_file_tmp.rename(output_file_final) for media in self.files: if creat_m4b_no_delete: continue media.unlink() self.files = [output_file_final] self.firstFile = output_file_final self.fileConunt = 1 self.singleFile = True self.changed = False self.size = Path(output_file_final).stat().st_size logger.info( f"{green_tag('SUCCESS')} - {self.bookPath.name} - delete media files" ) else: logger.error( f"{red_tag('ERROR')} - {self.bookPath.name} - Code: {result.returncode}, Stderr: {result.stderr}" ) def write_metadata(self, keys=None, dry_run=False): if self.action_required: logger.warning("Action required, resolve before writing metadata file") return self.write_metadata_from_fields() if keys is None: keys = [] for key, value in self.metadata.items(): if self.metadata_original.get(key) != value and not isinstance( value, list ): keys.append(key) self.metadata_original = self.metadata.copy() values = [self.metadata[key] for key in keys] if (not keys or not values) and self.metadata_original.get( "title" ) != self.title: logger.trace(f"Nothing to write to file for {self.name}") return if keys: for file in self.files: write_metadata(file, keys, values, dry_run) if len(self.title_list) == len(self.files): for title, file in zip(self.title_list, self.files): write_metadata(file, ["title"], [title], dry_run) self.changed = False def fix_title(self): if not self.title: if self.singleFile: self.title = self.name else: padding = len(str(len(self.files))) for idx, file in enumerate(self.files): self.title_list.append(f"{self.name} - {idx + 1:0{padding}d}") class AudiobookLibrary: def __init__( self, library_path: Path, api_url: str, api_key: str, sql_database_path: Path, dry_run: bool, path_filter: str = "", logfilename: str = "audiobook_library.log", ): self.path_filter = path_filter self.sql_database_instance = self.SqlDatabase(sql_database_path) self.api_instance = self.AudiobookshelfApi(api_url,api_key) self.dry_run = dry_run logging.setLoggerClass(CustomLogger) logger: CustomLogger = logging.getLogger(__name__) # type: ignore[assignment] logger.info("Started") self.libraryPath: Path = Path(library_path) self.audiobooks: list[Audiobook] = [] self._by_path = {} self._by_name = defaultdict(list) self._by_artist = defaultdict(list) self._by_composer = defaultdict(list) self.load_audiobooks_from_dir() self.size = sum( audiobook.size for audiobook in self.audiobooks ) # size in bytes class SqlDatabase: def __init__(self, sql_database_path: Path): self.tables = [] self.data = {} self.db = None self.path = sql_database_path self.books = [] def connect(self): if not self.path.is_file(): logger.error(f"{self.path} does not exist") raise FileExistsError self.db = sqlite3.connect(self.path) self.db.row_factory = sqlite3.Row return True def cursor(self): if not self.db: raise ValueError return self.db.cursor() def commit(self): if not self.db: raise ValueError self.db.commit() def close(self): if not self.db: raise ValueError self.db.close() def init(self): if not self.connect(): return False logger.info("Database initialized successfully") return self.load_database_data() def load_database_data(self) -> bool: logger.info("Load Database data") if not self.db: raise ValueError self.tables = self.db.execute( "SELECT name FROM sqlite_master WHERE type='table'" ).fetchall() for table in self.tables: table_name = table["name"] rows = [] for row in self.db.execute(f'SELECT * FROM "{table_name}"'): record = dict(row) # Auto-decode JSON columns for key, value in record.items(): if isinstance(value, str) and value[:1] in "[{": try: record[key] = json.loads(value) except json.JSONDecodeError: pass rows.append(record) self.data[table_name] = rows self.books = self.data["books"] return True def update_books(self): for book in self.books: self._save_book(book) def _save_book(self, book): book = book.copy() for key, value in book.items(): if isinstance(value, (list, dict)): book[key] = json.dumps(value, ensure_ascii=False) columns = [c for c in book if c != "id"] sql = f""" UPDATE books SET {", ".join(f'"{c}" = ?' for c in columns)} WHERE id = ? """ values = [book[c] for c in columns] values.append(book["id"]) if not self.db: raise ValueError self.db.execute(sql, values) # commit once after all updates def fix_genres(self): logger.info("Fix genres in database") for book in self.books: genres = book.get("genres") if not genres or not any(g.strip() for g in genres): continue new_genres = [] for genre in genres: genre = clean_genre(genre) if "," in genre: genre = genre.split(",") elif ";" in genre: genre = genre.split(";") elif " - " in genre: genre = genre.split(" - ") elif "Historical Adventure Thriller Mystery" in genre: genre = genre.split(" ") if isinstance(genre, str): if len(genre) > 50 or (5 > len(genre) > 0): genre = "" logger.trace(f"Too long or too short: {genre}") genre = genre.strip() genre = genre.lstrip() if genre: new_genres.append(genre) elif isinstance(genre, list): genre_buf = genre if len(genre) > 1: genre_buf = [g.strip() for g in genre if g] new_genres.extend(genre_buf) new_genres = sorted(set(new_genres)) logger.trace(f"{new_genres}") list_difference = False if len(new_genres) < len(genres): list_difference = True for old, new in zip(genres, new_genres): if old != new: list_difference = True break if not list_difference: logger.trace("No difference: Dont Commit to Database") continue logger.debug(f"{genres} -> {new_genres}") book["genres"] = new_genres def fix_comments(self): logger.info("Fix comments in database") for book in self.books: for file in book.get("audioFiles"): comment = file.get("metaTags").get("tagComment") if comment and len(comment) < 40: logger.debug(f"Fix short comment for {book.get('title')}") comment = "" file["metaTags"]["tagComment"] = comment class AudiobookshelfApi: def __init__(self, url: str, api_key: str, timeout: int = 30): self.url = url.rstrip("/") self.timeout = timeout self.session = requests.Session() self.session.headers["Authorization"] = f"Bearer {api_key}" self.books = [] def _request(self, method: str, endpoint: str, **kwargs) -> Any: resp = self.session.request( method, f"{self.url}{endpoint}", timeout=self.timeout, **kwargs, ) try: resp.raise_for_status() except requests.exceptions.HTTPError: logger.error(f"{method} {endpoint}") logger.error(resp) if not resp.content or resp.status_code != 200: logger.error(f"Request failed: {resp}") return None # No content if not resp.content: return None # Only parse JSON if response claims to be JSON content_type = resp.headers.get("Content-Type", "") if "application/json" in content_type: return resp.json() # Return raw text otherwise return resp.text def get(self, endpoint: str, **kwargs): return self._request("GET", endpoint, params=kwargs) def post(self, endpoint: str, data: dict, **kwargs): return self._request("POST", endpoint, json=data, params=kwargs) def put(self, endpoint: str, data: dict): return self._request("PUT", endpoint, json=data) def delete(self, endpoint: str): return self._request("DELETE", endpoint) def load_books(self) -> list[dict]: """Load all books from all libraries into self.books.""" self.books = [] libraries = self.get("/api/libraries")["libraries"] for library in libraries: library_id = library["id"] response = self.get( f"/api/libraries/{library_id}/items", minified="0" ) self.books.extend(response["results"]) return self.books def embed_metadata_for_book(self, item_id: str) -> None: book = next((b for b in self.books if b["id"] == item_id), None) if book: title = ( book.get("media", {}) .get("metadata", {}) .get("title", item_id) ) if book.get("media", {}).get("numAudioFiles", 0) > 0: logger.info(f"Embedding metadata: {title}") self.post( f"/api/tools/item/{item_id}/embed-metadata", data={}, backup=False, ) else: logger.info(f"{title} is an epub only book") else: logger.info(f"Book not found: {item_id}") def embed_metadata_for_all_books(self, args) -> None: for book in self.books: item_id = book["id"] book_name = ( book.get("media", {}) .get("metadata", {}) .get("title", item_id) ) book_path = book.get("path", "") logger.info(f"Embedding metadata: {book_name} ({book_path})") if args.dry_run: continue try: self.embed_metadata_for_book(item_id) except Exception as e: logger.error(f"Failed: {book_name}: {e}") def embed_metadata_from_file(self, filename: str) -> None: filename = Path(filename).expanduser().resolve() with open(filename, encoding="utf-8") as f: item_ids = [ line.strip() for line in f if line.strip() and not line.startswith("#") ] logger.info(f"Loaded {len(item_ids)} item IDs from {filename}") for item_id in item_ids: book = next( (b for b in self.books if b["id"] == item_id), None ) try: self.embed_metadata_for_book(item_id) except Exception as e: logger.error(f"Failed: {item_id}: {e}") def add(self, book: Audiobook): self.audiobooks.append(book) self._by_path[book.bookPath] = book self._by_name[book.name].append(book) if book.artist: self._by_artist[book.artist].append(book) if book.composer: self._by_composer[book.composer].append(book) def print_name(self, books): if not isinstance(books, list): books = [books] for book in books: if book.name: logger.info(book.name) else: logger.info(f"No name in {book.bookPath.name}") def get_by_path(self, path): return self._by_path.get(Path(path)) def get_by_name(self, name): return self._by_name.get(name, []) def get_by_artist(self, artist): return self._by_artist.get(artist, []) def get_by_composer(self, composer): return self._by_artist.get(composer, []) def find_path_contains(self, path) -> list[Audiobook]: return [ book for book in self.audiobooks if book.bookPath == path or book.bookPath.is_relative_to(path) ] def find_name_contains(self, text): text = text.lower() return [book for book in self.audiobooks if text in book.name.lower()] def find_artist_contains(self, text): text = text.lower() return [ book for book in self.audiobooks if book.artist and text in book.artist.lower() ] def load_audiobooks_from_dir(self): """ A book directory is any directory containing more than one audiobook file (.mp3, .m4a, .m4b) """ media_files = [ file for file in self.libraryPath.rglob("*") if file.is_file() and file.suffix.lower() in AUDIO_SUFFIXES and self.path_filter in str(file) ] dirs = {file.parent for file in media_files} for directory in dirs: self.add(Audiobook(directory, self.libraryPath)) for audiobook in self.audiobooks: audiobook.init_book() def load_metadata(self): for audiobook in self.audiobooks: audiobook.load_metadata() def write_metadata(self, keys=None): for audiobook in self.audiobooks: if audiobook.changed: logger.trace(f"Writing metadata: {audiobook.name}") audiobook.write_metadata(keys, dry_run=self.dry_run) def fix_artist_dir(self): for book in self.audiobooks: book.fix_artist_dir() def create_structured_directories(self, dry_run=False): for book in self.audiobooks: desired_path, desired_name = book.generate_dir_from_metadata() book.move(desired_path, desired_name, dry_run) def fix_artists(self, force_artist_from_path=False): for book in self.audiobooks: book.fix_artist_from_comment() book.fix_artist_from_path(force_artist_from_path) def fix_composers(self): for book in self.audiobooks: book.fix_composer_from_narrator() book.fix_composer_from_comment() book.fix_composer_from_path() def fix_names(self): for book in self.audiobooks: book.fix_name() def fix_comments(self): for book in self.audiobooks: if book.comment and len(book.comment) < 40: logger.info(f"Remove short comment of {book.name}: {book.comment}") book.comment = "" book.changed = True def fix_genres(self): for book in self.audiobooks: book.fix_genre() def fix_titles(self): for book in self.audiobooks: book.fix_title() def fix_series(self): for book in self.audiobooks: book.fix_series() def create_m4b_from_dir(self, keep_originals: bool = False) -> None: audiobooks = self.audiobooks jobs = [audiobook for audiobook in audiobooks if not audiobook.singleFile] for audiobook in jobs: logger.debug(f"Create m4b file for {audiobook.name}") if not self.dry_run: with ThreadPoolExecutor(max_workers=8) as executor: futures = { executor.submit( audiobook.create_m4b_from_book, keep_originals, ): audiobook for audiobook in jobs } for future in as_completed(futures): audiobook = futures[future] try: future.result() except Exception: logger.exception(f"Failed to create m4b for {audiobook.name}") def main(): parser = argparse.ArgumentParser(description="Audiobook Directory Processing") parser.add_argument("library", type=Path, help="Library path") parser_add_loglevel(parser) m4b_group = parser.add_argument_group("M4B creation") m4b_group.add_argument( "-cm", "--create-m4b", action="store_true", help="Create M4B files from multi-file audiobooks", ) m4b_group.add_argument( "--keep-originals", action="store_true", help="Do not delete source files after M4B creation", ) m4b_group.add_argument( "--name-filter", metavar="NAME", default="", help="Process only books whose name contains NAME", ) m4b_group.add_argument( "--path-filter", metavar="PATH", default="", help="Process only books whose path contains PATH", ) parser.add_argument( "--dry-run", action="store_true", help="Dry run only", ) parser.add_argument( "--sql-database", type=Path, default=Path("/docker/audiobookshelf/config/absdatabase.sqlite"), help="SQL Database Path", ) parser.add_argument( "--api-url", type=str, default="https://audiobooks.someoneslinux.selfhost.eu", help="API Url for Audiobookshelf API", ) parser.add_argument( "--api-key", type=str, default="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJrZXlJZCI6Ijc3NTc5NDE0LTczMGItNDllMy04YTgzLTk3M2Y1MjRiOGVlNyIsIm5hbWUiOiJBdWRpb2Jvb2tfbGlicmFyeS5weSIsInR5cGUiOiJhcGkiLCJpYXQiOjE3OTM3MDIyODd9.GJVOgYcWM4JpJyMhqzQAHJCfypKiWNBfJHwdM_uvZHU", help="API Key for Audiobookshelf API", ) parser.add_argument( "-em", "--encode-meta", action="store_true", help="Encode Metadata to Audiobook files", ) parser.add_argument( "-fd", "--fix-db", action="store_true", help="Fix the Database", ) parser.add_argument( "-fa", "--force-artist-from-path", action="store_true", help="Force overwrite Artist from Path, only use if Path is correct", ) parser.add_argument( "-m", "--move", action="store_true", help="Create structured directories for all books: /artist/series/book(read by X)/file.m4b", ) actions = [ "artis_dir", "artists", "composers", "series", "names", "genres", "comments", "titles", ] parser.add_argument( "--actions", nargs="+", choices=actions, default=actions, ) args = parser.parse_args() if "--sql-database" in sys.argv: args.fix_db = True setup_logging(args.log_level, "audiobook_library") library_dir = args.library.expanduser() library_dir = library_dir.resolve() if not library_dir.exists(): logger.error(f"Library directory does not exist: {library_dir}") sys.exit(1) database_path = args.sql_database.expanduser() database_path = database_path.resolve() library = AudiobookLibrary( library_dir, args.api_url,args.api_key,database_path, args.dry_run, path_filter=args.path_filter ) if args.encode_meta: library.api_instance.load_books() library.api_instance.embed_metadata_from_file("~/embed_ids.txt") #library.api_instance.embed_metadata_for_all_books(args) if "artist_dir" in args.actions: library.fix_artist_dir() if "artists" in args.actions: library.fix_artists(args.force_artist_from_path) if "composers" in args.actions: library.fix_composers() if "series" in args.actions: library.fix_series() if "names" in args.actions: library.fix_names() if "comments" in args.actions: library.fix_comments() if "genres" in args.actions: library.fix_genres() if "titles" in args.actions: library.fix_titles() if args.fix_db: if not database_path.is_file(): logger.error(f"Database is not a file: {database_path}") return library.sql_database_instance.init() library.sql_database_instance.fix_genres() library.sql_database_instance.fix_comments() if not args.dry_run: library.sql_database_instance.update_books() library.sql_database_instance.commit() library.sql_database_instance.close() if args.create_m4b: library.create_m4b_from_dir( keep_originals=args.keep_originals, ) library.write_metadata() if args.move: library.create_structured_directories(args.dry_run) logger.info("All Operations completed successfully") if __name__ == "__main__": main()