Files
Server-scripts/audiobook_metadata/metadata_audiobooks_library.py
T

1697 lines
55 KiB
Python
Executable File

#!/home/lucas/scripts/.venv/bin/python
import argparse
import json
import logging
import re
import sqlite3
import subprocess
import requests
import sys
from typing import Any
import tempfile
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from mutagen import File
from mutagen.id3 import (
COMM,
ID3,
TALB,
TCOM,
TCON,
TIT1,
TIT2,
TPE1,
TPE2,
)
from mutagen.mp4 import MP4, MP4Tags
from mutagen.oggopus import OggOpus
from serverlibrary import (
AUDIO_RE_SUFFIXES,
AUDIO_SUFFIXES,
CustomLogger,
blue_tag,
green_tag,
red_tag,
score_match_jaccard,
setup_logging,
words,
parser_add_loglevel
)
ID3_FRAME_CLASSES = {
"TIT2": TIT2,
"TALB": TALB,
"TPE1": TPE1,
"TPE2": TPE2,
"TCON": TCON,
"TCOM": TCOM,
"TIT1": TIT1,
}
logging.setLoggerClass(CustomLogger)
logger: CustomLogger = logging.getLogger(__name__) # type: ignore[assignment]
MP4_MAP = {
"artist": "\xa9ART",
"album": "\xa9alb",
"title": "\xa9nam",
"genre": "\xa9gen",
"comment": "\xa9cmt",
"composer": "\xa9wrt",
"albumartist": "aART",
"grouping": "\xa9grp",
"description": "desc",
}
ID3_MAP = {
"artist": "TPE1",
"album": "TALB",
"title": "TIT2",
"genre": "TCON",
"composer": "TCOM",
"albumartist": "TPE2",
"grouping": "TIT1",
}
metadata_fields = [
"artist",
"album",
"composer",
"comment",
"genre",
"grouping",
"albumartist",
"description",
"title",
]
SEP = r"\s*[-_:,.]+\s*"
series_patterns = [
rf"^(.+?){SEP}(?:Book|Band|Volume|Vol)\s+(\d+)(?:{SEP}(.+))$",
rf"^(.+?){SEP}(?:Book|Band|Volume|Vol)\s+(\d+)$",
rf"^(\d+){SEP}(.+)$",
rf"^(.+?){SEP}(\d+)$",
]
def ffmpeg_escape(path: Path) -> str:
return str(path.resolve()).replace("'", r"'\''")
def clean_genre(genre):
if not genre:
return ""
genre_old = genre
# genre = genre.lower()
genre = re.sub(r"(?i)^audio\s*books?$", "", genre)
genre = re.sub(r"(?i)^audiobooks?;?$", "", genre)
genre = re.sub(r"(?i)\b(hörb(uch|ücher)|hoerb(uch|uecher))\b", "", genre)
genre = re.sub(r"(?i)^(hörspiele?|hoerspiele?)$", "", genre)
genre = re.sub(r"(?i)^spoken\s*word$", "", genre)
genre = re.sub(r"(?i)gesprochene\s*inhalte$", "", genre)
genre = re.sub(r"(?i)^speech$", "", genre)
genre = re.sub(r"Fanasie", "Fantasie", genre)
genre = re.sub(r"Mit Videospiel-?", "Videospielbezug", genre)
genre = re.sub(r"(?i)nonfiction", "NonFiction", genre)
genre = re.sub(r"(?i)^speech$", "", genre)
genre = re.sub(r"(?i)^vocal$", "", genre)
genre = re.sub(r"(?i)^retail$", "", genre)
genre = re.sub(r"(?i)^other$", "", genre)
genre = re.sub(r"(?i)^misc$", "", genre)
genre = re.sub(r"(?i)^(sci[- ]?fi|sf)$", "Science Fiction", genre)
genre = re.sub(r"(?i)^\(?science fiction\)?$", "Science Fiction", genre)
genre = re.sub(r"Thrillers", "Thriller", genre)
genre = re.sub(r"(?i)\bchildren'?s audiobooks?\b", "", genre)
genre = re.sub(r"^-\s*|\s*-$", "", genre)
genre = re.sub(r"(?i)Engelske\s*\sLydbånd", "", genre)
genre = re.sub(r"^FIC028000.*$", "", genre)
genre = re.sub(r"^800.*$", "", genre)
genre = re.sub(r"^Fantastic Beasts.*$", "", genre)
genre = re.sub(r"^and thei.*$", "", genre)
genre = re.sub(r"^those you.*$", "", genre)
# Normalize separators
genre = re.sub(r"\s*[,:/]\s*", ";", genre)
genre = re.sub(r"\s*;\s*", "; ", genre)
# Trim punctuation / whitespace
genre = re.sub(r"^[\s,;:\-]+", "", genre)
genre = re.sub(r"[\s,;\-]+$", "", genre)
genre = genre.strip()
genre = genre.lstrip()
if genre != genre_old:
logger.debug(f"Update genre from {genre_old} to {genre}")
return genre
def ffmetadata_escape(value: str) -> str:
return (
value.replace("\\", "\\\\")
.replace("=", "\\=")
.replace(";", "\\;")
.replace("#", "\\#")
.replace("\n", "\\\n")
)
def get_chapters(media: Path) -> list[dict]:
result = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-show_chapters",
"-of",
"json",
str(media),
],
capture_output=True,
text=True,
check=True,
)
data = json.loads(result.stdout)
return data.get("chapters", [])
def _to_str(value) -> str:
if value is None:
return ""
if hasattr(value, "text"):
value = value.text
if isinstance(value, (list, tuple)):
return "; ".join(str(v) for v in value)
return str(value)
def get_metadata_all(file: Path) -> dict[str, str]:
metadata = dict.fromkeys(metadata_fields, "")
try:
audio = File(file)
if audio is None or audio.tags is None:
return metadata
# MP4/M4A/M4B
if isinstance(audio.tags, MP4Tags):
for field, tag in MP4_MAP.items():
value = audio.tags.get(tag)
if value:
metadata[field] = _to_str(value)
# ID3 (MP3 or mislabeled .m4b)
elif isinstance(audio.tags, ID3):
for field, frame_id in ID3_MAP.items():
frame = audio.tags.get(frame_id)
if frame:
metadata[field] = _to_str(frame)
comments = audio.tags.getall("COMM")
if comments:
metadata["comment"] = _to_str(comments[0])
# Optional fallback
if not metadata["description"]:
metadata["description"] = metadata["comment"]
# FLAC / Opus / Vorbis
else:
for field in metadata_fields:
value = audio.tags.get(field)
if value:
metadata[field] = _to_str(value)
except Exception:
logger.exception(f"Error getting metadata for {file}")
return metadata
def write_metadata(file, fields, values, dry_run=False):
changes = {
field.lower().replace(" ", ""): str(value)
for field, value in zip(fields, values)
}
logger.debug(f"Writing metadata for {file}: {changes}")
if dry_run:
return
try:
suffix = file.suffix.lower()
# MP3 / ID3
if suffix == ".mp3":
tags = ID3(file)
for field, value in changes.items():
if field == "comment":
tags.delall("COMM")
if value:
tags.add(
COMM(
encoding=3,
lang="eng",
desc="",
text=[value],
)
)
continue
frame_id = ID3_MAP.get(field)
if not frame_id:
continue
tags.delall(frame_id)
if value:
frame_cls = ID3_FRAME_CLASSES[frame_id]
tags.add(frame_cls(encoding=3, text=[value]))
tags.save(file)
# M4A / M4B / MP4
elif suffix in {".m4a", ".m4b", ".mp4"}:
tags = MP4(file)
for field, value in changes.items():
mp4_key = MP4_MAP.get(field)
if not mp4_key:
logger.debug(f"Unsupported MP4 tag: {field}")
continue
if value:
tags[mp4_key] = [value]
else:
tags.pop(mp4_key, None)
tags.save()
# Opus
elif suffix == ".opus":
tags = OggOpus(file)
for field, value in changes.items():
if value:
tags[field] = [value]
else:
tags.pop(field, None)
tags.save()
else:
logger.warning(f"Unsupported file type: {file}")
except Exception:
logger.exception(f"Error writing metadata for {file}")
def get_duration_ms(media: Path) -> int:
duration = subprocess.check_output(
[
"ffprobe",
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
str(media),
],
text=True,
).strip()
return int(float(duration) * 1000)
def normalize_name(name: str, series: str) -> str:
# Remove anything in (), [], {}
name_out = ""
# name = re.sub(
# rf"^(?:{re.escape(series)}){SEP}|{SEP}(?:{re.escape(series)})$",
# "", name, flags=re.IGNORECASE, ).strip()
name = re.sub(r"\s*[(\[{].*?[)\]}]\s*", " ", name).strip()
name = re.sub(r"\s+", " ", name)
for pattern in series_patterns:
match = re.match(pattern, name, re.IGNORECASE)
if match:
if pattern.startswith(r"^(\d+)"):
name_out = match.group(2).strip()
else:
name_out = match.group(1).strip()
break
else:
name_out = name
pattern = rf"^(.+?){SEP}(?:Book|Band|Volume)\s+(\d+){SEP}(.+)$"
match = re.match(pattern, name, re.IGNORECASE)
if match:
name_out1 = match.group(1).strip()
name_out3 = match.group(3).strip()
if score_match_jaccard(words(name_out1), words(series)) > score_match_jaccard(
words(name_out3), words(series)
):
return name_out3
else:
return name_out1
return name_out
def cleanup(text: str) -> str:
# Collapse repeated punctuation groups like ", ,", ". .", "! !", "? ?"
while True:
new_text = re.sub(r"([.,!?;:-])(?:\s+\1)+", r"\1", text)
if new_text == text:
break
text = new_text
# Optional: normalize multiple spaces
text = re.sub(r"\s{2,}", " ", text)
return text.strip()
def get_empty_keys(data: dict) -> str:
empty_keys = [str(key) for key, value in data.items() if not value]
return f"Empty: {', '.join(empty_keys)}" if empty_keys else ""
def get_codec_bitrate(file: Path | None) -> tuple[str, str]:
if not file:
raise ValueError
info = json.loads(
subprocess.check_output(
[
"ffprobe",
"-v",
"error",
"-select_streams",
"a:0",
"-show_entries",
"stream=codec_name,bit_rate",
"-of",
"json",
str(file),
],
text=True,
)
)
stream = info["streams"][0]
bitrate = int(stream.get("bit_rate") or 128000)
bitrate_k = f"{bitrate // 1000}k"
return stream["codec_name"].lower(), bitrate_k # codec, bits/s
class Audiobook:
def __init__(self, book_path: Path, library_root_path: Path) -> None:
self.action_required = False
self.changed = None
# Metadata
self.metadata = {}
self.metadata_original = {}
self.artist = ""
self.genre = ""
self.composer = ""
self.name = ""
self.comment = ""
self.description = ""
self.series_and_num = ""
self.series = ""
self.series_num = ""
self.title: str = ""
self.title_list: list = []
# File Paths
self.files: list[Path] = []
self.files_extra = []
self.bookPath = book_path
self.library_root_path = library_root_path
self.firstFile: Path | None = None
self.pdf_file = None
self.epub_file = None
# Files
self.singleFile = False
self.fileConunt = 0
self.size = 0
def init_book(self) -> None:
self.files = sorted(
(
file
for file in self.bookPath.iterdir()
if file.is_file() and file.suffix.lower() in AUDIO_SUFFIXES
),
key=self.extract_track_number,
)
epubs = list(self.bookPath.glob("*.epub"))
self.epub_file = epubs[0] if len(epubs) == 1 else None
pdfs = list(self.bookPath.glob("*.pdf"))
self.pdf_file = pdfs[0] if len(pdfs) == 1 else None
self.files_extra = [
file
for file in self.bookPath.glob("*")
if file.suffix.lower() not in AUDIO_SUFFIXES
]
self.firstFile = self.get_first_file()
self.size = sum(
file.stat().st_size
for pattern in AUDIO_RE_SUFFIXES
for file in self.bookPath.glob(pattern)
)
self.fileConunt = len(self.files)
self.singleFile = self.fileConunt == 1
# Metadata
self.load_metadata()
logger.info(
f"Found {self.bookPath.name}, {self.fileConunt} files, {get_empty_keys(self.metadata)}"
)
self.changed = False
def __repr__(self):
return (
f"Audiobook("
f"name={self.name!r}, "
f"artist={self.artist!r}, "
f"composer={self.composer!r}, "
f"path={str(self.bookPath)!r}),"
f"size={self.size!r}, "
f"fileConunt={self.fileConunt!r}, "
)
def get_first_file(self) -> Path:
return self.files[0]
def write_fields_from_metadata(self) -> None:
self.name = self.metadata["album"]
self.title = self.metadata["title"]
self.composer = self.metadata["composer"]
self.comment = self.metadata["comment"]
self.genre = self.metadata["genre"]
if "#" in self.metadata["grouping"]:
self.series, self.series_num = self.metadata["grouping"].rsplit("#", 1)
self.series = self.series.strip()
self.series_and_num = self.metadata["grouping"]
if self.metadata.get("artist"):
self.artist = self.metadata["artist"].strip()
elif self.metadata.get("albumartist"):
self.artist = self.metadata.get("albumartist")
self.description = self.metadata["description"]
def write_metadata_from_fields(self):
self.metadata["album"] = self.name
self.metadata["title"] = self.title
self.metadata["composer"] = self.composer
self.metadata["genre"] = self.genre
self.metadata["grouping"] = self.series_and_num
self.metadata["comment"] = self.comment
self.metadata["description"] = self.description
if self.artist:
self.metadata["artist"] = self.artist
self.metadata["albumartist"] = self.artist
def extract_track_number(self, file: Path) -> int:
stem = file.stem
title = self.name
# Remove title if it appears at the beginning
if title:
escaped_title = re.escape(title)
stem = re.sub(
rf"^{escaped_title}(?:\s*[-_:]\s*)?",
"",
stem,
flags=re.IGNORECASE,
).strip()
patterns = [
# 0099_Apollo, 01 Title, 001 Chapter, 02GLCK~1.WMA
r"^(\d+)(?=\D|$)",
# CD1-01, CD 1 - 01, CD1 - 01.
r"CD\s*(\d+)\s*-\s*(\d{1,3})",
# 018-01, 018 - 01, 138-01, 0001-0999
r"^(\d+)\s*-\s*(\d+)(?:\D|$)",
# Track 01
r"Track\s+(\d+)$",
# 01 - Kapitel 01
r"^(\d+)\s*-",
# Rubinrot 101, 102, 201, 202
r".*?(\d)(\d{2})$",
# A Thousand Ships - 007
r".*?-\s*(\d+)\s*$",
]
for pattern in patterns:
m = re.search(pattern, stem, re.IGNORECASE)
if not m:
continue
groups = m.groups()
if len(groups) == 1:
return int(groups[0])
if len(groups) == 2 and "CD" in pattern:
disc = int(groups[0])
track = int(groups[1])
return (disc - 1) * 1000 + track
if len(groups) == 2 and stem.startswith(groups[0]):
disc = int(groups[0])
track = int(groups[1])
if track < 100:
return disc * 1000 + track
if len(groups) == 2 and len(groups[0]) == 1:
disc = int(groups[0])
track = int(groups[1])
return disc * 1000 + track
return int(groups[-1])
return 0
def load_metadata(self):
self.metadata = get_metadata_all(self.get_first_file())
self.metadata_original = self.metadata.copy()
self.write_fields_from_metadata()
logger.trace(f"Load metadata {self.metadata}")
def fix_artist_dir(self):
rel_path = self.bookPath.relative_to(self.library_root_path).parts
if not self.artist:
logger.trace(f"No artist tag for {self.name}")
return
if len(rel_path) < 2:
desired_path, desired_name = self.generate_dir_from_metadata()
self.move(desired_path, desired_name)
logger.debug(f"Artist not visible in folder structure: {rel_path}")
def fix_artist_from_comment(self):
if not isinstance(self.comment, str):
logger.trace(f"No comment for {self.name}")
return
m = re.search(
r"(?:"
r"author(?:s)?:?|"
r"writer(?:s)?:?|"
r"co[- ]?author(?:s)?:?|"
r"co[- ]?writer(?:s)?:?|"
r"creator(?:s)?:?|"
r"contributor(?:s)?:?|"
r"composer(?:s)?:?|"
r"lyricist(?:s)?:?|"
r"songwriter(?:s)?:?|"
r"written\s+by:?|"
r"story\s+by:?|"
r"autorin(?:nen)?:?|"
r"autor\(en\):?|"
r"autor(?:en)?:?|"
r"text(?:e)?\s+von:?|"
r"geschrieben\s+von:?|"
r"verfasst\s+von:?|"
r"auteur(?:s)?:?|"
r"autore(?:i)?:?|"
r"autor(?:es)?:?|"
r"escritor(?:es|as)?:?"
r")\s*:?\s*([^\n.,;]+)",
self.comment,
re.IGNORECASE,
)
if not m:
logger.trace(f"No artist found in comment for {self.bookPath}")
return
if len(m.group(1).strip()) < 5:
logger.debug(f"Match group {m.group(1)} found but discarded")
return
logger.trace(f"Match groups: {m.group()}, Book: {self.name}")
self.changed = True
artist = re.sub(r"\s*\([^)]*\)", "", m.group(1).strip()).strip()
logger.trace(f"Remove {artist} from comment: {self.comment}")
self.comment = (self.comment[: m.start()] + self.comment[m.end() :]).strip(
" .,;\n"
)
self.comment = cleanup(self.comment)
if not self.artist:
self.artist = artist
self.changed = True
elif self.artist != artist:
logger.warning(
f"{red_tag('Action Required')} artist {artist} found not matching {self.artist} for {self.bookPath}"
)
self.action_required = True
else:
logger.debug(f"Artist {artist} matches file metadata for {self.name}")
def fix_artist_from_path(self, force_artist_from_path):
rel_path = self.bookPath.relative_to(self.library_root_path).parts
if len(rel_path) < 2:
logger.trace(f"Artist not visible in folder structure: {rel_path}")
return
if self.artist == rel_path[0]:
return
if not self.artist or force_artist_from_path:
self.changed = True
self.artist = rel_path[0]
logger.info(
f"[Updated] artist {self.artist} found in path for {self.bookPath}"
)
def fix_composer_from_narrator(self):
if not self.metadata.get("narrator"):
logger.trace(f"No Narrator found in metadata for {self.name}")
return
if not self.composer or len(self.composer) < 5:
self.composer = self.metadata["narrator"]
def fix_composer_from_comment(self):
m = re.search(
r"(?:read\s+by:?|"
r"spoken\s+by:?|"
r"erzähler:?|"
r"narrated\s+by:?|"
r"gesprochen\s+von:?|"
r"gelesen\s+von)\s+([^\n.,;]+)",
self.comment,
re.IGNORECASE,
)
if not m:
logger.trace(f"No composer found in comment for {self.name}")
return
self.changed = True
self.comment = (self.comment[: m.start()] + self.comment[m.end() :]).strip(
" .,;\n"
)
self.comment = cleanup(self.comment)
if not self.composer or len(self.composer) < 5:
logger.info(f"[Updated] composer found in comment for {self.name}")
self.composer = str(m.group(1).strip())
def fix_composer_from_path(self):
rel_path = self.bookPath.relative_to(self.library_root_path)
m = re.search(
r"(?:read\s+by:?|"
r"spoken\s+by:?|"
r"narrated\s+by:?|"
r"gesprochen\s+von:?|"
r"gelesen\s+von)\s+([^\n.,;]+)",
str(rel_path),
re.IGNORECASE,
)
if not m:
logger.trace(f"No composer found in path for {self.bookPath}")
return
if not self.composer or len(self.composer) < 5:
self.changed = True
logger.info(f"[Updated] composer found in Path for {self.bookPath}")
self.composer = str(m.group(1).strip())
def fix_name(self):
if not self.name:
self.name = normalize_name(self.bookPath.name, self.series)
self.changed = True
logger.info(f"[Updated] name {self.name} found in path for {self.bookPath}")
return
name = normalize_name(self.name, self.series)
if name != self.name:
logger.info(f"[Updated] name by cleaning {self.name} to {name}")
self.name = name
self.changed = True
def fix_series(self):
if self.series_and_num:
logger.trace(f"{self.name} already has series: {self.series_and_num}")
rel_path = self.bookPath.relative_to(self.library_root_path).parts
number = ""
if len(rel_path) <= 2:
logger.trace(f"No series found for {self.name}")
return
for i, pattern in enumerate(series_patterns):
match = re.match(pattern, rel_path[2], re.IGNORECASE)
if not match:
continue
if i == 0:
number = match.group(1)
elif i == 1:
number = match.group(2)
else:
number = match.group(2) or match.group(3)
break
number_str = ""
and_number = ""
if not number:
logger.trace(f"No series number found for {self.name}")
else:
number_str = " #" + number
and_number = " and number"
self.series_num = number
if self.series_and_num != rel_path[1] + number_str:
self.series_and_num = rel_path[1] + number_str
self.series = rel_path[1]
self.changed = True
logger.info(
f"[Updated] series{and_number} found for {self.name} to {self.series_and_num}"
)
return
def fix_genre(self):
self.genre = clean_genre(self.genre)
self.changed = True
def move(self, desired_path: Path, desired_name_stem: str, dry_run=False):
logger.trace(f"Moving {self.bookPath} to {desired_path}")
desired_path.mkdir(parents=True, exist_ok=True)
padding = len(str(len(self.files)))
once = True
for number, file in enumerate(self.files, start=1):
if self.singleFile:
new_name = desired_path / f"{desired_name_stem}{file.suffix.lower()}"
else:
new_name = (
desired_path
/ f"{number:0{padding}d} - {desired_name_stem}{file.suffix}"
)
if file == new_name:
continue
if once:
once = False
logger.debug(f"Moving {file} to {new_name}")
else:
logger.trace(f"Moving {file} to {new_name}")
if dry_run:
continue
self.files[number - 1] = file.rename(new_name)
for file in self.files_extra:
if file.suffix.lower() in {".cue", ".log"} or file in {
self.epub_file,
self.pdf_file,
}:
filename = f"{desired_name_stem}{file.suffix.lower()}"
else:
filename = file.name
new_file = desired_path / filename
if file == new_file:
continue
logger.trace(f"Moving {file} to {new_file}")
if dry_run:
continue
new_file = file.rename(new_file)
if file == self.epub_file:
self.epub_file = new_file
elif file == self.pdf_file:
self.pdf_file = new_file
if self.bookPath != desired_path:
logger.debug(f"Moving {self.bookPath} to {desired_path}")
remaining = list(self.bookPath.iterdir())
if not remaining and not dry_run:
self.bookPath.rmdir()
else:
logger.warning(
f"{red_tag('Action Required')} old path not empty after move: "
f"{self.bookPath}: {remaining}"
)
self.bookPath = desired_path
self.files = sorted(self.files, key=self.extract_track_number)
self.firstFile = self.get_first_file()
def generate_dir_from_metadata(self):
composer = ""
name = self.name
if self.composer:
composer = f" (by {self.composer})"
name = self.name + composer
artist = ""
if self.artist:
artist = re.split(r"[,;]", self.artist)[0].strip().replace("/", "_")
desired_path = self.library_root_path / artist / name
if self.series_and_num:
series = self.series_and_num
number = ""
if "#" in self.series_and_num:
series = self.series_and_num.split("#")
number = f"{series[1]} - "
series = series[0].rstrip()
desired_path = (
self.library_root_path
/ artist
/ series.replace("/", "_")
/ f"{number}{name}".replace("/", "_")
)
desired_name_stem = (
f"{self.artist} - {self.name}{composer}{self.bookPath.suffix}"
)
desired_name_stem = desired_name_stem.replace("/", "_")
return desired_path, desired_name_stem
def create_m4b_from_book(self, creat_m4b_no_delete: bool = False) -> None:
output_file_final = self.bookPath / f"{self.bookPath.name}.m4b"
if self.action_required:
logger.warning("Action required, resolve before creating m4b file")
return
if output_file_final.exists():
logger.info(f"{red_tag('SKIP')} {output_file_final}")
logger.info("Book already present")
return
logger.info(f"{blue_tag('create')} {output_file_final}")
with tempfile.TemporaryDirectory(dir=self.library_root_path.parent) as tmp:
tmpdir = Path(tmp)
output_file_tmp = tmpdir / f"{self.bookPath.name}.m4b"
concat_file = tmpdir / "concat.txt"
metadata_file = tmpdir / "chapters.txt"
# Inspect all inputs before deciding how to create chapters.
file_info = []
has_existing_chapters = False
for media in self.files:
chapters = get_chapters(media)
duration_ms = get_duration_ms(media)
file_info.append(
{
"media": media,
"chapters": chapters,
"duration_ms": duration_ms,
}
)
if chapters:
has_existing_chapters = True
if has_existing_chapters:
logger.debug(
"Existing chapter tracks found; preserving chapter metadata"
)
else:
logger.debug(
"No existing chapters found; generating chapters from filenames"
)
current_ms = 0
with (
concat_file.open("w", encoding="utf-8") as cf,
metadata_file.open("w", encoding="utf-8") as mf,
):
mf.write(";FFMETADATA1\n")
for key, value in self.metadata.items():
if key == "title":
mf.write(
f"{ffmetadata_escape(key)}={ffmetadata_escape(self.metadata['album'])}\n"
)
else:
mf.write(
f"{ffmetadata_escape(key)}={ffmetadata_escape(value)}\n"
)
for info in file_info:
media = info["media"]
media_name = info["media"].name
# Remove title
media_name = media_name.replace(self.name, "")
# Clean separators and whitespace
media_name = re.sub(r"[\s._-]+", " ", media_name).strip()
# Remove leading/trailing separators again if needed
media_name = re.sub(r"^[\s._-]+|[\s._-]+$", "", media_name)
info["media"].with_name(media_name)
chapters = info["chapters"]
duration_ms = info["duration_ms"]
cf.write(f"file '{ffmpeg_escape(media)}'\n")
if has_existing_chapters:
# Preserve actual chapter boundaries and titles.
for chapter in chapters:
start_ms = round(float(chapter["start_time"]) * 1000)
end_ms = round(float(chapter["end_time"]) * 1000)
tags = chapter.get("tags", {})
title = tags.get("title", media.stem)
mf.write("[CHAPTER]\n")
mf.write("TIMEBASE=1/1000\n")
mf.write(f"START={current_ms + start_ms}\n")
mf.write(f"END={current_ms + end_ms}\n")
mf.write(f"title={ffmetadata_escape(title)}\n")
else:
# No input contains chapters: one chapter per file.
mf.write("[CHAPTER]\n")
mf.write("TIMEBASE=1/1000\n")
mf.write(f"START={current_ms}\n")
mf.write(f"END={current_ms + duration_ms}\n")
mf.write(f"title={ffmetadata_escape(media.stem)}\n")
current_ms += duration_ms
codec, bitrate_k = get_codec_bitrate(self.firstFile)
codec_args = ["-c:a", "copy"]
if codec == "mp3":
codec_args = ["-c:a", "aac", "-b:a", bitrate_k]
elif codec == "flac":
codec_args = ["-c:a", "alac", "-b:a", bitrate_k]
elif codec not in {"aac", "alac"}:
codec_args = ["-c:a", "aac", "-b:a", bitrate_k]
cmd = [
"ffmpeg",
"-y",
"-f",
"concat",
"-safe",
"0",
"-i",
str(concat_file),
"-i",
str(metadata_file),
"-i",
str(self.firstFile),
# audio from concat
"-map",
"0:a",
# cover image from first media
"-map",
"2:v?",
# Metadata and chapters from our generated FFMETADATA file
"-map_metadata",
"1",
"-map_chapters",
"1",
*codec_args,
"-c:v",
"copy",
"-disposition:v:0",
"attached_pic",
str(output_file_tmp),
]
result = subprocess.run(
cmd,
capture_output=True,
text=True,
check=True,
)
logger.trace(result.stdout)
if result.returncode == 0:
output_file_tmp.rename(output_file_final)
for media in self.files:
if creat_m4b_no_delete:
continue
media.unlink()
self.files = [output_file_final]
self.firstFile = output_file_final
self.fileConunt = 1
self.singleFile = True
self.changed = False
self.size = Path(output_file_final).stat().st_size
logger.info(
f"{green_tag('SUCCESS')} - {self.bookPath.name} - delete media files"
)
else:
logger.error(
f"{red_tag('ERROR')} - {self.bookPath.name} - Code: {result.returncode}, Stderr: {result.stderr}"
)
def write_metadata(self, keys=None, dry_run=False):
if self.action_required:
logger.warning("Action required, resolve before writing metadata file")
return
self.write_metadata_from_fields()
if keys is None:
keys = []
for key, value in self.metadata.items():
if self.metadata_original.get(key) != value and not isinstance(
value, list
):
keys.append(key)
self.metadata_original = self.metadata.copy()
values = [self.metadata[key] for key in keys]
if (not keys or not values) and self.metadata_original.get(
"title"
) != self.title:
logger.trace(f"Nothing to write to file for {self.name}")
return
if keys:
for file in self.files:
write_metadata(file, keys, values, dry_run)
if len(self.title_list) == len(self.files):
for title, file in zip(self.title_list, self.files):
write_metadata(file, ["title"], [title], dry_run)
self.changed = False
def fix_title(self):
if not self.title:
if self.singleFile:
self.title = self.name
else:
padding = len(str(len(self.files)))
for idx, file in enumerate(self.files):
self.title_list.append(f"{self.name} - {idx + 1:0{padding}d}")
class AudiobookLibrary:
def __init__(
self,
library_path: Path,
api_url: str,
api_key: str,
sql_database_path: Path,
dry_run: bool,
path_filter: str = "",
logfilename: str = "audiobook_library.log",
):
self.path_filter = path_filter
self.sql_database_instance = self.SqlDatabase(sql_database_path)
self.api_instance = self.AudiobookshelfApi(api_url,api_key)
self.dry_run = dry_run
logging.setLoggerClass(CustomLogger)
logger: CustomLogger = logging.getLogger(__name__) # type: ignore[assignment]
logger.info("Started")
self.libraryPath: Path = Path(library_path)
self.audiobooks: list[Audiobook] = []
self._by_path = {}
self._by_name = defaultdict(list)
self._by_artist = defaultdict(list)
self._by_composer = defaultdict(list)
self.load_audiobooks_from_dir()
self.size = sum(
audiobook.size for audiobook in self.audiobooks
) # size in bytes
class SqlDatabase:
def __init__(self, sql_database_path: Path):
self.tables = []
self.data = {}
self.db = None
self.path = sql_database_path
self.books = []
def connect(self):
if not self.path.is_file():
logger.error(f"{self.path} does not exist")
raise FileExistsError
self.db = sqlite3.connect(self.path)
self.db.row_factory = sqlite3.Row
return True
def cursor(self):
if not self.db:
raise ValueError
return self.db.cursor()
def commit(self):
if not self.db:
raise ValueError
self.db.commit()
def close(self):
if not self.db:
raise ValueError
self.db.close()
def init(self):
if not self.connect():
return False
logger.info("Database initialized successfully")
return self.load_database_data()
def load_database_data(self) -> bool:
logger.info("Load Database data")
if not self.db:
raise ValueError
self.tables = self.db.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
for table in self.tables:
table_name = table["name"]
rows = []
for row in self.db.execute(f'SELECT * FROM "{table_name}"'):
record = dict(row)
# Auto-decode JSON columns
for key, value in record.items():
if isinstance(value, str) and value[:1] in "[{":
try:
record[key] = json.loads(value)
except json.JSONDecodeError:
pass
rows.append(record)
self.data[table_name] = rows
self.books = self.data["books"]
return True
def update_books(self):
for book in self.books:
self._save_book(book)
def _save_book(self, book):
book = book.copy()
for key, value in book.items():
if isinstance(value, (list, dict)):
book[key] = json.dumps(value, ensure_ascii=False)
columns = [c for c in book if c != "id"]
sql = f"""
UPDATE books
SET {", ".join(f'"{c}" = ?' for c in columns)}
WHERE id = ?
"""
values = [book[c] for c in columns]
values.append(book["id"])
if not self.db:
raise ValueError
self.db.execute(sql, values)
# commit once after all updates
def fix_genres(self):
logger.info("Fix genres in database")
for book in self.books:
genres = book.get("genres")
if not genres or not any(g.strip() for g in genres):
continue
new_genres = []
for genre in genres:
genre = clean_genre(genre)
if "," in genre:
genre = genre.split(",")
elif ";" in genre:
genre = genre.split(";")
elif " - " in genre:
genre = genre.split(" - ")
elif "Historical Adventure Thriller Mystery" in genre:
genre = genre.split(" ")
if isinstance(genre, str):
if len(genre) > 50 or (5 > len(genre) > 0):
genre = ""
logger.trace(f"Too long or too short: {genre}")
genre = genre.strip()
genre = genre.lstrip()
if genre:
new_genres.append(genre)
elif isinstance(genre, list):
genre_buf = genre
if len(genre) > 1:
genre_buf = [g.strip() for g in genre if g]
new_genres.extend(genre_buf)
new_genres = sorted(set(new_genres))
logger.trace(f"{new_genres}")
list_difference = False
if len(new_genres) < len(genres):
list_difference = True
for old, new in zip(genres, new_genres):
if old != new:
list_difference = True
break
if not list_difference:
logger.trace("No difference: Dont Commit to Database")
continue
logger.debug(f"{genres} -> {new_genres}")
book["genres"] = new_genres
def fix_comments(self):
logger.info("Fix comments in database")
for book in self.books:
for file in book.get("audioFiles"):
comment = file.get("metaTags").get("tagComment")
if comment and len(comment) < 40:
logger.debug(f"Fix short comment for {book.get('title')}")
comment = ""
file["metaTags"]["tagComment"] = comment
class AudiobookshelfApi:
def __init__(self, url: str, api_key: str, timeout: int = 30):
self.url = url.rstrip("/")
self.timeout = timeout
self.session = requests.Session()
self.session.headers["Authorization"] = f"Bearer {api_key}"
self.books = []
def _request(self, method: str, endpoint: str, **kwargs) -> Any:
resp = self.session.request(
method,
f"{self.url}{endpoint}",
timeout=self.timeout,
**kwargs,
)
try:
resp.raise_for_status()
except requests.exceptions.HTTPError:
logger.error(f"{method} {endpoint}")
logger.error(resp)
if not resp.content or resp.status_code != 200:
logger.error(f"Request failed: {resp}")
return None
# No content
if not resp.content:
return None
# Only parse JSON if response claims to be JSON
content_type = resp.headers.get("Content-Type", "")
if "application/json" in content_type:
return resp.json()
# Return raw text otherwise
return resp.text
def get(self, endpoint: str, **kwargs):
return self._request("GET", endpoint, params=kwargs)
def post(self, endpoint: str, data: dict, **kwargs):
return self._request("POST", endpoint, json=data, params=kwargs)
def put(self, endpoint: str, data: dict):
return self._request("PUT", endpoint, json=data)
def delete(self, endpoint: str):
return self._request("DELETE", endpoint)
def load_books(self) -> list[dict]:
"""Load all books from all libraries into self.books."""
self.books = []
libraries = self.get("/api/libraries")["libraries"]
for library in libraries:
library_id = library["id"]
response = self.get(
f"/api/libraries/{library_id}/items",
minified="0"
)
self.books.extend(response["results"])
return self.books
def embed_metadata_for_book(self, item_id: str) -> None:
book = next((b for b in self.books if b["id"] == item_id), None)
if book:
title = (
book.get("media", {})
.get("metadata", {})
.get("title", item_id)
)
if book.get("media", {}).get("numAudioFiles", 0) > 0:
logger.info(f"Embedding metadata: {title}")
self.post(
f"/api/tools/item/{item_id}/embed-metadata",
data={},
backup=False,
)
else:
logger.info(f"{title} is an epub only book")
else:
logger.info(f"Book not found: {item_id}")
def embed_metadata_for_all_books(self, args) -> None:
for book in self.books:
item_id = book["id"]
book_name = (
book.get("media", {})
.get("metadata", {})
.get("title", item_id)
)
book_path = book.get("path", "")
logger.info(f"Embedding metadata: {book_name} ({book_path})")
if args.dry_run:
continue
try:
self.embed_metadata_for_book(item_id)
except Exception as e:
logger.error(f"Failed: {book_name}: {e}")
def embed_metadata_from_file(self, filename: str) -> None:
filename = Path(filename).expanduser().resolve()
with open(filename, encoding="utf-8") as f:
item_ids = [
line.strip()
for line in f
if line.strip() and not line.startswith("#")
]
logger.info(f"Loaded {len(item_ids)} item IDs from {filename}")
for item_id in item_ids:
book = next(
(b for b in self.books if b["id"] == item_id),
None
)
try:
self.embed_metadata_for_book(item_id)
except Exception as e:
logger.error(f"Failed: {item_id}: {e}")
def add(self, book: Audiobook):
self.audiobooks.append(book)
self._by_path[book.bookPath] = book
self._by_name[book.name].append(book)
if book.artist:
self._by_artist[book.artist].append(book)
if book.composer:
self._by_composer[book.composer].append(book)
def print_name(self, books):
if not isinstance(books, list):
books = [books]
for book in books:
if book.name:
logger.info(book.name)
else:
logger.info(f"No name in {book.bookPath.name}")
def get_by_path(self, path):
return self._by_path.get(Path(path))
def get_by_name(self, name):
return self._by_name.get(name, [])
def get_by_artist(self, artist):
return self._by_artist.get(artist, [])
def get_by_composer(self, composer):
return self._by_artist.get(composer, [])
def find_path_contains(self, path) -> list[Audiobook]:
return [
book
for book in self.audiobooks
if book.bookPath == path or book.bookPath.is_relative_to(path)
]
def find_name_contains(self, text):
text = text.lower()
return [book for book in self.audiobooks if text in book.name.lower()]
def find_artist_contains(self, text):
text = text.lower()
return [
book
for book in self.audiobooks
if book.artist and text in book.artist.lower()
]
def load_audiobooks_from_dir(self):
"""
A book directory is any directory containing more than one
audiobook file (.mp3, .m4a, .m4b)
"""
media_files = [
file
for file in self.libraryPath.rglob("*")
if file.is_file()
and file.suffix.lower() in AUDIO_SUFFIXES
and self.path_filter in str(file)
]
dirs = {file.parent for file in media_files}
for directory in dirs:
self.add(Audiobook(directory, self.libraryPath))
for audiobook in self.audiobooks:
audiobook.init_book()
def load_metadata(self):
for audiobook in self.audiobooks:
audiobook.load_metadata()
def write_metadata(self, keys=None):
for audiobook in self.audiobooks:
if audiobook.changed:
logger.trace(f"Writing metadata: {audiobook.name}")
audiobook.write_metadata(keys, dry_run=self.dry_run)
def fix_artist_dir(self):
for book in self.audiobooks:
book.fix_artist_dir()
def create_structured_directories(self, dry_run=False):
for book in self.audiobooks:
desired_path, desired_name = book.generate_dir_from_metadata()
book.move(desired_path, desired_name, dry_run)
def fix_artists(self, force_artist_from_path=False):
for book in self.audiobooks:
book.fix_artist_from_comment()
book.fix_artist_from_path(force_artist_from_path)
def fix_composers(self):
for book in self.audiobooks:
book.fix_composer_from_narrator()
book.fix_composer_from_comment()
book.fix_composer_from_path()
def fix_names(self):
for book in self.audiobooks:
book.fix_name()
def fix_comments(self):
for book in self.audiobooks:
if book.comment and len(book.comment) < 40:
logger.info(f"Remove short comment of {book.name}: {book.comment}")
book.comment = ""
book.changed = True
def fix_genres(self):
for book in self.audiobooks:
book.fix_genre()
def fix_titles(self):
for book in self.audiobooks:
book.fix_title()
def fix_series(self):
for book in self.audiobooks:
book.fix_series()
def create_m4b_from_dir(self, keep_originals: bool = False) -> None:
audiobooks = self.audiobooks
jobs = [audiobook for audiobook in audiobooks if not audiobook.singleFile]
for audiobook in jobs:
logger.debug(f"Create m4b file for {audiobook.name}")
if not self.dry_run:
with ThreadPoolExecutor(max_workers=8) as executor:
futures = {
executor.submit(
audiobook.create_m4b_from_book,
keep_originals,
): audiobook
for audiobook in jobs
}
for future in as_completed(futures):
audiobook = futures[future]
try:
future.result()
except Exception:
logger.exception(f"Failed to create m4b for {audiobook.name}")
def main():
parser = argparse.ArgumentParser(description="Audiobook Directory Processing")
parser.add_argument("library", type=Path, help="Library path")
parser_add_loglevel(parser)
m4b_group = parser.add_argument_group("M4B creation")
m4b_group.add_argument(
"-cm",
"--create-m4b",
action="store_true",
help="Create M4B files from multi-file audiobooks",
)
m4b_group.add_argument(
"--keep-originals",
action="store_true",
help="Do not delete source files after M4B creation",
)
m4b_group.add_argument(
"--name-filter",
metavar="NAME",
default="",
help="Process only books whose name contains NAME",
)
m4b_group.add_argument(
"--path-filter",
metavar="PATH",
default="",
help="Process only books whose path contains PATH",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Dry run only",
)
parser.add_argument(
"--sql-database",
type=Path,
default=Path("/docker/audiobookshelf/config/absdatabase.sqlite"),
help="SQL Database Path",
)
parser.add_argument(
"--api-url",
type=str,
default="https://audiobooks.someoneslinux.selfhost.eu",
help="API Url for Audiobookshelf API",
)
parser.add_argument(
"--api-key",
type=str,
default="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJrZXlJZCI6Ijc3NTc5NDE0LTczMGItNDllMy04YTgzLTk3M2Y1MjRiOGVlNyIsIm5hbWUiOiJBdWRpb2Jvb2tfbGlicmFyeS5weSIsInR5cGUiOiJhcGkiLCJpYXQiOjE3OTM3MDIyODd9.GJVOgYcWM4JpJyMhqzQAHJCfypKiWNBfJHwdM_uvZHU",
help="API Key for Audiobookshelf API",
)
parser.add_argument(
"-em",
"--encode-meta",
action="store_true",
help="Encode Metadata to Audiobook files",
)
parser.add_argument(
"-fd",
"--fix-db",
action="store_true",
help="Fix the Database",
)
parser.add_argument(
"-fa",
"--force-artist-from-path",
action="store_true",
help="Force overwrite Artist from Path, only use if Path is correct",
)
parser.add_argument(
"-m",
"--move",
action="store_true",
help="Create structured directories for all books: /artist/series/book(read by X)/file.m4b",
)
actions = [
"artis_dir",
"artists",
"composers",
"series",
"names",
"genres",
"comments",
"titles",
]
parser.add_argument(
"--actions",
nargs="+",
choices=actions,
default=actions,
)
args = parser.parse_args()
if "--sql-database" in sys.argv:
args.fix_db = True
setup_logging(args.log_level, "audiobook_library")
library_dir = args.library.expanduser()
library_dir = library_dir.resolve()
if not library_dir.exists():
logger.error(f"Library directory does not exist: {library_dir}")
sys.exit(1)
database_path = args.sql_database.expanduser()
database_path = database_path.resolve()
library = AudiobookLibrary(
library_dir, args.api_url,args.api_key,database_path, args.dry_run, path_filter=args.path_filter
)
if args.encode_meta:
library.api_instance.load_books()
library.api_instance.embed_metadata_from_file("~/embed_ids.txt")
#library.api_instance.embed_metadata_for_all_books(args)
if "artist_dir" in args.actions:
library.fix_artist_dir()
if "artists" in args.actions:
library.fix_artists(args.force_artist_from_path)
if "composers" in args.actions:
library.fix_composers()
if "series" in args.actions:
library.fix_series()
if "names" in args.actions:
library.fix_names()
if "comments" in args.actions:
library.fix_comments()
if "genres" in args.actions:
library.fix_genres()
if "titles" in args.actions:
library.fix_titles()
if args.fix_db:
if not database_path.is_file():
logger.error(f"Database is not a file: {database_path}")
return
library.sql_database_instance.init()
library.sql_database_instance.fix_genres()
library.sql_database_instance.fix_comments()
if not args.dry_run:
library.sql_database_instance.update_books()
library.sql_database_instance.commit()
library.sql_database_instance.close()
if args.create_m4b:
library.create_m4b_from_dir(
keep_originals=args.keep_originals,
)
library.write_metadata()
if args.move:
library.create_structured_directories(args.dry_run)
logger.info("All Operations completed successfully")
if __name__ == "__main__":
main()