Files
Server-scripts/audiobook_metadata/metadata_audiobooks_library.py
T
Lucas Vivo 93efb2cb02 feat(audiobook_library): sorting key, normalize names
new sorting key to extract track numbers
get codec from audio file
normalize album name to remove brackets and series name
create book dir from metadata
load all tables into database object
2026-10-30 10:27:41 +01:00

1251 lines
42 KiB
Python
Executable File

#!/home/lucas/scripts/.venv/bin/python
import argparse
import json
import logging
import re
import subprocess
import sys
import tempfile
import sqlite3
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from serverlibrary import *
logging.setLoggerClass(CustomLogger)
logger: CustomLogger = logging.getLogger(__name__) # type: ignore[assignment]
metadata_fields = ["artist", "album", "composer", "comment", "genre", "grouping", "album artist", "description"]
SEP = r"\s*[-_:,.]+\s*"
series_patterns = [
rf'^(\d+){SEP}(.+)$',
rf'^(.+?){SEP}(\d+)$',
rf'^(.+?)(?:{SEP}?(?:Book|Band|Volume)\s+(\d+))?$'
]
def ffmpeg_escape(path: Path) -> str:
return str(path.resolve()).replace("'", r"'\''")
def clean_genre(genre):
if not genre:
return ""
genre_old = genre
#genre = genre.lower()
genre = re.sub(r"(?i)^audio\s*books?$", "", genre)
genre = re.sub(r"(?i)^audiobooks?;?$", "", genre)
genre = re.sub(r"(?i)\b(hörb(uch|ücher)|hoerb(uch|uecher))\b", "", genre)
genre = re.sub(r"(?i)^(hörspiele?|hoerspiele?)$", "", genre)
genre = re.sub(r"(?i)^spoken\s*word$", "", genre)
genre = re.sub(r"(?i)gesprochene\s*inhalte$", "", genre)
genre = re.sub(r"(?i)^speech$", "", genre)
genre = re.sub(r"Fanasie", "Fantasie", genre)
genre = re.sub(r"Mit Videospiel-?", "Videospielbezug", genre)
genre = re.sub(r"(?i)nonfiction", "NonFiction", genre)
genre = re.sub(r"(?i)^speech$", "", genre)
genre = re.sub(r"(?i)^vocal$", "", genre)
genre = re.sub(r"(?i)^retail$", "", genre)
genre = re.sub(r"(?i)^other$", "", genre)
genre = re.sub(r"(?i)^misc$", "", genre)
genre = re.sub(r"(?i)^(sci[- ]?fi|sf)$", "Science Fiction", genre)
genre = re.sub(r"(?i)^\(?science fiction\)?$", "Science Fiction", genre)
genre = re.sub(r"Thrillers", "Thriller", genre)
genre = re.sub(r"(?i)\bchildren'?s audiobooks?\b", "", genre)
genre = re.sub(r"^-\s*|\s*-$", "", genre)
genre = re.sub(r"(?i)Engelske\s*\sLydbånd", "", genre)
genre = re.sub(r"^FIC028000.*$", "", genre)
genre = re.sub(r"^800.*$", "", genre)
genre = re.sub(r"^Fantastic Beasts.*$", "", genre)
genre = re.sub(r"^and thei.*$", "", genre)
genre = re.sub(r"^those you.*$", "", genre)
# Normalize separators
genre = re.sub(r"\s*[,:/]\s*", ";", genre)
genre = re.sub(r"\s*;\s*", "; ", genre)
# Trim punctuation / whitespace
genre = re.sub(r"^[\s,;:\-]+", "", genre)
genre = re.sub(r"[\s,;\-]+$", "", genre)
genre = genre.strip()
genre = genre.lstrip()
if genre != genre_old:
logger.debug(f"Update genre from {genre_old} --->>> {genre}")
return genre
def ffmetadata_escape(value: str) -> str:
return (
value.replace("\\", "\\\\")
.replace("=", "\\=")
.replace(";", "\\;")
.replace("#", "\\#")
.replace("\n", "\\\n")
)
def get_chapters(media: Path) -> list[dict]:
result = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-show_chapters",
"-of",
"json",
str(media),
],
capture_output=True,
text=True,
check=True,
)
data = json.loads(result.stdout)
return data.get("chapters", [])
def get_metadata_all(file: Path) -> dict[str, str]:
metadata = {}
try:
result = subprocess.run(
[
"ffprobe",
"-v",
"quiet",
"-print_format",
"json",
"-show_format",
str(file),
],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
check=True,
)
metadata = {
str(k): str(v)
for k, v in json.loads(result.stdout)
.get("format", {})
.get("tags", {})
.items()
}
except (OSError, subprocess.SubprocessError, json.JSONDecodeError) as error:
print(f"Error getting metadata for {file}: {error}")
for field in metadata_fields:
metadata.setdefault(field, "")
return metadata
def get_metadata_fields(file, fields) -> dict:
metadata = dict.fromkeys(fields, "")
for field in fields:
cmd = ["kid3-cli", "-c", f'get "{field}"', str(file)]
try:
result = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
check=True,
)
metadata[field] = result.stdout.strip()
except (OSError, subprocess.SubprocessError) as error:
print(f"Error getting metadata for {file}: {error}")
return metadata
return metadata
def get_duration_ms(media: Path) -> int:
duration = subprocess.check_output(
[
"ffprobe",
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
str(media),
],
text=True,
).strip()
return int(float(duration) * 1000)
def write_metadata(file, fields, values,dry_run=False):
cmd_base = [
"kid3-cli",
"-c",
"tag 2"
]
for field, value in zip(fields, values):
cmd_base.extend( ["-c", f'set "{field}" "{value}"'])
cmd_base.extend(["-c", "save", str(file)])
try:
logger.trace(f"Writing metadata for {file}")
logger.trace(f"Cmd: {" ".join(cmd_base)}")
if dry_run:
return
result = subprocess.run(
cmd_base,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
check=True,
)
except (OSError, subprocess.SubprocessError) as error:
print(f"Error writing metadata for {file}: {error}")
return
return
def normalize_name(name: str,series: str) -> str:
# Remove anything in (), [], {}
name_out = ""
name = re.sub(series,"",name)
name = re.sub(r"\s*[\(\[\{].*?[\)\]\}]\s*", " ", name).strip()
name = re.sub(r"\s+", " ", name)
for pattern in series_patterns:
match = re.match(pattern, name, re.IGNORECASE)
if match:
if pattern.startswith(r"^(\d+)"):
name_out = match.group(2).strip()
else:
name_out = match.group(1).strip()
break
else:
name_out = name
pattern = rf'^(.+?){SEP}(?:Book|Band|Volume)\s+(\d+){SEP}(.+)$'
match = re.match(pattern, name, re.IGNORECASE)
if match:
name_out1 = match.group(1).strip()
name_out3 = match.group(3).strip()
if score_match_jaccard(words(name_out1), words(series)) > score_match_jaccard(words(name_out3), words(series)):
return name_out3
else:
return name_out1
return name_out
def cleanup(text: str) -> str:
# Collapse repeated punctuation groups like ", ,", ". .", "! !", "? ?"
while True:
new_text = re.sub(r'([.,!?;:-])(?:\s+\1)+', r'\1', text)
if new_text == text:
break
text = new_text
# Optional: normalize multiple spaces
text = re.sub(r'\s{2,}', ' ', text)
return text.strip()
def get_empty_keys(data: dict) -> str:
empty_keys = [str(key) for key, value in data.items() if not value]
return f"Empty: {', '.join(empty_keys)}" if empty_keys else ""
def get_codec_bitrate(file: Path) -> tuple[str, str]:
info = json.loads(
subprocess.check_output(
[
"ffprobe",
"-v", "error",
"-select_streams", "a:0",
"-show_entries", "stream=codec_name,bit_rate",
"-of", "json",
str(file),
],
text=True,
)
)
stream = info["streams"][0]
bitrate = int(stream.get("bit_rate") or 128000)
bitrate_k = f"{bitrate // 1000}k"
return stream["codec_name"].lower(), bitrate_k # codec, bits/s
class Audiobook:
def __init__(self, book_path: Path, library_root_path : Path) -> None:
self.series_num = None
self.pdf_file = None
self.description = None
self.epub_file = None
self.files_extra = []
self.action_required = False
self.series_and_num = ""
self.series = ""
self.singleFile = False
self.fileConunt = 0
self.size = 0
self.firstFile = None
self.changed = None
self.metadata = {}
self.artist = ""
self.genre = ""
self.composer = ""
self.name = ""
self.comment = ""
self.files: list[Path] = []
self.bookPath = book_path
self.library_root_path = library_root_path
def init_book(self) -> None:
self.files = sorted(
(
file
for file in self.bookPath.iterdir()
if file.is_file() and file.suffix.lower() in AUDIO_SUFFIXES
),
key=self.extract_track_number,
)
epubs = list(self.bookPath.glob("*.epub"))
self.epub_file = epubs[0] if len(epubs) == 1 else None
pdfs = list(self.bookPath.glob("*.pdf"))
self.pdf_file = pdfs[0] if len(pdfs) == 1 else None
self.files_extra = [
file
for file in self.bookPath.glob("*") if file.suffix.lower() not in AUDIO_SUFFIXES
]
self.firstFile = self.get_first_file()
self.size = sum(
file.stat().st_size
for pattern in AUDIO_RE_SUFFIXES
for file in self.bookPath.glob(pattern)
)
self.fileConunt = len(self.files)
self.singleFile = self.fileConunt == 1
# Metadata
self.load_metadata()
logger.info(f"Found {self.bookPath.name}, {self.fileConunt} files, {get_empty_keys(self.metadata)}")
self.changed = False
def __repr__(self):
return (
f"Audiobook("
f"name={self.name!r}, "
f"artist={self.artist!r}, "
f"composer={self.composer!r}, "
f"path={str(self.bookPath)!r}),"
f"size={self.size!r}, "
f"fileConunt={self.fileConunt!r}, "
)
def get_first_file(self) -> Path:
return self.files[0]
def write_fields_from_metadata(self) -> None:
self.name = self.metadata["album"]
self.composer = self.metadata["composer"]
self.comment = self.metadata["comment"]
self.genre = self.metadata["genre"]
if "#" in self.metadata["grouping"]:
self.series, self.series_num = self.metadata["grouping"].rsplit("#", 1)
self.series = self.series.strip()
self.series_and_num = self.metadata["grouping"]
artist = self.metadata.get("artist", "").strip()
self.artist = artist or self.metadata.get("album artist")
self.description = self.metadata["description"]
def write_metadata_from_fields(self):
self.metadata["album"] = self.name
self.metadata["composer"] = self.composer
self.metadata["genre"] = self.genre
self.metadata["grouping"] = self.series_and_num
self.metadata["comment"] = self.comment
self.metadata["description"] = self.description
self.metadata["artist"] = self.artist
self.metadata["album artist"] = self.artist
def extract_track_number(self, file: Path) -> int:
stem = file.stem
title = self.name
# Remove title if it appears at the beginning
if title:
escaped_title = re.escape(title)
stem = re.sub(
rf"^{escaped_title}(?:\s*[-_:]\s*)?",
"",
stem,
flags=re.IGNORECASE,
).strip()
patterns = [
# 01 Title, 02GLCK~1.WMA
r"^(\d{2,3})(?=\D|$)",
# CD1-01, CD 1 - 01, CD1 - 01.
r"CD\s*(\d+)\s*-\s*(\d{1,3})",
# 018-01, 018 - 01, 138-01, 01-01
r"^(\d{1,3})\s*-\s*(\d{1,3})(?:\D|$)",
# Track 01
r"Track\s+(\d+)$",
# 01 - Kapitel 01
r"^(\d+)\s*-",
# Rubinrot 101, 102, 201, 202
r".*?(\d)(\d{2})$",
# A Thousand Ships - 007
r".*?-\s*(\d+)\s*$",
]
for pattern in patterns:
m = re.search(pattern, stem, re.IGNORECASE)
if not m:
continue
groups = m.groups()
if len(groups) == 1:
return int(groups[0])
if len(groups) == 2 and "CD" in pattern:
disc = int(groups[0])
track = int(groups[1])
return (disc - 1) * 1000 + track
if len(groups) == 2 and stem.startswith(groups[0]):
disc = int(groups[0])
track = int(groups[1])
if track < 100:
return disc * 1000 + track
if len(groups) == 2 and len(groups[0]) == 1:
disc = int(groups[0])
track = int(groups[1])
return disc * 1000 + track
return int(groups[-1])
return 0
def load_metadata(self):
self.metadata = get_metadata_all(self.get_first_file())
self.write_fields_from_metadata()
logger.trace(
f"Load metadata {self.metadata}"
)
def fix_artist_dir(self):
rel_path = self.bookPath.relative_to(self.library_root_path).parts
if not self.artist:
logger.trace(f"No artist tag for {self.name}")
return
if len(rel_path) < 2:
desired_path,desired_name = self.generate_dir_from_metadata()
self.move(desired_path,desired_name)
logger.debug(f"Artist not visible in folder structure: {rel_path}")
def fix_artist_from_comment(self):
m = re.search(
r"(?:"
r"author(?:s)?:?|"
r"writer(?:s)?:?|"
r"co[- ]?author(?:s)?:?|"
r"co[- ]?writer(?:s)?:?|"
r"creator(?:s)?:?|"
r"contributor(?:s)?:?|"
r"composer(?:s)?:?|"
r"lyricist(?:s)?:?|"
r"songwriter(?:s)?:?|"
r"written\s+by:?|"
r"story\s+by:?|"
r"autorin(?:nen)?:?|"
r"autor\(en\):?|"
r"autor(?:en)?:?|"
r"text(?:e)?\s+von:?|"
r"geschrieben\s+von:?|"
r"verfasst\s+von:?|"
r"auteur(?:s)?:?|"
r"autore(?:i)?:?|"
r"autor(?:es)?:?|"
r"escritor(?:es|as)?:?"
r")\s*:?\s*([^\n.,;]+)",
self.comment,
re.IGNORECASE,
)
if not m:
logger.trace(f"No artist found in comment for {self.bookPath}")
return
self.changed = True
artist = re.sub(r"\s*\([^)]*\)", "", m.group(1).strip()).strip()
logger.trace(f"Remove {artist} from comment: {self.comment}")
self.comment = (self.comment[: m.start()] + self.comment[m.end() :]).strip(
" .,;\n"
)
self.comment = cleanup(self.comment)
if not self.artist:
self.artist = artist
self.changed = True
elif self.artist != artist:
logger.warning(
f"{red_tag("Action Required")} artist {artist} found not matching {self.artist} for {self.bookPath}"
)
self.action_required = True
else:
logger.debug(f"Artist {artist} matches file metadata for {self.name}")
def fix_artist_from_path(self):
rel_path = self.bookPath.relative_to(self.library_root_path).parts
if len(rel_path) < 2:
logger.trace(f"Artist not visible in folder structure: {rel_path}")
return
if not self.artist:
self.changed = True
self.artist = rel_path[0]
logger.info(f"[Updated] artist found in path for {rel_path}")
def fix_composer_from_comment(self):
m = re.search(
r"(?:read\s+by:?|"
r"spoken\s+by:?|"
r"narrated\s+by:?|"
r"gesprochen\s+von:?|"
r"gelesen\s+von)\s+([^\n.,;]+)",
self.comment,
re.IGNORECASE,
)
if not m:
logger.trace(f"No composer found in comment for {self.name}")
return
self.changed = True
self.comment = (self.comment[: m.start()] + self.comment[m.end() :]).strip(
" .,;\n"
)
self.comment = cleanup(self.comment)
if not self.composer or len(self.composer) < 5:
logger.info(f"[Updated] composer found in comment for {self.name}")
self.composer = m.group(1).strip()
def fix_composer_from_path(self):
rel_path = self.bookPath.relative_to(self.library_root_path)
m = re.search(
r"(?:read\s+by:?|"
r"spoken\s+by:?|"
r"narrated\s+by:?|"
r"gesprochen\s+von:?|"
r"gelesen\s+von)\s+([^\n.,;]+)",
str(rel_path),
re.IGNORECASE,
)
if not m:
logger.trace(f"No composer found in path for {self.bookPath}")
return
if not self.composer or len(self.composer) < 5:
self.changed = True
logger.info(f"[Updated] composer found in Path for {self.bookPath}")
self.composer = m.group(1).strip()
def fix_name(self):
if not self.name:
self.name = normalize_name(self.bookPath.name)
self.changed = True
logger.info(f"[Updated] name found in path for {self.bookPath}")#
return
name = normalize_name(self.name, self.series)
if name != self.name:
logger.info(f"[Updated] name by cleaning {self.name} to {name}")
self.name = name
self.changed = True
def fix_series(self):
if self.series_and_num:
logger.trace(f"{self.name} already has series: {self.series_and_num}")
rel_path = self.bookPath.relative_to(self.library_root_path).parts
number=""
if len(rel_path) <= 2:
logger.trace(f"No series found for {self.name}")
return
for i, pattern in enumerate(series_patterns):
match = re.match(pattern, rel_path[2], re.IGNORECASE)
if not match:
continue
if i == 0:
number = match.group(1)
elif i == 1:
number = match.group(2)
else:
number = match.group(2) or match.group(3)
break
number_str = ""
and_number =""
if not number:
logger.trace(f"No series number found for {self.name}")
else:
number_str = " #" + number
and_number = " and number"
self.series_num = number
self.series_and_num = rel_path[1] + number_str
self.series =rel_path[1]
self.changed = True
logger.info(f"[Updated] series{and_number} found for {self.name} to {self.series_and_num}")
return
def fix_genre(self):
self.genre = clean_genre(self.genre)
self.changed = True
def move(self, desired_path: Path, desired_name_stem: str, dry_run=False):
logger.trace(f"Moving {self.bookPath} to {desired_path}")
if dry_run:
return
desired_path.mkdir(parents=True, exist_ok=True)
padding = len(str(len(self.files)))
for number, file in enumerate(self.files, start=1):
if self.singleFile:
new_name = f"{desired_name_stem}{file.suffix.lower()}"
else:
new_name = f"{number:0{padding}d} - {desired_name_stem}{file.suffix}"
self.files[number-1] = file.rename(desired_path / new_name)
for file in self.files_extra:
if file.suffix.lower() in {".cue", ".log"} or file in {
self.epub_file,
self.pdf_file,
}:
filename = f"{desired_name_stem}{file.suffix.lower()}"
else:
filename = file.name
new_file = file.rename(desired_path / filename)
if file == self.epub_file:
self.epub_file = new_file
elif file == self.pdf_file:
self.pdf_file = new_file
if self.bookPath != desired_path:
remaining = list(self.bookPath.iterdir())
if not remaining:
self.bookPath.rmdir()
else:
logger.warning(f"{red_tag('Action Required')} old path not empty after move: "
f"{self.bookPath}: {remaining}"
)
self.bookPath = desired_path
self.files = sorted(self.files, key=self.extract_track_number)
self.firstFile = self.get_first_file()
logger.debug(f"Moving {self.bookPath} to {desired_path}")
def generate_dir_from_metadata(self):
composer =""
name = self.name
if self.composer:
composer = f" (by {self.composer})"
name = self.name + composer
desired_path = self.library_root_path / self.artist / name
if self.series_and_num:
series = self.series_and_num
number = ""
if "#" in self.series_and_num:
series = self.series_and_num.split("#")
number = f"{series[1]} - "
series = series[0].rstrip()
desired_path = self.library_root_path / self.artist / series / f"{number}{name}"
desired_name_stem = self.artist + " - " + self.name + composer + self.bookPath.suffix
return desired_path, desired_name_stem
def create_m4b_from_book(self, creat_m4b_no_delete: bool = False) -> None:
output_file_final = self.bookPath / f"{self.bookPath.name}.m4b"
if self.action_required:
logger.warning(f"Action required, resolve before creating m4b file")
return
if output_file_final.exists():
logger.info(f"{red_tag("SKIP")} {output_file_final}")
logger.info("Book already present")
return
logger.info(f"{blue_tag("create")} {output_file_final}")
with tempfile.TemporaryDirectory(dir=self.library_root_path.parent) as tmp:
tmpdir = Path(tmp)
output_file_tmp = tmpdir / f"{self.bookPath.name}.m4b"
concat_file = tmpdir / "concat.txt"
metadata_file = tmpdir / "chapters.txt"
# Inspect all inputs before deciding how to create chapters.
file_info = []
has_existing_chapters = False
for media in self.files:
chapters = get_chapters(media)
duration_ms = get_duration_ms(media)
file_info.append(
{
"media": media,
"chapters": chapters,
"duration_ms": duration_ms,
}
)
if chapters:
has_existing_chapters = True
if has_existing_chapters:
logger.debug(
"Existing chapter tracks found; preserving chapter metadata"
)
else:
logger.debug(
"No existing chapters found; generating chapters from filenames"
)
current_ms = 0
with (
concat_file.open("w", encoding="utf-8") as cf,
metadata_file.open("w", encoding="utf-8") as mf,
):
mf.write(";FFMETADATA1\n")
for key, value in self.metadata.items():
if key == "title":
mf.write(f"{ffmetadata_escape(key)}={ffmetadata_escape(self.metadata["album"])}\n")
else:
mf.write(f"{ffmetadata_escape(key)}={ffmetadata_escape(value)}\n")
for info in file_info:
media = info["media"]
media_name = info["media"].name
# Remove title
media_name = media_name.replace(self.name, "")
# Clean separators and whitespace
media_name = re.sub(r"[\s._-]+", " ", media_name).strip()
# Remove leading/trailing separators again if needed
media_name = re.sub(r"^[\s._-]+|[\s._-]+$", "", media_name)
info["media"].with_name(media_name)
chapters = info["chapters"]
duration_ms = info["duration_ms"]
cf.write(f"file '{ffmpeg_escape(media)}'\n")
if has_existing_chapters:
# Preserve actual chapter boundaries and titles.
for chapter in chapters:
start_ms = round(float(chapter["start_time"]) * 1000)
end_ms = round(float(chapter["end_time"]) * 1000)
tags = chapter.get("tags", {})
title = tags.get("title", media.stem)
mf.write("[CHAPTER]\n")
mf.write("TIMEBASE=1/1000\n")
mf.write(f"START={current_ms + start_ms}\n")
mf.write(f"END={current_ms + end_ms}\n")
mf.write(f"title={ffmetadata_escape(title)}\n")
else:
# No input contains chapters: one chapter per file.
mf.write("[CHAPTER]\n")
mf.write("TIMEBASE=1/1000\n")
mf.write(f"START={current_ms}\n")
mf.write(f"END={current_ms + duration_ms}\n")
mf.write(f"title={ffmetadata_escape(media.stem)}\n")
current_ms += duration_ms
codec, bitrate_k = get_codec_bitrate(self.firstFile)
codec_args = ["-c:a", "copy"]
if codec == "mp3":
codec_args = ["-c:a", "aac", "-b:a", bitrate_k]
elif codec == "flac":
codec_args = ["-c:a", "alac", "-b:a", bitrate_k]
elif codec not in {"aac", "alac"}:
codec_args = ["-c:a", "aac", "-b:a", bitrate_k]
cmd = [
"ffmpeg",
"-y",
"-f",
"concat",
"-safe",
"0",
"-i",
str(concat_file),
"-i",
str(metadata_file),
"-i",
str(self.firstFile),
# audio from concat
"-map",
"0:a",
# cover image from first media
"-map",
"2:v?",
# Metadata and chapters from our generated FFMETADATA file
"-map_metadata",
"1",
"-map_chapters",
"1",
*codec_args,
"-c:v",
"copy",
"-disposition:v:0",
"attached_pic",
str(output_file_tmp),
]
result = subprocess.run(
cmd,
capture_output=True,
text=True,
check=True,
)
logger.trace(result.stdout)
if result.returncode == 0:
output_file_tmp.rename(output_file_final)
for media in self.files:
if creat_m4b_no_delete:
continue
media.unlink()
self.files = [output_file_final]
self.firstFile = output_file_final
self.fileConunt = 1
self.singleFile = True
self.changed = False
self.size = Path(output_file_final).stat().st_size
logger.info(
f"{green_tag("SUCCESS")} - {self.bookPath.name} - delete media files"
)
else:
logger.error(
f"{red_tag("ERROR")} - {self.bookPath.name} - Code: {result.returncode}, Stderr: {result.stderr}"
)
def write_metadata(self, keys=None, dry_run=False):
if self.action_required:
logger.warning(f"Action required, resolve before writing metadata file")
return
self.write_metadata_from_fields()
if keys is None:
keys = self.metadata.keys()
values = self.metadata.values()
else:
values = [self.metadata[key] for key in keys ]
for file in self.files:
write_metadata(file, keys, values,dry_run)
self.changed = False
class AudiobookLibrary:
def __init__(
self, library_path: Path, sql_database_path: Path, dry_run: bool, logfilename: str = "audiobook_library.log"
):
self.sql_database_instance = self.SqlDatabase(sql_database_path)
self.dry_run = dry_run
logging.setLoggerClass(CustomLogger)
logger: CustomLogger = logging.getLogger(__name__) # type: ignore[assignment]
logger.info("Started")
self.libraryPath: Path = Path(library_path)
self.audiobooks: list[Audiobook] = []
self._by_path = {}
self._by_name = defaultdict(list)
self._by_artist = defaultdict(list)
self._by_composer = defaultdict(list)
self.load_audiobooks_from_dir()
self.size = sum(
audiobook.size for audiobook in self.audiobooks
) # size in bytes
class SqlDatabase:
def __init__(self, sql_database_path: Path):
self.tables = []
self.data = {}
self.db = None
self.path = sql_database_path
self.books = []
def connect(self):
if not self.path.is_file():
logger.error(f"{self.path} does not exist")
raise FileExistsError
self.db = sqlite3.connect(self.path)
self.db.row_factory = sqlite3.Row
return True
def cursor(self):
return self.db.cursor()
def commit(self):
self.db.commit()
def close(self):
self.db.close()
def init(self):
if not self.connect():
return False
return self.load_database_data()
def load_database_data(self) -> bool:
self.tables = self.db.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
for table in self.tables:
table_name = table["name"]
rows = []
for row in self.db.execute(f'SELECT * FROM "{table_name}"'):
record = dict(row)
# Auto-decode JSON columns
for key, value in record.items():
if isinstance(value, str) and value[:1] in "[{":
try:
record[key] = json.loads(value)
except json.JSONDecodeError:
pass
rows.append(record)
self.data[table_name] = rows
self.books = self.data["books"]
return True
def update_books(self):
for book in self.books:
self._save_book(book)
def _save_book(self, book):
book = book.copy()
for key, value in book.items():
if isinstance(value, (list, dict)):
book[key] = json.dumps(value, ensure_ascii=False)
columns = [c for c in book if c != "id"]
sql = f"""
UPDATE books
SET {", ".join(f'"{c}" = ?' for c in columns)}
WHERE id = ?
"""
values = [book[c] for c in columns]
values.append(book["id"])
self.db.execute(sql, values)
# commit once after all updates
def fix_genres(self):
for book in self.books:
genres = book.get("genres")
if not genres or not any(g.strip() for g in genres):
continue
new_genres = []
for genre in genres:
genre = clean_genre(genre)
if "," in genre:
genre = genre.split(",")
elif " - " in genre:
genre = genre.split(" - ")
elif "Historical Adventure Thriller Mystery" in genre:
genre = genre.split(" ")
if isinstance(genre, str):
if len(genre) > 50 or (5 > len(genre) > 0):
genre = ""
logger.trace(f"Too long or too short: {genre}")
genre = genre.strip()
genre = genre.lstrip()
new_genres.append(genre)
elif isinstance(genre, list):
if len(genres) > 1:
genres = [g for g in genres if g]
new_genres.extend(genre)
new_genres = sorted(set(new_genres))
list_difference = False
if len(new_genres) < len(genres):
list_difference = True
for old, new in zip(genres, new_genres):
if old != new:
list_difference = True
break
if not list_difference:
logger.trace("No difference: Dont Commit to Database")
continue
logger.debug(f"{genres} -> {new_genres}")
book["genres"] = new_genres
def fix_comments(self):
for book in self.books:
for file in book.get("audioFiles"):
comment = file.get("metaTags").get("tagComment")
if comment and len(comment) < 40:
logger.debug(f"Fix short comment for {book.get("title")}")
comment = ""
file["metaTags"]["tagComment"] = comment
def add(self, book: Audiobook):
self.audiobooks.append(book)
self._by_path[book.bookPath] = book
self._by_name[book.name].append(book)
if book.artist:
self._by_artist[book.artist].append(book)
if book.composer:
self._by_composer[book.composer].append(book)
def print_name(self, books):
if not isinstance(books, list):
books = [books]
for book in books:
if book.name:
logger.info(book.name)
else:
logger.info(f"No name in {book.bookPath.name}")
def get_by_path(self, path):
return self._by_path.get(Path(path))
def get_by_name(self, name):
return self._by_name.get(name, [])
def get_by_artist(self, artist):
return self._by_artist.get(artist, [])
def get_by_composer(self, composer):
return self._by_artist.get(composer, [])
def find_path_contains(self, path) -> list[Audiobook]:
return [
book
for book in self.audiobooks
if book.bookPath == path or book.bookPath.is_relative_to(path)
]
def find_name_contains(self, text):
text = text.lower()
return [book for book in self.audiobooks if text in book.name.lower()]
def find_artist_contains(self, text):
text = text.lower()
return [
book
for book in self.audiobooks
if book.artist and text in book.artist.lower()
]
def load_audiobooks_from_dir(self):
"""
A book directory is any directory containing more than one
audiobook file (.mp3, .m4a, .m4b)
"""
media_files = [
file
for file in self.libraryPath.rglob("*")
if file.is_file() and file.suffix.lower() in AUDIO_SUFFIXES
]
dirs = {file.parent for file in media_files}
for directory in dirs:
self.add(Audiobook(directory,self.libraryPath))
for audiobook in self.audiobooks:
audiobook.init_book()
def load_metadata(self):
for audiobook in self.audiobooks:
audiobook.load_metadata()
def write_metadata(self, keys=None):
for audiobook in self.audiobooks:
if audiobook.changed:
logger.trace(f"Writing metadata: {audiobook.name}")
audiobook.write_metadata(keys,dry_run=self.dry_run)
def fix_artist_dir(self):
for book in self.audiobooks:
book.fix_artist_dir()
def create_structured_directories(self):
for book in self.audiobooks:
desired_path, desired_name = book.generate_dir_from_metadata()
book.move(desired_path, desired_name)
def fix_artists(self):
for book in self.audiobooks:
book.fix_artist_from_comment()
book.fix_artist_from_path()
def fix_composers(self):
for book in self.audiobooks:
book.fix_composer_from_comment()
book.fix_composer_from_path()
def fix_names(self):
for book in self.audiobooks:
book.fix_name()
def fix_comments(self):
for book in self.audiobooks:
if book.comment and len(book.comment) < 40:
logger.info(f"Remove short comment of {book.name}: {book.comment}")
book.comment = ""
book.changed = True
def fix_genres(self):
for book in self.audiobooks:
book.fix_genre()
def fix_series(self):
for book in self.audiobooks:
book.fix_series()
def create_m4b_from_dir(
self, path: Path = Path(""), name: str = "", keep_originals: bool = False
) -> None:
audiobooks = []
if path:
audiobooks = self.find_path_contains(path)
if name:
audiobooks.extend(self.find_name_contains(name))
if not name and not path:
audiobooks = self.audiobooks
jobs = [audiobook for audiobook in audiobooks if not audiobook.singleFile]
for audiobook in jobs:
logger.debug(f"Create m4b file for {audiobook.name}")
if not self.dry_run:
with ThreadPoolExecutor(max_workers=8) as executor:
futures = {
executor.submit(
audiobook.create_m4b_from_book,
keep_originals,
): audiobook
for audiobook in jobs
}
for future in as_completed(futures):
audiobook = futures[future]
try:
future.result()
except Exception:
logger.exception(f"Failed to create m4b for {audiobook.name}")
def main():
parser = argparse.ArgumentParser(description="Audiobook Directory Processing")
parser.add_argument("library", type=Path, help="Library path")
parser.add_argument(
"-ll",
"--log-level",
type=str.upper,
default="INFO",
choices=["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
help="Set log level",
)
m4b_group = parser.add_argument_group("M4B creation")
m4b_group.add_argument(
"--create-m4b",
action="store_true",
help="Create M4B files from multi-file audiobooks",
)
m4b_group.add_argument(
"--keep-originals",
action="store_true",
help="Do not delete source files after M4B creation",
)
m4b_group.add_argument(
"--name-filter",
metavar="NAME",
default="",
help="Process only books whose name contains NAME",
)
m4b_group.add_argument(
"--path-filter",
metavar="PATH",
default="",
help="Process only books whose path contains PATH",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Dry run only",
)
parser.add_argument(
"--sql-database",
type=Path,
default=Path("/docker/audiobookshelf/config/absdatabase.sqlite"),
help="SQL Database Path",
)
parser.add_argument(
"-fd"
"--fix-db",
action="store_true",
help="Fix the Database",
)
parser.add_argument(
"-m",
"--move",
action="store_true",
help="Create structured directories for all books: /artist/series/book(read by X)/file.m4b",
)
args = parser.parse_args()
if "--sql-database" in sys.argv:
args.fix_db = True
setup_logging(args.log_level, "audiobook_library")
library_dir = args.library.expanduser()
library_dir = library_dir.resolve()
if not library_dir.exists():
logger.error(f"Library directory does not exist: {library_dir}")
sys.exit(1)
database_path = args.sql_database.expanduser()
database_path = database_path.resolve()
if args.fix_db and not database_path.is_file():
logger.error(f"Database is not a file: {database_path}")
return
library = AudiobookLibrary(library_dir, database_path, args.dry_run)
if args.fix_db:
library.sql_database_instance.init()
library.fix_artist_dir()
library.fix_artists()
library.fix_composers()
library.fix_series()
library.fix_names()
library.fix_comments()
library.fix_genres()
if args.create_m4b:
library.create_m4b_from_dir(
name=args.name_filter,
path=args.path_filter,
keep_originals=args.keep_originals,
)
library.write_metadata()
if args.fix_db:
library.sql_database_instance.fix_genres()
library.sql_database_instance.fix_comments()
if not args.dry_run:
library.sql_database_instance.update_books()
library.sql_database_instance.commit()
library.sql_database_instance.close()
if args.move:
library.create_structured_directories()
print("Fin")
if __name__ == "__main__":
main()