new sorting key to extract track numbers get codec from audio file normalize album name to remove brackets and series name create book dir from metadata load all tables into database object
1251 lines
42 KiB
Python
Executable File
1251 lines
42 KiB
Python
Executable File
#!/home/lucas/scripts/.venv/bin/python
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import sqlite3
|
|
from collections import defaultdict
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
from serverlibrary import *
|
|
|
|
logging.setLoggerClass(CustomLogger)
|
|
logger: CustomLogger = logging.getLogger(__name__) # type: ignore[assignment]
|
|
|
|
metadata_fields = ["artist", "album", "composer", "comment", "genre", "grouping", "album artist", "description"]
|
|
SEP = r"\s*[-_:,.]+\s*"
|
|
series_patterns = [
|
|
rf'^(\d+){SEP}(.+)$',
|
|
rf'^(.+?){SEP}(\d+)$',
|
|
rf'^(.+?)(?:{SEP}?(?:Book|Band|Volume)\s+(\d+))?$'
|
|
]
|
|
def ffmpeg_escape(path: Path) -> str:
|
|
return str(path.resolve()).replace("'", r"'\''")
|
|
|
|
def clean_genre(genre):
|
|
if not genre:
|
|
return ""
|
|
genre_old = genre
|
|
#genre = genre.lower()
|
|
genre = re.sub(r"(?i)^audio\s*books?$", "", genre)
|
|
genre = re.sub(r"(?i)^audiobooks?;?$", "", genre)
|
|
genre = re.sub(r"(?i)\b(hörb(uch|ücher)|hoerb(uch|uecher))\b", "", genre)
|
|
genre = re.sub(r"(?i)^(hörspiele?|hoerspiele?)$", "", genre)
|
|
genre = re.sub(r"(?i)^spoken\s*word$", "", genre)
|
|
genre = re.sub(r"(?i)gesprochene\s*inhalte$", "", genre)
|
|
genre = re.sub(r"(?i)^speech$", "", genre)
|
|
genre = re.sub(r"Fanasie", "Fantasie", genre)
|
|
genre = re.sub(r"Mit Videospiel-?", "Videospielbezug", genre)
|
|
genre = re.sub(r"(?i)nonfiction", "NonFiction", genre)
|
|
|
|
genre = re.sub(r"(?i)^speech$", "", genre)
|
|
genre = re.sub(r"(?i)^vocal$", "", genre)
|
|
genre = re.sub(r"(?i)^retail$", "", genre)
|
|
genre = re.sub(r"(?i)^other$", "", genre)
|
|
genre = re.sub(r"(?i)^misc$", "", genre)
|
|
|
|
genre = re.sub(r"(?i)^(sci[- ]?fi|sf)$", "Science Fiction", genre)
|
|
genre = re.sub(r"(?i)^\(?science fiction\)?$", "Science Fiction", genre)
|
|
genre = re.sub(r"Thrillers", "Thriller", genre)
|
|
genre = re.sub(r"(?i)\bchildren'?s audiobooks?\b", "", genre)
|
|
genre = re.sub(r"^-\s*|\s*-$", "", genre)
|
|
genre = re.sub(r"(?i)Engelske\s*\sLydbånd", "", genre)
|
|
genre = re.sub(r"^FIC028000.*$", "", genre)
|
|
genre = re.sub(r"^800.*$", "", genre)
|
|
genre = re.sub(r"^Fantastic Beasts.*$", "", genre)
|
|
genre = re.sub(r"^and thei.*$", "", genre)
|
|
genre = re.sub(r"^those you.*$", "", genre)
|
|
# Normalize separators
|
|
genre = re.sub(r"\s*[,:/]\s*", ";", genre)
|
|
genre = re.sub(r"\s*;\s*", "; ", genre)
|
|
|
|
# Trim punctuation / whitespace
|
|
genre = re.sub(r"^[\s,;:\-]+", "", genre)
|
|
genre = re.sub(r"[\s,;\-]+$", "", genre)
|
|
|
|
genre = genre.strip()
|
|
genre = genre.lstrip()
|
|
if genre != genre_old:
|
|
logger.debug(f"Update genre from {genre_old} --->>> {genre}")
|
|
return genre
|
|
|
|
def ffmetadata_escape(value: str) -> str:
|
|
return (
|
|
value.replace("\\", "\\\\")
|
|
.replace("=", "\\=")
|
|
.replace(";", "\\;")
|
|
.replace("#", "\\#")
|
|
.replace("\n", "\\\n")
|
|
)
|
|
|
|
def get_chapters(media: Path) -> list[dict]:
|
|
result = subprocess.run(
|
|
[
|
|
"ffprobe",
|
|
"-v",
|
|
"error",
|
|
"-show_chapters",
|
|
"-of",
|
|
"json",
|
|
str(media),
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
|
|
data = json.loads(result.stdout)
|
|
return data.get("chapters", [])
|
|
|
|
def get_metadata_all(file: Path) -> dict[str, str]:
|
|
metadata = {}
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
[
|
|
"ffprobe",
|
|
"-v",
|
|
"quiet",
|
|
"-print_format",
|
|
"json",
|
|
"-show_format",
|
|
str(file),
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
metadata = {
|
|
str(k): str(v)
|
|
for k, v in json.loads(result.stdout)
|
|
.get("format", {})
|
|
.get("tags", {})
|
|
.items()
|
|
}
|
|
except (OSError, subprocess.SubprocessError, json.JSONDecodeError) as error:
|
|
print(f"Error getting metadata for {file}: {error}")
|
|
for field in metadata_fields:
|
|
metadata.setdefault(field, "")
|
|
return metadata
|
|
|
|
def get_metadata_fields(file, fields) -> dict:
|
|
metadata = dict.fromkeys(fields, "")
|
|
for field in fields:
|
|
cmd = ["kid3-cli", "-c", f'get "{field}"', str(file)]
|
|
try:
|
|
result = subprocess.run(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
metadata[field] = result.stdout.strip()
|
|
except (OSError, subprocess.SubprocessError) as error:
|
|
print(f"Error getting metadata for {file}: {error}")
|
|
return metadata
|
|
return metadata
|
|
|
|
def get_duration_ms(media: Path) -> int:
|
|
duration = subprocess.check_output(
|
|
[
|
|
"ffprobe",
|
|
"-v",
|
|
"error",
|
|
"-show_entries",
|
|
"format=duration",
|
|
"-of",
|
|
"default=noprint_wrappers=1:nokey=1",
|
|
str(media),
|
|
],
|
|
text=True,
|
|
).strip()
|
|
|
|
return int(float(duration) * 1000)
|
|
|
|
def write_metadata(file, fields, values,dry_run=False):
|
|
cmd_base = [
|
|
"kid3-cli",
|
|
"-c",
|
|
"tag 2"
|
|
]
|
|
for field, value in zip(fields, values):
|
|
cmd_base.extend( ["-c", f'set "{field}" "{value}"'])
|
|
cmd_base.extend(["-c", "save", str(file)])
|
|
try:
|
|
logger.trace(f"Writing metadata for {file}")
|
|
logger.trace(f"Cmd: {" ".join(cmd_base)}")
|
|
if dry_run:
|
|
return
|
|
result = subprocess.run(
|
|
cmd_base,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
except (OSError, subprocess.SubprocessError) as error:
|
|
print(f"Error writing metadata for {file}: {error}")
|
|
return
|
|
return
|
|
|
|
def normalize_name(name: str,series: str) -> str:
|
|
# Remove anything in (), [], {}
|
|
name_out = ""
|
|
name = re.sub(series,"",name)
|
|
name = re.sub(r"\s*[\(\[\{].*?[\)\]\}]\s*", " ", name).strip()
|
|
name = re.sub(r"\s+", " ", name)
|
|
|
|
for pattern in series_patterns:
|
|
match = re.match(pattern, name, re.IGNORECASE)
|
|
if match:
|
|
if pattern.startswith(r"^(\d+)"):
|
|
name_out = match.group(2).strip()
|
|
else:
|
|
name_out = match.group(1).strip()
|
|
break
|
|
else:
|
|
name_out = name
|
|
|
|
pattern = rf'^(.+?){SEP}(?:Book|Band|Volume)\s+(\d+){SEP}(.+)$'
|
|
match = re.match(pattern, name, re.IGNORECASE)
|
|
if match:
|
|
name_out1 = match.group(1).strip()
|
|
name_out3 = match.group(3).strip()
|
|
if score_match_jaccard(words(name_out1), words(series)) > score_match_jaccard(words(name_out3), words(series)):
|
|
return name_out3
|
|
else:
|
|
return name_out1
|
|
return name_out
|
|
|
|
def cleanup(text: str) -> str:
|
|
# Collapse repeated punctuation groups like ", ,", ". .", "! !", "? ?"
|
|
while True:
|
|
new_text = re.sub(r'([.,!?;:-])(?:\s+\1)+', r'\1', text)
|
|
if new_text == text:
|
|
break
|
|
text = new_text
|
|
|
|
# Optional: normalize multiple spaces
|
|
text = re.sub(r'\s{2,}', ' ', text)
|
|
|
|
return text.strip()
|
|
|
|
def get_empty_keys(data: dict) -> str:
|
|
empty_keys = [str(key) for key, value in data.items() if not value]
|
|
|
|
return f"Empty: {', '.join(empty_keys)}" if empty_keys else ""
|
|
|
|
def get_codec_bitrate(file: Path) -> tuple[str, str]:
|
|
info = json.loads(
|
|
subprocess.check_output(
|
|
[
|
|
"ffprobe",
|
|
"-v", "error",
|
|
"-select_streams", "a:0",
|
|
"-show_entries", "stream=codec_name,bit_rate",
|
|
"-of", "json",
|
|
str(file),
|
|
],
|
|
text=True,
|
|
)
|
|
)
|
|
|
|
stream = info["streams"][0]
|
|
|
|
bitrate = int(stream.get("bit_rate") or 128000)
|
|
bitrate_k = f"{bitrate // 1000}k"
|
|
return stream["codec_name"].lower(), bitrate_k # codec, bits/s
|
|
|
|
|
|
class Audiobook:
|
|
def __init__(self, book_path: Path, library_root_path : Path) -> None:
|
|
self.series_num = None
|
|
self.pdf_file = None
|
|
self.description = None
|
|
self.epub_file = None
|
|
self.files_extra = []
|
|
self.action_required = False
|
|
self.series_and_num = ""
|
|
self.series = ""
|
|
self.singleFile = False
|
|
self.fileConunt = 0
|
|
self.size = 0
|
|
self.firstFile = None
|
|
self.changed = None
|
|
self.metadata = {}
|
|
self.artist = ""
|
|
self.genre = ""
|
|
self.composer = ""
|
|
self.name = ""
|
|
self.comment = ""
|
|
self.files: list[Path] = []
|
|
self.bookPath = book_path
|
|
self.library_root_path = library_root_path
|
|
|
|
def init_book(self) -> None:
|
|
self.files = sorted(
|
|
(
|
|
file
|
|
for file in self.bookPath.iterdir()
|
|
if file.is_file() and file.suffix.lower() in AUDIO_SUFFIXES
|
|
),
|
|
key=self.extract_track_number,
|
|
)
|
|
epubs = list(self.bookPath.glob("*.epub"))
|
|
self.epub_file = epubs[0] if len(epubs) == 1 else None
|
|
pdfs = list(self.bookPath.glob("*.pdf"))
|
|
self.pdf_file = pdfs[0] if len(pdfs) == 1 else None
|
|
|
|
self.files_extra = [
|
|
file
|
|
for file in self.bookPath.glob("*") if file.suffix.lower() not in AUDIO_SUFFIXES
|
|
]
|
|
self.firstFile = self.get_first_file()
|
|
self.size = sum(
|
|
file.stat().st_size
|
|
for pattern in AUDIO_RE_SUFFIXES
|
|
for file in self.bookPath.glob(pattern)
|
|
)
|
|
self.fileConunt = len(self.files)
|
|
self.singleFile = self.fileConunt == 1
|
|
# Metadata
|
|
self.load_metadata()
|
|
logger.info(f"Found {self.bookPath.name}, {self.fileConunt} files, {get_empty_keys(self.metadata)}")
|
|
self.changed = False
|
|
|
|
def __repr__(self):
|
|
return (
|
|
f"Audiobook("
|
|
f"name={self.name!r}, "
|
|
f"artist={self.artist!r}, "
|
|
f"composer={self.composer!r}, "
|
|
f"path={str(self.bookPath)!r}),"
|
|
f"size={self.size!r}, "
|
|
f"fileConunt={self.fileConunt!r}, "
|
|
)
|
|
|
|
def get_first_file(self) -> Path:
|
|
return self.files[0]
|
|
|
|
def write_fields_from_metadata(self) -> None:
|
|
self.name = self.metadata["album"]
|
|
self.composer = self.metadata["composer"]
|
|
self.comment = self.metadata["comment"]
|
|
self.genre = self.metadata["genre"]
|
|
if "#" in self.metadata["grouping"]:
|
|
self.series, self.series_num = self.metadata["grouping"].rsplit("#", 1)
|
|
self.series = self.series.strip()
|
|
self.series_and_num = self.metadata["grouping"]
|
|
artist = self.metadata.get("artist", "").strip()
|
|
self.artist = artist or self.metadata.get("album artist")
|
|
self.description = self.metadata["description"]
|
|
|
|
def write_metadata_from_fields(self):
|
|
self.metadata["album"] = self.name
|
|
self.metadata["composer"] = self.composer
|
|
self.metadata["genre"] = self.genre
|
|
self.metadata["grouping"] = self.series_and_num
|
|
self.metadata["comment"] = self.comment
|
|
self.metadata["description"] = self.description
|
|
self.metadata["artist"] = self.artist
|
|
self.metadata["album artist"] = self.artist
|
|
|
|
def extract_track_number(self, file: Path) -> int:
|
|
stem = file.stem
|
|
title = self.name
|
|
# Remove title if it appears at the beginning
|
|
if title:
|
|
escaped_title = re.escape(title)
|
|
stem = re.sub(
|
|
rf"^{escaped_title}(?:\s*[-_:]\s*)?",
|
|
"",
|
|
stem,
|
|
flags=re.IGNORECASE,
|
|
).strip()
|
|
|
|
patterns = [
|
|
# 01 Title, 02GLCK~1.WMA
|
|
r"^(\d{2,3})(?=\D|$)",
|
|
# CD1-01, CD 1 - 01, CD1 - 01.
|
|
r"CD\s*(\d+)\s*-\s*(\d{1,3})",
|
|
# 018-01, 018 - 01, 138-01, 01-01
|
|
r"^(\d{1,3})\s*-\s*(\d{1,3})(?:\D|$)",
|
|
# Track 01
|
|
r"Track\s+(\d+)$",
|
|
# 01 - Kapitel 01
|
|
r"^(\d+)\s*-",
|
|
# Rubinrot 101, 102, 201, 202
|
|
r".*?(\d)(\d{2})$",
|
|
# A Thousand Ships - 007
|
|
r".*?-\s*(\d+)\s*$",
|
|
]
|
|
for pattern in patterns:
|
|
m = re.search(pattern, stem, re.IGNORECASE)
|
|
if not m:
|
|
continue
|
|
|
|
groups = m.groups()
|
|
if len(groups) == 1:
|
|
return int(groups[0])
|
|
|
|
if len(groups) == 2 and "CD" in pattern:
|
|
disc = int(groups[0])
|
|
track = int(groups[1])
|
|
return (disc - 1) * 1000 + track
|
|
|
|
if len(groups) == 2 and stem.startswith(groups[0]):
|
|
disc = int(groups[0])
|
|
track = int(groups[1])
|
|
|
|
if track < 100:
|
|
return disc * 1000 + track
|
|
|
|
if len(groups) == 2 and len(groups[0]) == 1:
|
|
disc = int(groups[0])
|
|
track = int(groups[1])
|
|
return disc * 1000 + track
|
|
|
|
return int(groups[-1])
|
|
|
|
return 0
|
|
|
|
def load_metadata(self):
|
|
self.metadata = get_metadata_all(self.get_first_file())
|
|
self.write_fields_from_metadata()
|
|
logger.trace(
|
|
f"Load metadata {self.metadata}"
|
|
)
|
|
|
|
def fix_artist_dir(self):
|
|
rel_path = self.bookPath.relative_to(self.library_root_path).parts
|
|
if not self.artist:
|
|
logger.trace(f"No artist tag for {self.name}")
|
|
return
|
|
if len(rel_path) < 2:
|
|
desired_path,desired_name = self.generate_dir_from_metadata()
|
|
self.move(desired_path,desired_name)
|
|
logger.debug(f"Artist not visible in folder structure: {rel_path}")
|
|
|
|
def fix_artist_from_comment(self):
|
|
m = re.search(
|
|
r"(?:"
|
|
r"author(?:s)?:?|"
|
|
r"writer(?:s)?:?|"
|
|
r"co[- ]?author(?:s)?:?|"
|
|
r"co[- ]?writer(?:s)?:?|"
|
|
r"creator(?:s)?:?|"
|
|
r"contributor(?:s)?:?|"
|
|
r"composer(?:s)?:?|"
|
|
r"lyricist(?:s)?:?|"
|
|
r"songwriter(?:s)?:?|"
|
|
r"written\s+by:?|"
|
|
r"story\s+by:?|"
|
|
r"autorin(?:nen)?:?|"
|
|
r"autor\(en\):?|"
|
|
r"autor(?:en)?:?|"
|
|
r"text(?:e)?\s+von:?|"
|
|
r"geschrieben\s+von:?|"
|
|
r"verfasst\s+von:?|"
|
|
r"auteur(?:s)?:?|"
|
|
r"autore(?:i)?:?|"
|
|
r"autor(?:es)?:?|"
|
|
r"escritor(?:es|as)?:?"
|
|
r")\s*:?\s*([^\n.,;]+)",
|
|
self.comment,
|
|
re.IGNORECASE,
|
|
)
|
|
if not m:
|
|
logger.trace(f"No artist found in comment for {self.bookPath}")
|
|
return
|
|
self.changed = True
|
|
artist = re.sub(r"\s*\([^)]*\)", "", m.group(1).strip()).strip()
|
|
logger.trace(f"Remove {artist} from comment: {self.comment}")
|
|
self.comment = (self.comment[: m.start()] + self.comment[m.end() :]).strip(
|
|
" .,;\n"
|
|
)
|
|
self.comment = cleanup(self.comment)
|
|
if not self.artist:
|
|
self.artist = artist
|
|
self.changed = True
|
|
|
|
elif self.artist != artist:
|
|
logger.warning(
|
|
f"{red_tag("Action Required")} artist {artist} found not matching {self.artist} for {self.bookPath}"
|
|
)
|
|
self.action_required = True
|
|
else:
|
|
logger.debug(f"Artist {artist} matches file metadata for {self.name}")
|
|
|
|
def fix_artist_from_path(self):
|
|
rel_path = self.bookPath.relative_to(self.library_root_path).parts
|
|
|
|
if len(rel_path) < 2:
|
|
logger.trace(f"Artist not visible in folder structure: {rel_path}")
|
|
return
|
|
if not self.artist:
|
|
self.changed = True
|
|
|
|
self.artist = rel_path[0]
|
|
logger.info(f"[Updated] artist found in path for {rel_path}")
|
|
|
|
def fix_composer_from_comment(self):
|
|
m = re.search(
|
|
r"(?:read\s+by:?|"
|
|
r"spoken\s+by:?|"
|
|
r"narrated\s+by:?|"
|
|
r"gesprochen\s+von:?|"
|
|
r"gelesen\s+von)\s+([^\n.,;]+)",
|
|
self.comment,
|
|
re.IGNORECASE,
|
|
)
|
|
if not m:
|
|
logger.trace(f"No composer found in comment for {self.name}")
|
|
return
|
|
self.changed = True
|
|
self.comment = (self.comment[: m.start()] + self.comment[m.end() :]).strip(
|
|
" .,;\n"
|
|
)
|
|
self.comment = cleanup(self.comment)
|
|
if not self.composer or len(self.composer) < 5:
|
|
logger.info(f"[Updated] composer found in comment for {self.name}")
|
|
self.composer = m.group(1).strip()
|
|
|
|
def fix_composer_from_path(self):
|
|
rel_path = self.bookPath.relative_to(self.library_root_path)
|
|
m = re.search(
|
|
r"(?:read\s+by:?|"
|
|
r"spoken\s+by:?|"
|
|
r"narrated\s+by:?|"
|
|
r"gesprochen\s+von:?|"
|
|
r"gelesen\s+von)\s+([^\n.,;]+)",
|
|
str(rel_path),
|
|
re.IGNORECASE,
|
|
)
|
|
if not m:
|
|
logger.trace(f"No composer found in path for {self.bookPath}")
|
|
return
|
|
if not self.composer or len(self.composer) < 5:
|
|
self.changed = True
|
|
logger.info(f"[Updated] composer found in Path for {self.bookPath}")
|
|
self.composer = m.group(1).strip()
|
|
|
|
def fix_name(self):
|
|
if not self.name:
|
|
self.name = normalize_name(self.bookPath.name)
|
|
self.changed = True
|
|
logger.info(f"[Updated] name found in path for {self.bookPath}")#
|
|
return
|
|
name = normalize_name(self.name, self.series)
|
|
if name != self.name:
|
|
logger.info(f"[Updated] name by cleaning {self.name} to {name}")
|
|
self.name = name
|
|
self.changed = True
|
|
|
|
def fix_series(self):
|
|
if self.series_and_num:
|
|
logger.trace(f"{self.name} already has series: {self.series_and_num}")
|
|
rel_path = self.bookPath.relative_to(self.library_root_path).parts
|
|
number=""
|
|
if len(rel_path) <= 2:
|
|
logger.trace(f"No series found for {self.name}")
|
|
return
|
|
for i, pattern in enumerate(series_patterns):
|
|
match = re.match(pattern, rel_path[2], re.IGNORECASE)
|
|
if not match:
|
|
continue
|
|
if i == 0:
|
|
number = match.group(1)
|
|
elif i == 1:
|
|
number = match.group(2)
|
|
else:
|
|
number = match.group(2) or match.group(3)
|
|
break
|
|
number_str = ""
|
|
and_number =""
|
|
if not number:
|
|
logger.trace(f"No series number found for {self.name}")
|
|
else:
|
|
number_str = " #" + number
|
|
and_number = " and number"
|
|
self.series_num = number
|
|
self.series_and_num = rel_path[1] + number_str
|
|
self.series =rel_path[1]
|
|
self.changed = True
|
|
logger.info(f"[Updated] series{and_number} found for {self.name} to {self.series_and_num}")
|
|
return
|
|
|
|
def fix_genre(self):
|
|
self.genre = clean_genre(self.genre)
|
|
self.changed = True
|
|
|
|
def move(self, desired_path: Path, desired_name_stem: str, dry_run=False):
|
|
logger.trace(f"Moving {self.bookPath} to {desired_path}")
|
|
if dry_run:
|
|
return
|
|
desired_path.mkdir(parents=True, exist_ok=True)
|
|
padding = len(str(len(self.files)))
|
|
for number, file in enumerate(self.files, start=1):
|
|
if self.singleFile:
|
|
new_name = f"{desired_name_stem}{file.suffix.lower()}"
|
|
else:
|
|
new_name = f"{number:0{padding}d} - {desired_name_stem}{file.suffix}"
|
|
self.files[number-1] = file.rename(desired_path / new_name)
|
|
for file in self.files_extra:
|
|
if file.suffix.lower() in {".cue", ".log"} or file in {
|
|
self.epub_file,
|
|
self.pdf_file,
|
|
}:
|
|
filename = f"{desired_name_stem}{file.suffix.lower()}"
|
|
else:
|
|
filename = file.name
|
|
new_file = file.rename(desired_path / filename)
|
|
if file == self.epub_file:
|
|
self.epub_file = new_file
|
|
elif file == self.pdf_file:
|
|
self.pdf_file = new_file
|
|
|
|
if self.bookPath != desired_path:
|
|
remaining = list(self.bookPath.iterdir())
|
|
if not remaining:
|
|
self.bookPath.rmdir()
|
|
else:
|
|
logger.warning(f"{red_tag('Action Required')} old path not empty after move: "
|
|
f"{self.bookPath}: {remaining}"
|
|
)
|
|
self.bookPath = desired_path
|
|
self.files = sorted(self.files, key=self.extract_track_number)
|
|
self.firstFile = self.get_first_file()
|
|
logger.debug(f"Moving {self.bookPath} to {desired_path}")
|
|
|
|
def generate_dir_from_metadata(self):
|
|
composer =""
|
|
name = self.name
|
|
if self.composer:
|
|
composer = f" (by {self.composer})"
|
|
name = self.name + composer
|
|
desired_path = self.library_root_path / self.artist / name
|
|
if self.series_and_num:
|
|
series = self.series_and_num
|
|
number = ""
|
|
if "#" in self.series_and_num:
|
|
series = self.series_and_num.split("#")
|
|
number = f"{series[1]} - "
|
|
series = series[0].rstrip()
|
|
desired_path = self.library_root_path / self.artist / series / f"{number}{name}"
|
|
desired_name_stem = self.artist + " - " + self.name + composer + self.bookPath.suffix
|
|
return desired_path, desired_name_stem
|
|
|
|
def create_m4b_from_book(self, creat_m4b_no_delete: bool = False) -> None:
|
|
output_file_final = self.bookPath / f"{self.bookPath.name}.m4b"
|
|
if self.action_required:
|
|
logger.warning(f"Action required, resolve before creating m4b file")
|
|
return
|
|
if output_file_final.exists():
|
|
logger.info(f"{red_tag("SKIP")} {output_file_final}")
|
|
logger.info("Book already present")
|
|
return
|
|
|
|
logger.info(f"{blue_tag("create")} {output_file_final}")
|
|
|
|
with tempfile.TemporaryDirectory(dir=self.library_root_path.parent) as tmp:
|
|
tmpdir = Path(tmp)
|
|
|
|
output_file_tmp = tmpdir / f"{self.bookPath.name}.m4b"
|
|
concat_file = tmpdir / "concat.txt"
|
|
metadata_file = tmpdir / "chapters.txt"
|
|
|
|
# Inspect all inputs before deciding how to create chapters.
|
|
file_info = []
|
|
has_existing_chapters = False
|
|
|
|
for media in self.files:
|
|
chapters = get_chapters(media)
|
|
duration_ms = get_duration_ms(media)
|
|
|
|
file_info.append(
|
|
{
|
|
"media": media,
|
|
"chapters": chapters,
|
|
"duration_ms": duration_ms,
|
|
}
|
|
)
|
|
|
|
if chapters:
|
|
has_existing_chapters = True
|
|
|
|
if has_existing_chapters:
|
|
logger.debug(
|
|
"Existing chapter tracks found; preserving chapter metadata"
|
|
)
|
|
else:
|
|
logger.debug(
|
|
"No existing chapters found; generating chapters from filenames"
|
|
)
|
|
|
|
current_ms = 0
|
|
|
|
with (
|
|
concat_file.open("w", encoding="utf-8") as cf,
|
|
metadata_file.open("w", encoding="utf-8") as mf,
|
|
):
|
|
mf.write(";FFMETADATA1\n")
|
|
|
|
for key, value in self.metadata.items():
|
|
if key == "title":
|
|
mf.write(f"{ffmetadata_escape(key)}={ffmetadata_escape(self.metadata["album"])}\n")
|
|
else:
|
|
mf.write(f"{ffmetadata_escape(key)}={ffmetadata_escape(value)}\n")
|
|
for info in file_info:
|
|
media = info["media"]
|
|
media_name = info["media"].name
|
|
# Remove title
|
|
media_name = media_name.replace(self.name, "")
|
|
# Clean separators and whitespace
|
|
media_name = re.sub(r"[\s._-]+", " ", media_name).strip()
|
|
# Remove leading/trailing separators again if needed
|
|
media_name = re.sub(r"^[\s._-]+|[\s._-]+$", "", media_name)
|
|
info["media"].with_name(media_name)
|
|
chapters = info["chapters"]
|
|
duration_ms = info["duration_ms"]
|
|
|
|
cf.write(f"file '{ffmpeg_escape(media)}'\n")
|
|
|
|
if has_existing_chapters:
|
|
# Preserve actual chapter boundaries and titles.
|
|
for chapter in chapters:
|
|
start_ms = round(float(chapter["start_time"]) * 1000)
|
|
end_ms = round(float(chapter["end_time"]) * 1000)
|
|
|
|
tags = chapter.get("tags", {})
|
|
title = tags.get("title", media.stem)
|
|
|
|
mf.write("[CHAPTER]\n")
|
|
mf.write("TIMEBASE=1/1000\n")
|
|
mf.write(f"START={current_ms + start_ms}\n")
|
|
mf.write(f"END={current_ms + end_ms}\n")
|
|
mf.write(f"title={ffmetadata_escape(title)}\n")
|
|
else:
|
|
# No input contains chapters: one chapter per file.
|
|
mf.write("[CHAPTER]\n")
|
|
mf.write("TIMEBASE=1/1000\n")
|
|
mf.write(f"START={current_ms}\n")
|
|
mf.write(f"END={current_ms + duration_ms}\n")
|
|
mf.write(f"title={ffmetadata_escape(media.stem)}\n")
|
|
|
|
current_ms += duration_ms
|
|
|
|
codec, bitrate_k = get_codec_bitrate(self.firstFile)
|
|
codec_args = ["-c:a", "copy"]
|
|
if codec == "mp3":
|
|
codec_args = ["-c:a", "aac", "-b:a", bitrate_k]
|
|
elif codec == "flac":
|
|
codec_args = ["-c:a", "alac", "-b:a", bitrate_k]
|
|
elif codec not in {"aac", "alac"}:
|
|
codec_args = ["-c:a", "aac", "-b:a", bitrate_k]
|
|
cmd = [
|
|
"ffmpeg",
|
|
"-y",
|
|
"-f",
|
|
"concat",
|
|
"-safe",
|
|
"0",
|
|
"-i",
|
|
str(concat_file),
|
|
"-i",
|
|
str(metadata_file),
|
|
"-i",
|
|
str(self.firstFile),
|
|
# audio from concat
|
|
"-map",
|
|
"0:a",
|
|
# cover image from first media
|
|
"-map",
|
|
"2:v?",
|
|
# Metadata and chapters from our generated FFMETADATA file
|
|
"-map_metadata",
|
|
"1",
|
|
"-map_chapters",
|
|
"1",
|
|
*codec_args,
|
|
"-c:v",
|
|
"copy",
|
|
"-disposition:v:0",
|
|
"attached_pic",
|
|
str(output_file_tmp),
|
|
]
|
|
result = subprocess.run(
|
|
cmd,
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
logger.trace(result.stdout)
|
|
if result.returncode == 0:
|
|
output_file_tmp.rename(output_file_final)
|
|
for media in self.files:
|
|
if creat_m4b_no_delete:
|
|
continue
|
|
media.unlink()
|
|
self.files = [output_file_final]
|
|
self.firstFile = output_file_final
|
|
self.fileConunt = 1
|
|
self.singleFile = True
|
|
self.changed = False
|
|
self.size = Path(output_file_final).stat().st_size
|
|
logger.info(
|
|
f"{green_tag("SUCCESS")} - {self.bookPath.name} - delete media files"
|
|
)
|
|
else:
|
|
logger.error(
|
|
f"{red_tag("ERROR")} - {self.bookPath.name} - Code: {result.returncode}, Stderr: {result.stderr}"
|
|
)
|
|
|
|
|
|
|
|
def write_metadata(self, keys=None, dry_run=False):
|
|
if self.action_required:
|
|
logger.warning(f"Action required, resolve before writing metadata file")
|
|
return
|
|
self.write_metadata_from_fields()
|
|
if keys is None:
|
|
keys = self.metadata.keys()
|
|
values = self.metadata.values()
|
|
else:
|
|
values = [self.metadata[key] for key in keys ]
|
|
for file in self.files:
|
|
write_metadata(file, keys, values,dry_run)
|
|
self.changed = False
|
|
|
|
class AudiobookLibrary:
|
|
def __init__(
|
|
self, library_path: Path, sql_database_path: Path, dry_run: bool, logfilename: str = "audiobook_library.log"
|
|
):
|
|
|
|
self.sql_database_instance = self.SqlDatabase(sql_database_path)
|
|
self.dry_run = dry_run
|
|
logging.setLoggerClass(CustomLogger)
|
|
logger: CustomLogger = logging.getLogger(__name__) # type: ignore[assignment]
|
|
logger.info("Started")
|
|
self.libraryPath: Path = Path(library_path)
|
|
self.audiobooks: list[Audiobook] = []
|
|
self._by_path = {}
|
|
self._by_name = defaultdict(list)
|
|
self._by_artist = defaultdict(list)
|
|
self._by_composer = defaultdict(list)
|
|
|
|
self.load_audiobooks_from_dir()
|
|
self.size = sum(
|
|
audiobook.size for audiobook in self.audiobooks
|
|
) # size in bytes
|
|
|
|
class SqlDatabase:
|
|
def __init__(self, sql_database_path: Path):
|
|
self.tables = []
|
|
self.data = {}
|
|
self.db = None
|
|
self.path = sql_database_path
|
|
self.books = []
|
|
|
|
def connect(self):
|
|
if not self.path.is_file():
|
|
logger.error(f"{self.path} does not exist")
|
|
raise FileExistsError
|
|
self.db = sqlite3.connect(self.path)
|
|
self.db.row_factory = sqlite3.Row
|
|
return True
|
|
|
|
def cursor(self):
|
|
return self.db.cursor()
|
|
|
|
def commit(self):
|
|
self.db.commit()
|
|
|
|
def close(self):
|
|
self.db.close()
|
|
|
|
def init(self):
|
|
if not self.connect():
|
|
return False
|
|
|
|
return self.load_database_data()
|
|
|
|
def load_database_data(self) -> bool:
|
|
self.tables = self.db.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
|
).fetchall()
|
|
for table in self.tables:
|
|
table_name = table["name"]
|
|
rows = []
|
|
for row in self.db.execute(f'SELECT * FROM "{table_name}"'):
|
|
record = dict(row)
|
|
# Auto-decode JSON columns
|
|
for key, value in record.items():
|
|
if isinstance(value, str) and value[:1] in "[{":
|
|
try:
|
|
record[key] = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
rows.append(record)
|
|
|
|
self.data[table_name] = rows
|
|
self.books = self.data["books"]
|
|
return True
|
|
|
|
def update_books(self):
|
|
for book in self.books:
|
|
self._save_book(book)
|
|
|
|
def _save_book(self, book):
|
|
book = book.copy()
|
|
|
|
for key, value in book.items():
|
|
if isinstance(value, (list, dict)):
|
|
book[key] = json.dumps(value, ensure_ascii=False)
|
|
|
|
columns = [c for c in book if c != "id"]
|
|
|
|
sql = f"""
|
|
UPDATE books
|
|
SET {", ".join(f'"{c}" = ?' for c in columns)}
|
|
WHERE id = ?
|
|
"""
|
|
|
|
values = [book[c] for c in columns]
|
|
values.append(book["id"])
|
|
|
|
self.db.execute(sql, values)
|
|
|
|
# commit once after all updates
|
|
|
|
def fix_genres(self):
|
|
for book in self.books:
|
|
genres = book.get("genres")
|
|
if not genres or not any(g.strip() for g in genres):
|
|
continue
|
|
new_genres = []
|
|
for genre in genres:
|
|
genre = clean_genre(genre)
|
|
if "," in genre:
|
|
genre = genre.split(",")
|
|
elif " - " in genre:
|
|
genre = genre.split(" - ")
|
|
elif "Historical Adventure Thriller Mystery" in genre:
|
|
genre = genre.split(" ")
|
|
|
|
if isinstance(genre, str):
|
|
if len(genre) > 50 or (5 > len(genre) > 0):
|
|
genre = ""
|
|
logger.trace(f"Too long or too short: {genre}")
|
|
genre = genre.strip()
|
|
genre = genre.lstrip()
|
|
new_genres.append(genre)
|
|
elif isinstance(genre, list):
|
|
if len(genres) > 1:
|
|
genres = [g for g in genres if g]
|
|
new_genres.extend(genre)
|
|
|
|
new_genres = sorted(set(new_genres))
|
|
list_difference = False
|
|
if len(new_genres) < len(genres):
|
|
list_difference = True
|
|
for old, new in zip(genres, new_genres):
|
|
if old != new:
|
|
list_difference = True
|
|
break
|
|
|
|
if not list_difference:
|
|
logger.trace("No difference: Dont Commit to Database")
|
|
continue
|
|
|
|
logger.debug(f"{genres} -> {new_genres}")
|
|
book["genres"] = new_genres
|
|
|
|
def fix_comments(self):
|
|
for book in self.books:
|
|
for file in book.get("audioFiles"):
|
|
comment = file.get("metaTags").get("tagComment")
|
|
if comment and len(comment) < 40:
|
|
logger.debug(f"Fix short comment for {book.get("title")}")
|
|
comment = ""
|
|
file["metaTags"]["tagComment"] = comment
|
|
|
|
def add(self, book: Audiobook):
|
|
self.audiobooks.append(book)
|
|
|
|
self._by_path[book.bookPath] = book
|
|
self._by_name[book.name].append(book)
|
|
|
|
if book.artist:
|
|
self._by_artist[book.artist].append(book)
|
|
if book.composer:
|
|
self._by_composer[book.composer].append(book)
|
|
|
|
def print_name(self, books):
|
|
if not isinstance(books, list):
|
|
books = [books]
|
|
for book in books:
|
|
if book.name:
|
|
logger.info(book.name)
|
|
else:
|
|
logger.info(f"No name in {book.bookPath.name}")
|
|
|
|
def get_by_path(self, path):
|
|
return self._by_path.get(Path(path))
|
|
|
|
def get_by_name(self, name):
|
|
return self._by_name.get(name, [])
|
|
|
|
def get_by_artist(self, artist):
|
|
return self._by_artist.get(artist, [])
|
|
|
|
def get_by_composer(self, composer):
|
|
return self._by_artist.get(composer, [])
|
|
|
|
def find_path_contains(self, path) -> list[Audiobook]:
|
|
return [
|
|
book
|
|
for book in self.audiobooks
|
|
if book.bookPath == path or book.bookPath.is_relative_to(path)
|
|
]
|
|
|
|
def find_name_contains(self, text):
|
|
text = text.lower()
|
|
return [book for book in self.audiobooks if text in book.name.lower()]
|
|
|
|
def find_artist_contains(self, text):
|
|
text = text.lower()
|
|
return [
|
|
book
|
|
for book in self.audiobooks
|
|
if book.artist and text in book.artist.lower()
|
|
]
|
|
|
|
def load_audiobooks_from_dir(self):
|
|
"""
|
|
A book directory is any directory containing more than one
|
|
audiobook file (.mp3, .m4a, .m4b)
|
|
"""
|
|
media_files = [
|
|
file
|
|
for file in self.libraryPath.rglob("*")
|
|
if file.is_file() and file.suffix.lower() in AUDIO_SUFFIXES
|
|
]
|
|
dirs = {file.parent for file in media_files}
|
|
for directory in dirs:
|
|
self.add(Audiobook(directory,self.libraryPath))
|
|
for audiobook in self.audiobooks:
|
|
audiobook.init_book()
|
|
|
|
def load_metadata(self):
|
|
for audiobook in self.audiobooks:
|
|
audiobook.load_metadata()
|
|
|
|
def write_metadata(self, keys=None):
|
|
for audiobook in self.audiobooks:
|
|
if audiobook.changed:
|
|
logger.trace(f"Writing metadata: {audiobook.name}")
|
|
audiobook.write_metadata(keys,dry_run=self.dry_run)
|
|
|
|
def fix_artist_dir(self):
|
|
for book in self.audiobooks:
|
|
book.fix_artist_dir()
|
|
|
|
def create_structured_directories(self):
|
|
for book in self.audiobooks:
|
|
desired_path, desired_name = book.generate_dir_from_metadata()
|
|
book.move(desired_path, desired_name)
|
|
|
|
def fix_artists(self):
|
|
for book in self.audiobooks:
|
|
book.fix_artist_from_comment()
|
|
book.fix_artist_from_path()
|
|
|
|
def fix_composers(self):
|
|
for book in self.audiobooks:
|
|
book.fix_composer_from_comment()
|
|
book.fix_composer_from_path()
|
|
|
|
def fix_names(self):
|
|
for book in self.audiobooks:
|
|
book.fix_name()
|
|
|
|
def fix_comments(self):
|
|
for book in self.audiobooks:
|
|
if book.comment and len(book.comment) < 40:
|
|
logger.info(f"Remove short comment of {book.name}: {book.comment}")
|
|
book.comment = ""
|
|
book.changed = True
|
|
|
|
def fix_genres(self):
|
|
for book in self.audiobooks:
|
|
book.fix_genre()
|
|
|
|
def fix_series(self):
|
|
for book in self.audiobooks:
|
|
book.fix_series()
|
|
|
|
def create_m4b_from_dir(
|
|
self, path: Path = Path(""), name: str = "", keep_originals: bool = False
|
|
) -> None:
|
|
audiobooks = []
|
|
if path:
|
|
audiobooks = self.find_path_contains(path)
|
|
if name:
|
|
audiobooks.extend(self.find_name_contains(name))
|
|
if not name and not path:
|
|
audiobooks = self.audiobooks
|
|
|
|
jobs = [audiobook for audiobook in audiobooks if not audiobook.singleFile]
|
|
|
|
for audiobook in jobs:
|
|
logger.debug(f"Create m4b file for {audiobook.name}")
|
|
|
|
if not self.dry_run:
|
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
futures = {
|
|
executor.submit(
|
|
audiobook.create_m4b_from_book,
|
|
keep_originals,
|
|
): audiobook
|
|
for audiobook in jobs
|
|
}
|
|
|
|
for future in as_completed(futures):
|
|
audiobook = futures[future]
|
|
try:
|
|
future.result()
|
|
except Exception:
|
|
logger.exception(f"Failed to create m4b for {audiobook.name}")
|
|
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Audiobook Directory Processing")
|
|
|
|
parser.add_argument("library", type=Path, help="Library path")
|
|
|
|
|
|
|
|
parser.add_argument(
|
|
"-ll",
|
|
"--log-level",
|
|
type=str.upper,
|
|
default="INFO",
|
|
choices=["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
|
help="Set log level",
|
|
)
|
|
m4b_group = parser.add_argument_group("M4B creation")
|
|
|
|
m4b_group.add_argument(
|
|
"--create-m4b",
|
|
action="store_true",
|
|
help="Create M4B files from multi-file audiobooks",
|
|
)
|
|
|
|
m4b_group.add_argument(
|
|
"--keep-originals",
|
|
action="store_true",
|
|
help="Do not delete source files after M4B creation",
|
|
)
|
|
|
|
m4b_group.add_argument(
|
|
"--name-filter",
|
|
metavar="NAME",
|
|
default="",
|
|
help="Process only books whose name contains NAME",
|
|
)
|
|
|
|
m4b_group.add_argument(
|
|
"--path-filter",
|
|
metavar="PATH",
|
|
default="",
|
|
help="Process only books whose path contains PATH",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
action="store_true",
|
|
help="Dry run only",
|
|
)
|
|
parser.add_argument(
|
|
"--sql-database",
|
|
type=Path,
|
|
default=Path("/docker/audiobookshelf/config/absdatabase.sqlite"),
|
|
help="SQL Database Path",
|
|
)
|
|
parser.add_argument(
|
|
"-fd"
|
|
"--fix-db",
|
|
action="store_true",
|
|
help="Fix the Database",
|
|
)
|
|
parser.add_argument(
|
|
"-m",
|
|
"--move",
|
|
action="store_true",
|
|
help="Create structured directories for all books: /artist/series/book(read by X)/file.m4b",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
if "--sql-database" in sys.argv:
|
|
args.fix_db = True
|
|
|
|
setup_logging(args.log_level, "audiobook_library")
|
|
library_dir = args.library.expanduser()
|
|
library_dir = library_dir.resolve()
|
|
if not library_dir.exists():
|
|
logger.error(f"Library directory does not exist: {library_dir}")
|
|
sys.exit(1)
|
|
|
|
database_path = args.sql_database.expanduser()
|
|
database_path = database_path.resolve()
|
|
if args.fix_db and not database_path.is_file():
|
|
logger.error(f"Database is not a file: {database_path}")
|
|
return
|
|
|
|
library = AudiobookLibrary(library_dir, database_path, args.dry_run)
|
|
if args.fix_db:
|
|
library.sql_database_instance.init()
|
|
|
|
library.fix_artist_dir()
|
|
library.fix_artists()
|
|
library.fix_composers()
|
|
library.fix_series()
|
|
library.fix_names()
|
|
library.fix_comments()
|
|
library.fix_genres()
|
|
|
|
if args.create_m4b:
|
|
library.create_m4b_from_dir(
|
|
name=args.name_filter,
|
|
path=args.path_filter,
|
|
keep_originals=args.keep_originals,
|
|
)
|
|
|
|
library.write_metadata()
|
|
|
|
if args.fix_db:
|
|
library.sql_database_instance.fix_genres()
|
|
library.sql_database_instance.fix_comments()
|
|
if not args.dry_run:
|
|
library.sql_database_instance.update_books()
|
|
library.sql_database_instance.commit()
|
|
library.sql_database_instance.close()
|
|
|
|
if args.move:
|
|
library.create_structured_directories()
|
|
|
|
print("Fin")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|