#!/home/lucas/scripts/.venv/bin/python3 import argparse import json import logging import re from datetime import datetime, timedelta import sys from pathlib import Path import filecmp import requests weight_jaccard = 0.5 weight_fuzzy = 1 - weight_jaccard MATCH_THRESHOLD = 0.85 from serverlibrary import ( VIDEO_SUFFIXES, CustomLogger, combined_score, safe_rename, score_match_fuzzy, score_match_jaccard, score_match_recall_jaccard, setup_logging, tmdb_get, tvdb_get, words, ) logging.setLoggerClass(CustomLogger) logger: CustomLogger = logging.getLogger(__name__) def normalize( text: str, title: str | None = None, year: int | str | None = None ) -> str: text = text.lower() # remove series/movie title if title: text = re.sub(re.escape(title.lower()), " ", text) # remove release year if year: text = re.sub(rf"\b{re.escape(str(year))}\b", " ", text) # remove file extension text = re.sub(r"\.[a-z0-9]{2,4}$", " ", text) # remove text inside square brackets text = re.sub(r"\[[^\]]*\]", " ", text) # remove episode numbers (S01E01) text = re.sub(r"\bs\d+e\d+\b", " ", text, flags=re.IGNORECASE) # remove common release tags text = re.sub( r"\b(?:1080p|720p|2160p|x264|x265|h264|h265|bluray|web[- ]?dl|webrip|dvdrip)\b", " ", text, flags=re.IGNORECASE, ) # normalize separators text = re.sub(r"[._-]", " ", text) # German normalization text = text.replace("ß", "ss") # "(123 456)" -> "(123)" text = re.sub(r"\((\d+)\s+\d+\)", r"(\1)", text) # "(123___456)" -> "(123)" text = re.sub(r"\((\d+)_+\d+\)", r"(\1)", text) # remove remaining punctuation text = re.sub(r"[^\w\s]", " ", text) # collapse whitespace text = re.sub(r"\s+", " ", text).strip() return text def files_equal(file1: Path, file2: Path) -> bool: if filecmp.cmp(file1, file2, shallow=False): return True logger.debug(f"Files {file1.name} and {file2.name} are not equal") return False def parse_series_name(series_name: str) -> tuple[str, str, str]: """ Universe (1987) [66757] -> ("Universe", 1987, 66757) Universe (1987) -> ("Universe", 1987, None) Universe -> ("Universe", None, None) """ tmdb_id = "" year = "" id_match = re.search(r"\[(\d+)]", series_name) if id_match: tmdb_id = id_match.group(1) year_match = re.search(r"\((\d{4})\)", series_name) if year_match: year = year_match.group(1) title = re.sub(r"\[\d+]", "", series_name) title = re.sub(r"\(\d{4}\)", "", title) title = title.strip() return title, year, tmdb_id def find_by_tvdb_id(tvdb_id: str, language="deu"): """ Try to determine whether a TMDB ID is a TV show or movie. Returns: ("tv", data) ("movie", data) (None, None) """ return tvdb_get( f"/series/{tvdb_id}/extended", {"language": language}, )["data"] def get_tmdb_last_updated(tmdb_id: str) -> str: latest = None data = tmdb_get(f"/tv/{tmdb_id}/changes") for change in data["changes"]: for item in change["items"]: ts = datetime.strptime( item["time"], "%Y-%m-%d %H:%M:%S UTC", ) if latest is None or ts > latest: latest = ts if latest is None: logger.debug("Use default lastChange of 1 week") latest = datetime.now() - timedelta(days=7) return latest.strftime("%Y-%m-%d %H:%M:%S") def find_by_tmdb_id(tmdb_id: str, language="de-DE", type="tv"): """ Try to determine whether a TMDB ID is a TV show or movie. Returns: ("tv", data) ("movie", data) (None, None) """ logger.trace(f"Find entry to id {tmdb_id}") result = tmdb_get( f"/{type}/{tmdb_id}", { "language": language, }, ) result["type"] = type result["year"] = ( f"{(result.get('first_air_date') or result.get('release_date') or '')[:4]}" ) result["tmdb_id"] = result["id"] result["tvdb_id"] = None result["origin_country"] = result["origin_country"][0] result["name"] = result.get("name") or result.get("title") return result def get_all_episodes_by_tvdbid(tvdb_id: str, language: str = "deu"): series_data = find_by_tvdb_id(tvdb_id) episodes_data = [] for season in series_data.get("seasons", []): logger.info(f"Requesting Season: {season.get('number')}") season_data = {} try: season_data = tvdb_get( f"/seasons/{season['id']}/extended", {"language": language}, )["data"] except requests.exceptions.RequestException: logger.error( "Failed to fetch season %s", season["id"], ) for episode in season_data.get("episodes", []): if language in episode["nameTranslations"] and ( not episode["name"] or series_data["originalLanguage"] != language ): episode["name"] = tvdb_get( f"/episodes/{episode['id']}/translations/{language}" )["data"]["name"] episodes_data.append( { "id": episode["id"], "season": episode["seasonNumber"], "episode": episode["number"], "name": episode["name"], } ) series_data["number_of_seasons"] = len(series_data.get("seasons")) series_data["number_of_episodes"] = len(episodes_data) network = series_data.get("originalNetwork") or series_data.get("latestNetwork") if isinstance(network, dict): network = network.get("name") series_data["network"] = network return series_data, episodes_data def get_all_episodes_by_tmdbid(tmdb_id, language: str = "de-DE"): def get_season(tmdb_id, season_number): return tmdb_get( f"/tv/{tmdb_id}/season/{season_number}", { "language": language, }, ) series_data = find_by_tmdb_id(tmdb_id) episodes_data = [] for season in series_data.get("seasons", []): season_number = season["season_number"] season_data = get_season(series_data.get("id"), season_number) for episode in season_data.get("episodes", []): episodes_data.append( { "id": episode["id"], "season": season_number, "episode": episode["episode_number"], "name": episode["name"], } ) network = series_data["networks"] if isinstance(network, list): network = network[0].get("name") series_data["network"] = network return series_data, episodes_data def get_tvdbid_by_tmdb(series_path: Path, language: str = "deu"): title, year, tmdb_id = parse_series_name(series_path.name) if tmdb_id: results = tvdb_get( f"/search/remoteid/{tmdb_id}", {"language": language}, )["data"] else: return get_tvdbid_by_name(series_path) result = None for res in results: if "series" not in res: continue if normalize(title) in res["series"].get("name", "").casefold() and str( res["series"].get("year", "") ) == str(year): result = res["series"] logger.info(result) if not result: logger.error(f"Results: {results}") raise Exception(f"Couldnt find results for Series {title} ") result["tvdb_id"] = result["id"] result["type"] = "tv" if result.get("translations"): result["original_name"] = result.get("translations").get( result.get("primary_language") ) result["tmdb_id"] = tmdb_id result["origin_country"] = result.get("originalCountry") status = result.get("status") result["name"] = result.get("name") or result.get("title") if isinstance(status, dict): result["status"] = status.get("name") return result def databes_entry_is_newer(last_updated: str, file: Path) -> bool: tvdb_time = datetime.strptime( last_updated, "%Y-%m-%d %H:%M:%S", ) file_time = datetime.fromtimestamp(file.stat().st_mtime) if tvdb_time > file_time: logger.debug( f"Database entry is newer than {file.name}: {file_time} > {tvdb_time}" ) return tvdb_time > file_time def get_tvdbid_by_name(series_path: Path, language: str = "deu"): title, year, tmdb_id = parse_series_name(series_path.name) logger.trace(f"Find series {title} by tvdb name search") params = { "query": title, "type": "series", "language": language, } if year: params["year"] = str(year) results = tvdb_get( "/search", params, )["data"] if not results: raise RuntimeError(f"Series not found: {series_path}") # Filter by year if available filtered = [] for result in results: result_tmdb_id = result.get("remote_ids") result["tmdb_id"] = None if not result_tmdb_id: continue for id in result_tmdb_id: if id.get("sourceName") == "TheMovieDB.com": result["tmdb_id"] = id.get("id") if tmdb_id and result["tmdb_id"] and result["tmdb_id"] == tmdb_id: filtered.append(result) if filtered: results = filtered result = results[0] result["lastUpdated"] = tvdb_get(f"/series/{result['tvdb_id']}")["data"][ "lastUpdated" ] result["id"] = result["tvdb_id"] result["type"] = "tv" if result.get("translations"): result["original_name"] = result.get("translations").get( result.get("primary_language") ) result["origin_country"] = "" if result.get("country"): result["origin_country"] = result.get("country")[0] status = result.get("status") result["name"] = result.get("name") or result.get("title") if isinstance(status, dict): result["status"] = status.get("name") if result["primary_language"] != language: result["name"] = result["translations"][language] result["overview"] = result["overviews"][language] return result def get_tmdbid_by_name(series_path: Path, type="tv", language="de-DE"): """ Search TMDB by title. media_type: tv | movie """ title, year, _ = parse_series_name(series_path.name) logger.trace(f"Find series {title} by tmdb name search") params = { "query": title, "language": "en-EN", } if year: if type == "tv": params["first_air_date_year"] = year else: params["year"] = year results = tmdb_get( f"/search/{type}", params, ) if not results: raise RuntimeError(f"Series not found: {series_path}") result = results["results"][0] result["type"] = type result["year"] = ( f"{(result.get('first_air_date') or result.get('release_date') or '')[:4]}" ) result["tvdb_id"] = None result["origin_country"] = result["origin_country"][0] result["name"] = result.get("name") or result.get("title") return result def print_results( media_data: dict, series_data: dict, episodes: list, extended: bool = False ): logger.info(f"TVDB ID : {media_data.get('tvdb_id')}") logger.info(f"TMDB ID : {media_data.get('tmdb_id')}") logger.info(f"Title {media_data.get('name')}") logger.info( f"Original : {media_data.get('original_name') or media_data.get('original_title')}" ) logger.info(f"Country : {media_data['origin_country']}") logger.info(f"Network : {series_data['network']}") if extended: logger.info(f"Year : {media_data['year']}") logger.info(f"Rating : {media_data.get('vote_average')}") logger.info(f"Votes : {media_data.get('vote_count')}") logger.info(f"Status : {media_data.get('status')}") logger.info(f"Overview : {media_data.get('overview')}") genres = ", ".join(g["name"] for g in series_data.get("genres", [])) logger.info(f"Genres : {genres}") logger.info( f"Poster : https://image.tmdb.org/t/p/original{media_data.get('poster_path', '')}" ) logger.info( f"Backdrop : https://image.tmdb.org/t/p/original{media_data.get('backdrop_path', '')}" ) if media_data.get("type") == "tv": logger.info(f"Seasons : {series_data.get('number_of_seasons')}") logger.info(f"Episodes : {series_data.get('number_of_episodes')}") def best_episode_title_match(filename, episodes): search_words = words(normalize(filename)) logger.debug(f"Search Words: {search_words}") best_score = 0.0 best_episode = None for episode in episodes: if not episode["name"] or episode["name"] == "Unkown": continue candidate_words = words(normalize(episode["name"])) fuzzy = score_match_fuzzy( search_words, candidate_words, ) jaccard = score_match_jaccard( search_words, candidate_words, ) recall_jaccard1 = score_match_recall_jaccard( search_words, candidate_words, ) recall_jaccard2 = score_match_recall_jaccard( candidate_words, search_words, ) score = combined_score( fuzzy, jaccard, weight_fuzzy=weight_fuzzy, weight_jaccard=weight_jaccard, ) score = max(score, recall_jaccard1, recall_jaccard2) # if "hyänen" in search_words and score > 0.6: # logger.warning( # f"SEARCH WORDS: {search_words}, CANDIDATE WORDS: {candidate_words}" # ) if score > best_score: logger.trace(f"{search_words}:\t\t{candidate_words},\t\t{score}") best_score = score best_episode = episode return best_episode, best_score def translate_series_folder(episodes, series_path, media_data: dict, args): series = series_path.name logger.info(f"Execute translate_series for {media_data['name']}") episode_map = {(ep["season"], ep["episode"]): ep for ep in episodes} file_list = [] new_file_list = [] for file in series_path.rglob("*"): if ( not file.is_file() or file.suffix.lower() not in VIDEO_SUFFIXES or "Season" not in file.parent.name ): continue logger.trace(f"Translate file: {file.name}") match_SE_name = re.search(r"S(\d{2}|\d{4})E(\d{2,3})", file.name, re.IGNORECASE) if not match_SE_name: logger.error(f"No season/episode number found: {file.name}") continue season = int(match_SE_name.group(1)) episode = int(match_SE_name.group(2)) episode_number = episode_map.get((season, episode)) if not episode_number: logger.warning(f"Episode not found: S{season:02d}E{episode:02d}") continue episode_season_number_match = re.match( r"^(.*?S(\d{2}|\d{4})E\d{2,3})\s*-\s*.*?\s*((?:\[[^\]]+\])+)?$", file.stem, re.IGNORECASE, ) if not episode_season_number_match: logger.warning(f"S{season:02d}E{episode:02d} not found") continue prefix = f"{media_data['name']} ({media_data['year']}) - S{season:02d}E{episode:02d}" # Andor (2022) - S01E07 quality = ( episode_season_number_match.group(3) or "" ) # [WEBRip-1080p][AAC 5.1][h265] title = episode_number["name"].replace("/", "_") if title == "_": title = "slash" new_name = ( f"{prefix} - {title}{(' ' + quality) if quality else ''}{file.suffix}" ) logger.trace(f"Prefix: {prefix}, Title: {title}, Quality: {quality}") logger.trace(f"{file.name} -> {new_name}") file_list.append(file) new_file_list.append(new_name) new_folder_name = ( f"{media_data['name']} ({media_data['year']}) [{media_data['tmdb_id']}]" ) if args.translate_series_folder: logger.trace(f"{series} -> {new_folder_name}") file_list.append(series_path) new_file_list.append(new_folder_name) else: logger.info(f"Dont translate Series folder: {series}") has_changes = False for file, new_file in zip(file_list, new_file_list): if file.name != new_file: has_changes = True break if not has_changes: logger.info("No changes necessary.") return for file, new_file in zip(file_list, new_file_list): if args.dry_run: logger.debug(f"{file.name} -> {new_file}") continue safe_rename(file, new_file, logger=logger) logger.info("All files renamed") return def reference_extra_episodes(episodes, series_path: Path, media_data: dict, args): logger.info(f"Execute reference_extra_episodes for {media_data['name']}") extra_episodes_path = series_path / "Extra Episodes" for file in extra_episodes_path.glob("*"): if not file.is_file() or file.suffix.lower() not in VIDEO_SUFFIXES: continue logger.trace(f"Reference file: {file.name}") filename = re.sub(media_data["name"], "", file.stem) episode, score = best_episode_title_match( filename, episodes, ) if episode is None: continue if score < MATCH_THRESHOLD: logger.warning( f"LOW CONFIDENCE: {filename} {score:.2f}: {normalize(episode.get('name'))}" ) continue safe_title = re.sub( r'[<>:"/\\|?*]', "", episode["name"], ) new_name = ( f"{media_data['name']}" f" - S{episode['season']:02d}" f"E{episode['episode']:02d}" f" - {safe_title}" f"{file.suffix}" ) target = file.parent.parent / f"Season {episode['season']:02d}" / new_name if score < 1: logger.debug(f"{file.name} -> {new_name}, score={score:.3f}") elif score == 1: logger.trace(f"{file.name} -> {new_name}, score={score:.3f}") if args.dry_run: logger.debug(f"{file.name} -> {target}") continue # logger.info(f"{file.name} -> {target}, score={score:.3f}") safe_rename(file, target, logger=logger) def get_series_data(series_path: Path, database: str = "tvdb"): series_name, _, tmdb_id = parse_series_name(series_path.name) if database == "tvdb": # if tmdb_id: # media_data = get_tvdbid_by_tmdb(series_path) # else: media_data = get_tvdbid_by_name(series_path) elif database == "tmdb": if tmdb_id: media_data = find_by_tmdb_id(tmdb_id) else: media_data = get_tmdbid_by_name(series_path) media_data["lastUpdated"] = get_tmdb_last_updated(tmdb_id) else: raise ValueError(f"Wrong database name: {database}") buffer_file = series_path / f"__{series_name}_{database}_media_data.json" with open( buffer_file, "w", encoding="utf-8", ) as f: json.dump(media_data, f, ensure_ascii=False, indent=2) if ( (series_path / f"{series_name}_{database}_media_data.json").is_file() and files_equal( series_path / f"__{series_name}_{database}_media_data.json", series_path / f"{series_name}_{database}_media_data.json", ) and not databes_entry_is_newer( media_data["lastUpdated"], series_path / f"{series_name}_{database}_media_data.json", ) ): buffer_file.unlink() logger.info("Load from json:") with open( series_path / f"{series_name}_{database}_episodes.json", encoding="utf-8", ) as f: episodes = json.load(f) with open( series_path / f"{series_name}_{database}_media_data.json", encoding="utf-8", ) as f: media_data = json.load(f) with open( series_path / f"{series_name}_{database}_series_data.json", encoding="utf-8", ) as f: series_data = json.load(f) else: if database == "tvdb": logger.info("Query from TVDB") series_data, episodes = get_all_episodes_by_tvdbid(media_data["id"]) elif database == "tmdb": logger.info("Query from TMDB") series_data, episodes = get_all_episodes_by_tmdbid(media_data["id"]) logger.info("Dump to json") safe_rename( buffer_file, Path(f"{series_name}_{database}_media_data.json"), overwrite=True, logger=logger, ) with open( series_path / f"{series_name}_{database}_series_data.json", "w", encoding="utf-8", ) as f: json.dump(series_data, f, ensure_ascii=False, indent=2) with open( series_path / f"{series_name}_{database}_episodes.json", "w", encoding="utf-8", ) as f: json.dump(episodes, f, ensure_ascii=False, indent=2) return series_data, episodes, media_data def process_series(series_path: Path, args): series_name = series_path.name logger.debug(f"Pricess series with name : {series_name}") series_data, episodes, media_data = get_series_data(series_path, args.database) logger.info( f"Found series name from TVDB: {media_data['name']}, ID: {media_data['id']}, Year: {media_data.get('year')}" ) print_results(media_data, series_data, episodes, extended=args.extended) reference_extra_episodes(episodes, series_path, media_data, args) translate_series_folder(episodes, series_path, media_data, args) def main(): parser = argparse.ArgumentParser() parser.add_argument( "path", help='Either a series folder "Maya the Bee (1975) [10620]" or a root folder containing such folders', ) parser.add_argument( "-ll", "--log-level", type=str.upper, default="INFO", choices=["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], help="Set log level", ) parser.add_argument( "-db", "--database", type=str, default="tvdb", help="Set which database to use for queries. dafault: TvDB", ) parser.add_argument( "-lsd--leave-series-dir", dest="translate_series_folder", action="store_false", help="Dont Translate Series Folder", ) parser.add_argument( "-e", "--extended", action="store_true", help="Print Extended Infos", ) parser.add_argument( "--dry-run", action="store_true", help="Dry run only", ) args = parser.parse_args() setup_logging(args.log_level, "sonarr_translation_tool") input_path = Path(args.path).expanduser().resolve() if not input_path.is_dir(): parser.error(f"Directory not found: {input_path}") logger.info(f"Pricess folder: {input_path}") # Input is already a series folder if re.match(r"^(.*?)\s*\((\d{4})\)(?:\s*\[(\d*)])?$", input_path.name): logger.info(f"Process series: {input_path.name}") process_series(input_path, args) # Input is a root folder containing series folders else: for series_path in sorted(input_path.iterdir()): if series_path.is_dir(): process_series(series_path, args) sys.exit(0) if __name__ == "__main__": main()