Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Utilize AniDb online cache #2037

Open
wants to merge 3 commits into
base: nightly
Choose a base branch
from
Open

Utilize AniDb online cache #2037

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d92f5bb
Utilize AniDb online cache
salomj May 10, 2024
c65cd6d
Make language case-insensitive
salomj May 10, 2024
0a0fb74
Merge
salomj Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
227 changes: 190 additions & 37 deletions modules/anidb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
builders = ["anidb_id", "anidb_relation", "anidb_popular", "anidb_tag"]
base_url = "https://anidb.net"
api_url = "http://api.anidb.net:9001/httpapi"
cache_url = "https://raw.githubusercontent.com/notseteve/AnimeAggregations/main/anime"
urls = {
"anime": f"{base_url}/anime",
"popular": f"{base_url}/latest/anime/popular/?h=1",
Expand All @@ -16,12 +17,150 @@
"login": f"{base_url}/perl-bin/animedb.pl"
}
weights = {"anidb": 1000, "anidb_3_0": 600, "anidb_2_5": 500, "anidb_2_0": 400, "anidb_1_5": 300, "anidb_1_0": 200, "anidb_0_5": 100}
LANGUAGE_MATCHER = {
"AF" : "AFRIKAANS",
"SQ" : "ALBANIAN",
"AL" : "ALBANIAN",
"AR" : "ARABIC",
"BN" : "BENGALI",
"BD" : "BENGALI",
"BS" : "BOSNIAN",
"BG" : "BULGARIAN",
"MY" : "BURMESE",
"ZH" : "CHINESE",
"ZH-NAN" : "CHINESE_TAIWAN",
"ZH-HANT" : "CHINESE_TRADITIONAL",
"X-ZHT" : "CHINESE_TRANSLITERATED",
"ZH-HANS" : "CHINESE_SIMPLIFIED",
"ZH-CMN" : "CHINESE_SIMPLIFIED",
"HR" : "CROATIAN",
"CS" : "CZECH",
"DA" : "DANISH",
"NL" : "DUTCH",
"EN" : "ENGLISH",
"EO" : "ESPERANTO",
"ET" : "ESTONIAN",
"FIL" : "FILIPINO",
"FI" : "FINNISH",
"FR" : "FRENCH",
"KA" : "GEORGIAN",
"DE" : "GERMAN",
"EL" : "GREEK",
"GRC" : "GREEK_ANCIENT",
"HT" : "HAITIAN",
"HE" : "HEBREW",
"HI" : "HINDI",
"HU" : "HUNGARIAN",
"IS" : "ICELANDIC",
"ID" : "INDONESIAN",
"IT" : "ITALIAN",
"JA" : "JAPANESE",
"X-JAT" : "JAPANESE_TRANSLITERATED",
"JV" : "JAVANESE",
"KO" : "KOREAN",
"X-KOT" : "KOREAN_TRANSLITERATED",
"LA" : "LATIN",
"LV" : "LATVIAN",
"LT" : "LITHUANIAN",
"MS" : "MALAY",
"MN" : "MONGOLIAN",
"NE" : "NEPALI",
"NO" : "NORWEGIAN",
"FA" : "PERSIAN",
"PL" : "POLISH",
"PT" : "PORTUGUESE",
"PT-BR" : "PORTUGUESE_BRAZIL",
"RO" : "ROMANIAN",
"RU" : "RUSSIAN",
"SR" : "SERBIAN",
"SI" : "SINHALA",
"SK" : "SLOVAK",
"SL" : "SLOVENIAN",
"ES" : "SPANISH",
"ES-PV" : "SPANISH_BASQUE",
"ES-CT" : "SPANISH_CATALAN",
"ES-CA" : "SPANISH_CATALAN",
"ES-GA" : "SPANISH_GALICIA",
"ES-419" : "SPANISH_LATIN",
"SV" : "SWEDISH",
"TL" : "TAGALOG",
"TA" : "TAMIL",
"TT" : "TATAR",
"TE" : "TELUGU",
"TH" : "THAI",
"X-THT" : "THAI_TRANSLITERATED",
"TR" : "TURKISH",
"UK" : "UKRAINIAN",
"UR" : "URDU",
"VI" : "VIETNAMESE",
"X-UNK" : "UNKNOWN",
"X-OTHER" : "OTHER"
}

class AniDBObj:
def __init__(self, anidb, anidb_id, data):
def __init__(self, anidb, anidb_id, data, from_json=False):
self._anidb = anidb
self.anidb_id = anidb_id
self._data = data
self.main_title = None
self.titles = []
self.official_title = None
self.studio = None
self.rating = None
self.average = None
self.score = None
self.released = None
self.tags = {}
self.mal_id = None
self.imdb_id = None
self.tmdb_id = None
self.tmdb_type = None

def _parseJSON(self):
self.all_titles = data["titles"]
for title in self.all_titles:
if title["type"] == "MAIN":
self.main_title = title["title"]
if title["type"] == "OFFICIAL":
self.titles.append(title)
if title["type"] == "OFFICIAL" and self._anidb.language and (title["language"] == self._anidb.language.upper() or title["language"] == language_matcher[self._anidb.language.upper()]):
self.official_title = title["title"]
if not self.official_title:
self.official_title = self.main_title

if "creators" in data:
for creator_id, creators in data["creators"].items():
for creator in creators:
if creator["type"] == "ANIMATION_WORK":
self.studio = creator["name"]

if "ratings" in data and "PERMANENT" in data["ratings"]:
self.rating = float(data["ratings"]["PERMANENT"]["rating"])
if "ratings" in data and "TEMPORARY" in data["ratings"]:
self.average = float(data["ratings"]["TEMPORARY"]["rating"])
if "ratings" in data and "REVIEW" in data["ratings"]:
self.score = float(data["ratings"]["REVIEW"]["rating"])
if "start_date" in data:
self.released = datetime.strptime(data["start_date"], "%Y-%m-%d")

if "tags" in data:
for tag_id, tag in data["tags"].items():
if "info_box" in tag and (tag["info_box"] or tag["info_box"] == "true"):
self.tags[tag["name"]] = 1001
elif "weight" in tag:
self.tags[tag["name"]] = float(tag["weight"])

if "resources" in data and "MAL" in data["resources"]:
self.mal_id = int(data["resources"]["MAL"][0])
if "resources" in data and "IMDB" in data["resources"]:
self.imdb_id = data["resources"]["IMDB"][0]
if "resources" in data and "TMDB" in data["resources"]:
if data["resources"]["TMDB"][0].startswith("movie/"):
self.tmdb_id = int(data["resources"]["TMDB"][0][6:])
self.tmdb_type = "movie"
elif data["resources"]["TMDB"][0].startswith("tv/"):
self.tmdb_id = int(data["resources"]["TMDB"][0][3:])
self.tmdb_type = "show"

def _parse(attr, xpath, is_list=False, is_dict=False, is_int=False, is_float=False, is_date=False, fail=False):
try:
Expand Down Expand Up @@ -63,30 +202,35 @@ def _parse(attr, xpath, is_list=False, is_dict=False, is_int=False, is_float=Fal
else:
return None

self.main_title = _parse("main_title", "//anime/titles/title[@type='main']/text()", fail=True)
self.titles = _parse("titles", "//anime/titles/title[@type='official']", is_dict=True)
self.official_title = self.titles[self._anidb.language] if self._anidb.language in self.titles else self.main_title
self.studio = _parse("studio", "//anime/creators/name[@type='Animation Work']/text()")
self.rating = _parse("rating", "//anime/ratings/permanent/text()", is_float=True)
self.average = _parse("average", "//anime/ratings/temporary/text()", is_float=True)
self.score = _parse("score", "//anime/ratings/review/text()", is_float=True)
self.released = _parse("released", "//anime/startdate/text()", is_date=True)
self.tags = _parse("tags", "//anime/tags/tag", is_dict=True)
self.mal_id = _parse("mal_id", "//anime/resources/resource[@type='2']/externalentity/identifier/text()", is_int=True)
self.imdb_id = _parse("imdb_id", "//anime/resources/resource[@type='43']/externalentity/identifier/text()")
if isinstance(data, dict):
self.tmdb_id = _parse("tmdb_id", "", is_int=True)
self.tmdb_type = _parse("tmdb_type", "")
else:
tmdb = _parse("tmdb", "//anime/resources/resource[@type='44']/externalentity/identifier/text()", is_list=True)
self.tmdb_id = None
self.tmdb_type = None
for i in tmdb:
try:
self.tmdb_id = int(i)
except ValueError:
self.tmdb_type = i
def _parseXML(self):
self.main_title = _parse("main_title", "//anime/titles/title[@type='main']/text()", fail=True)
self.titles = _parse("titles", "//anime/titles/title[@type='official']", is_dict=True)
self.official_title = self.titles[self._anidb.language] if self._anidb.language in self.titles else self.main_title
self.studio = _parse("studio", "//anime/creators/name[@type='Animation Work']/text()")
self.rating = _parse("rating", "//anime/ratings/permanent/text()", is_float=True)
self.average = _parse("average", "//anime/ratings/temporary/text()", is_float=True)
self.score = _parse("score", "//anime/ratings/review/text()", is_float=True)
self.released = _parse("released", "//anime/startdate/text()", is_date=True)
self.tags = _parse("tags", "//anime/tags/tag", is_dict=True)
self.mal_id = _parse("mal_id", "//anime/resources/resource[@type='2']/externalentity/identifier/text()", is_int=True)
self.imdb_id = _parse("imdb_id", "//anime/resources/resource[@type='43']/externalentity/identifier/text()")
if isinstance(data, dict):
self.tmdb_id = _parse("tmdb_id", "", is_int=True)
self.tmdb_type = _parse("tmdb_type", "")
else:
tmdb = _parse("tmdb", "//anime/resources/resource[@type='44']/externalentity/identifier/text()", is_list=True)
self.tmdb_id = None
self.tmdb_type = None
for i in tmdb:
try:
self.tmdb_id = int(i)
except ValueError:
self.tmdb_type = i

if from_json:
_parseJSON(self)
else:
_parseXML(self)

class AniDB:
def __init__(self, requests, cache, data):
Expand Down Expand Up @@ -185,22 +329,31 @@ def _tag(self, tag, limit):
def get_anime(self, anidb_id, ignore_cache=False):
expired = None
anidb_dict = None
from_json = False
if self.cache and not ignore_cache:
anidb_dict, expired = self.cache.query_anidb(anidb_id, self.expiration)
if expired or not anidb_dict:
time_check = time.time()
if self._delay is not None:
while time_check - self._delay < 2:
time_check = time.time()
anidb_dict = self._request(api_url, params={
"client": self.client,
"clientver": self.version,
"protover": 1,
"request": "anime",
"aid": anidb_id
})
self._delay = time.time()
obj = AniDBObj(self, anidb_id, anidb_dict)
if not ignore_cache:
try:
anidb_dict = self.get_json(f"{cache_url}/{anidb_id}.json")
if anidb_dict:
from_json = True
except ValueError:
pass
if not from_json:
time_check = time.time()
if self._delay is not None:
while time_check - self._delay < 2:
time_check = time.time()
anidb_dict = self._request(api_url, params={
"client": self.client,
"clientver": self.version,
"protover": 1,
"request": "anime",
"aid": anidb_id
})
self._delay = time.time()
obj = AniDBObj(self, anidb_id, anidb_dict, from_json)
if self.cache and not ignore_cache:
self.cache.update_anidb(expired, anidb_id, obj, self.expiration)
return obj
Expand Down