diff --git a/scripts/utils/maintainer.py b/scripts/utils/maintainer.py index 17d98a8..4afa880 100644 --- a/scripts/utils/maintainer.py +++ b/scripts/utils/maintainer.py @@ -1,6 +1,17 @@ import os +import re +import time +import requests -from scripts.utils.helpers import MODEL_PATH, save_language +from .helpers import MODEL_PATH, save_language, get_language + +key_lang = {'af': 'Afrikaans', 'ar': 'Arabic', 'ca': 'Catalan', 'cs': 'Czech', 'da': 'Danish', 'de': 'German', 'en': 'English', 'es': 'Spanish', + 'et': 'Estonian', 'fi': 'Finnish', 'fr': 'French', 'hr': 'Croatian', 'hu': 'Hungarian', 'id': 'Indonesian', 'is': 'Icelandic', 'it': 'Italian', + 'ja': 'Japanese', 'ko': 'Korean', 'lt': 'Lithuanian', 'lv': 'Latvian', 'nl': 'Dutch', 'no': 'Norwegian', 'pl': 'Polish', 'pt': 'Portuguese', + 'ro': 'Romanian', 'ru': 'Russian', 'sk': 'Slovak', 'sl': 'Slovenian', 'sr': 'Serbian', 'sv': 'Swedish', 'th': 'Thai', 'tr': 'Turkish', + 'uk': 'Ukrainian', 'vi': 'Vietnamese', 'zh': 'Chinese'} +languages = ['af', 'ar', 'ca', 'cs', 'da', 'de', 'en', 'es', 'et', 'fi', 'fr', 'hr', 'hu', 'id', 'is', 'it', 'ja', + 'ko', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'th', 'tr', 'uk', 'vi', 'zh'] def get_labels(model, language=None): @@ -38,3 +49,114 @@ def create_all_languages(): 'ko', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'th', 'tr', 'uk', 'zh'] for language in languages: create_language(language) + + +def measure_translations(language): + en_labels = get_language('en') + labels = get_language(language) + + count = 0 + count_need = 0 + for en, lab in zip(en_labels.items(), labels.items()): + count += 1 + if en[1] == lab[1]: + count_need += 1 + + count_trans = len(labels) - count_need + return f'| {key_lang[language]} | {count_trans} | {count_trans / len(labels):.1%} |' + + +def measure_all_languages(): + stats = {} + for language in languages: + stats[key_lang[language]] = measure_translations(language) + print('| Language | Translated species | Translated species (%) |') + print('| -------- | ------- | ------ |') + for _, stat in sorted(stats.items()): + if 'English' in stat: + continue + print(stat) + + +def scrape_wikipedia(sci_name, language, failed=None): + if failed is None: + failed = [] + url = f"https://{language}.wikipedia.org/api/rest_v1/page/summary/{sci_name.replace(" ", "_")}" + headers = {'Accept-Encoding': 'gzip', 'User-agent': 'BirdNET-Pi'} + try: + resp = requests.get(url=url, headers=headers, timeout=10).json() + except Exception: + time.sleep(1) + try: + resp = requests.get(url=url, headers=headers, timeout=10).json() + except Exception as e: + failed.append((language, sci_name, str(e))) + return + + type = resp['type'] + if type == 'Internal error': + return + if type == 'disambiguation': + failed.append((language, sci_name, type)) + return + + try: + com_name = resp['title'] + except KeyError: + return + + if com_name and com_name != sci_name: + cleaned = re.sub(r'\(\S+\)', '', com_name).strip() + if cleaned != com_name: + print(f'*** checkme {com_name}') + return com_name + + +def add_translations(language, failed=None): + if failed is None: + failed = [] + en_labels = get_language('en') + labels = get_language(language) + labels_updated = get_language(language) + + count = 0 + count_trans = 0 + count_needed = 0 + + for en, lab in zip(en_labels.items(), labels.items()): + count += 1 + if en[1] == lab[1] and ' ' in en[0]: + count_needed += 1 + translated = scrape_wikipedia(en[0], language, failed) + if translated and translated != en[1]: + labels_updated[en[0]] = translated + print(f'{lab[1]} -> {translated}') + count_trans += 1 + + time.sleep(0.1) + if (count % 100) == 0: + print(f'{count_trans=}', f'{count_needed=}', f'{count / len(labels):.2%}') + + print(f'{count_trans=}', f'{count_trans / len(labels):.2%}', f'{count_needed=}', f'{count_needed / len(labels):.2%}', + f'{((count_needed - count_trans) / len(labels)):.2%}') + print(failed) + save_language(labels_updated, language) + return failed + +# Remove these +# ar: (طائر) +# de: (Vogel) +# fi: (lintu) +# is: (fugl) +# nl: [(vogel), (soort)] +# sv: (fågel) + + +def update_all_languages(): + failed = [] + for language in languages: + print(f'start {language}') + ret = add_translations(language) + print(f'finished {language}') + failed.extend(ret) + print(failed)