add translation tools to maintainer.py

scripts/utils/maintainer.py

@@ -1,6 +1,17 @@
 import os
+import re
+import time
+import requests
 
-from scripts.utils.helpers import MODEL_PATH, save_language
+from .helpers import MODEL_PATH, save_language, get_language
+
+key_lang = {'af': 'Afrikaans', 'ar': 'Arabic', 'ca': 'Catalan', 'cs': 'Czech', 'da': 'Danish', 'de': 'German', 'en': 'English', 'es': 'Spanish',
+            'et': 'Estonian', 'fi': 'Finnish', 'fr': 'French', 'hr': 'Croatian', 'hu': 'Hungarian', 'id': 'Indonesian', 'is': 'Icelandic', 'it': 'Italian',
+            'ja': 'Japanese', 'ko': 'Korean', 'lt': 'Lithuanian', 'lv': 'Latvian', 'nl': 'Dutch', 'no': 'Norwegian', 'pl': 'Polish', 'pt': 'Portuguese',
+            'ro': 'Romanian', 'ru': 'Russian', 'sk': 'Slovak', 'sl': 'Slovenian', 'sr': 'Serbian', 'sv': 'Swedish', 'th': 'Thai', 'tr': 'Turkish',
+            'uk': 'Ukrainian', 'vi': 'Vietnamese', 'zh': 'Chinese'}
+languages = ['af', 'ar', 'ca', 'cs', 'da', 'de', 'en', 'es', 'et', 'fi', 'fr', 'hr', 'hu', 'id', 'is', 'it', 'ja',
+             'ko', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'th', 'tr', 'uk', 'vi', 'zh']
 
 
 def get_labels(model, language=None):
@@ -38,3 +49,114 @@ def create_all_languages():
                  'ko', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'th', 'tr', 'uk', 'zh']
     for language in languages:
         create_language(language)
+
+
+def measure_translations(language):
+    en_labels = get_language('en')
+    labels = get_language(language)
+
+    count = 0
+    count_need = 0
+    for en, lab in zip(en_labels.items(), labels.items()):
+        count += 1
+        if en[1] == lab[1]:
+            count_need += 1
+
+    count_trans = len(labels) - count_need
+    return f'| {key_lang[language]} | {count_trans} | {count_trans / len(labels):.1%} |'
+
+
+def measure_all_languages():
+    stats = {}
+    for language in languages:
+        stats[key_lang[language]] = measure_translations(language)
+    print('| Language | Translated species | Translated species (%) |')
+    print('| -------- | ------- | ------ |')
+    for _, stat in sorted(stats.items()):
+        if 'English' in stat:
+            continue
+        print(stat)
+
+
+def scrape_wikipedia(sci_name, language, failed=None):
+    if failed is None:
+        failed = []
+    url = f"https://{language}.wikipedia.org/api/rest_v1/page/summary/{sci_name.replace(" ", "_")}"
+    headers = {'Accept-Encoding': 'gzip', 'User-agent': 'BirdNET-Pi'}
+    try:
+        resp = requests.get(url=url, headers=headers, timeout=10).json()
+    except Exception:
+        time.sleep(1)
+        try:
+            resp = requests.get(url=url, headers=headers, timeout=10).json()
+        except Exception as e:
+            failed.append((language, sci_name, str(e)))
+            return
+
+    type = resp['type']
+    if type == 'Internal error':
+        return
+    if type == 'disambiguation':
+        failed.append((language, sci_name, type))
+        return
+
+    try:
+        com_name = resp['title']
+    except KeyError:
+        return
+
+    if com_name and com_name != sci_name:
+        cleaned = re.sub(r'\(\S+\)', '', com_name).strip()
+        if cleaned != com_name:
+            print(f'*** checkme {com_name}')
+        return com_name
+
+
+def add_translations(language, failed=None):
+    if failed is None:
+        failed = []
+    en_labels = get_language('en')
+    labels = get_language(language)
+    labels_updated = get_language(language)
+
+    count = 0
+    count_trans = 0
+    count_needed = 0
+
+    for en, lab in zip(en_labels.items(), labels.items()):
+        count += 1
+        if en[1] == lab[1] and ' ' in en[0]:
+            count_needed += 1
+            translated = scrape_wikipedia(en[0], language, failed)
+            if translated and translated != en[1]:
+                labels_updated[en[0]] = translated
+                print(f'{lab[1]} -> {translated}')
+                count_trans += 1
+
+            time.sleep(0.1)
+        if (count % 100) == 0:
+            print(f'{count_trans=}', f'{count_needed=}', f'{count / len(labels):.2%}')
+
+    print(f'{count_trans=}', f'{count_trans / len(labels):.2%}', f'{count_needed=}', f'{count_needed / len(labels):.2%}',
+          f'{((count_needed - count_trans) / len(labels)):.2%}')
+    print(failed)
+    save_language(labels_updated, language)
+    return failed
+
+# Remove these
+# ar: (طائر)
+# de: (Vogel)
+# fi: (lintu)
+# is: (fugl)
+# nl: [(vogel), (soort)]
+# sv: (fågel)
+
+
+def update_all_languages():
+    failed = []
+    for language in languages:
+        print(f'start {language}')
+        ret = add_translations(language)
+        print(f'finished {language}')
+        failed.extend(ret)
+    print(failed)