81 lines
3.7 KiB
Python
81 lines
3.7 KiB
Python
"""Local CPU translation model registry.
|
|
|
|
The local fallback translates *to English*. Models are downloaded lazily from
|
|
Hugging Face the first time a selected source language needs local fallback.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LocalModelSpec:
|
|
code: str
|
|
label: str
|
|
model_id: str
|
|
license: str
|
|
note: str = ""
|
|
|
|
|
|
# Keep the order close to the legacy src_lang combo so existing users can find
|
|
# familiar entries. ``auto`` / ``en`` / ``[No Translation]`` are intentionally
|
|
# not part of this registry: local fallback performs no language detection and
|
|
# all registered models translate a non-English source into English. ``auto`` is handled
|
|
# by resolving it to the configured Local CPU Auto Source Language before model selection.
|
|
LOCAL_MODEL_SPECS = {
|
|
"ja": LocalModelSpec("ja", "Japanese", "Helsinki-NLP/opus-mt-ja-en", "Apache-2.0"),
|
|
"zh-CN": LocalModelSpec(
|
|
"zh-CN", "Chinese (Simplified)", "Helsinki-NLP/opus-mt-zh-en", "CC-BY-4.0",
|
|
"Shares the Chinese -> English model with zh-TW.",
|
|
),
|
|
"zh-TW": LocalModelSpec(
|
|
"zh-TW", "Chinese (Traditional)", "Helsinki-NLP/opus-mt-zh-en", "CC-BY-4.0",
|
|
"Shares the Chinese -> English model with zh-CN.",
|
|
),
|
|
"ko": LocalModelSpec("ko", "Korean", "Helsinki-NLP/opus-mt-ko-en", "Apache-2.0"),
|
|
"fr": LocalModelSpec("fr", "French", "Helsinki-NLP/opus-mt-fr-en", "Apache-2.0"),
|
|
"de": LocalModelSpec("de", "German", "Helsinki-NLP/opus-mt-de-en", "Apache-2.0"),
|
|
"es": LocalModelSpec("es", "Spanish", "Helsinki-NLP/opus-mt-es-en", "Apache-2.0"),
|
|
"it": LocalModelSpec("it", "Italian", "Helsinki-NLP/opus-mt-it-en", "Apache-2.0"),
|
|
"ru": LocalModelSpec("ru", "Russian", "Helsinki-NLP/opus-mt-ru-en", "CC-BY-4.0"),
|
|
"pt": LocalModelSpec(
|
|
"pt", "Portuguese", "Helsinki-NLP/opus-mt-ROMANCE-en", "Apache-2.0",
|
|
"Uses the shared Romance-languages -> English Marian model.",
|
|
),
|
|
"nl": LocalModelSpec("nl", "Dutch", "Helsinki-NLP/opus-mt-nl-en", "Apache-2.0"),
|
|
"pl": LocalModelSpec("pl", "Polish", "Helsinki-NLP/opus-mt-pl-en", "Apache-2.0"),
|
|
"tr": LocalModelSpec("tr", "Turkish", "Helsinki-NLP/opus-mt-tr-en", "Apache-2.0"),
|
|
"ar": LocalModelSpec("ar", "Arabic", "Helsinki-NLP/opus-mt-ar-en", "Apache-2.0"),
|
|
"hi": LocalModelSpec("hi", "Hindi", "Helsinki-NLP/opus-mt-hi-en", "Apache-2.0"),
|
|
"bn": LocalModelSpec("bn", "Bengali", "Helsinki-NLP/opus-mt-bn-en", "Apache-2.0"),
|
|
"pa": LocalModelSpec("pa", "Punjabi", "Helsinki-NLP/opus-mt-pa-en", "Apache-2.0"),
|
|
"jw": LocalModelSpec(
|
|
"jw", "Javanese", "Helsinki-NLP/opus-mt-mul-en", "Apache-2.0",
|
|
"Uses the shared multilingual -> English Marian model (Javanese is 'jav' in OPUS).",
|
|
),
|
|
"ms": LocalModelSpec(
|
|
"ms", "Malay", "Helsinki-NLP/opus-mt-mul-en", "Apache-2.0",
|
|
"Uses the shared multilingual -> English Marian model.",
|
|
),
|
|
"vi": LocalModelSpec("vi", "Vietnamese", "Helsinki-NLP/opus-mt-vi-en", "Apache-2.0"),
|
|
"th": LocalModelSpec("th", "Thai", "Helsinki-NLP/opus-mt-th-en", "Apache-2.0"),
|
|
"id": LocalModelSpec("id", "Indonesian", "Helsinki-NLP/opus-mt-id-en", "Apache-2.0"),
|
|
}
|
|
|
|
LOCAL_CPU_LANGUAGE_ORDER = tuple(LOCAL_MODEL_SPECS.keys())
|
|
DEFAULT_LOCAL_CPU_LANGUAGES = ("ja",)
|
|
DEFAULT_LOCAL_AUTO_SOURCE_LANGUAGE = "ja"
|
|
|
|
|
|
def normalize_local_languages(value):
|
|
"""Return a de-duplicated, registry-ordered list of supported language codes."""
|
|
if isinstance(value, str):
|
|
raw = [part.strip() for part in value.split(",") if part.strip()]
|
|
elif isinstance(value, (list, tuple, set)):
|
|
raw = [str(part).strip() for part in value if str(part).strip()]
|
|
else:
|
|
raw = []
|
|
selected = set(raw)
|
|
return [code for code in LOCAL_CPU_LANGUAGE_ORDER if code in selected]
|