guess_language / guess_language / __init__.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
"""Guess the natural language of a text
"""
#   © 2012 spirit <hiddenspirit@gmail.com>
#   https://bitbucket.org/spirit/guess_language
#
#   Original Python package:
#   Copyright (c) 2008, Kent S Johnson
#   http://code.google.com/p/guess-language/
#
#   Original C++ version for KDE:
#   Copyright (c) 2006 Jacob R Rideout <kde@jacobrideout.net>
#   http://websvn.kde.org/branches/work/sonnet-refactoring/common/nlp/guesslanguage.cpp?view=markup
#
#   Original Language::Guess Perl module:
#   Copyright (c) 2004-2006 Maciej Ceglowski
#   http://web.archive.org/web/20090228163219/http://languid.cantbedone.org/
#
#   Note: Language::Guess is GPL-licensed. KDE developers received permission
#   from the author to distribute their port under LGPL:
#   http://lists.kde.org/?l=kde-sonnet&m=116910092228811&w=2
#
#   This program is free software: you can redistribute it and/or modify it
#   under the terms of the GNU Lesser General Public License as published
#   by the Free Software Foundation, either version 3 of the License,
#   or (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty
#   of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#   See the GNU Lesser General Public License for more details.
#
#   You should have received a copy of the GNU Lesser General Public License
#   along with this program. If not, see <http://www.gnu.org/licenses/>.

import functools
import re
import unicodedata
import warnings

from collections import defaultdict, OrderedDict

from .data import BLOCKS, BLOCK_RSHIFT


__all__ = [
    "guess_language", "use_enchant",
]

MAX_LENGTH = 4096
MIN_LENGTH = 20
MAX_GRAMS = 300
WORD_RE = re.compile(r"(?:[^\W\d_]|['’])+", re.U)
MODEL_ROOT = __name__ + ".data.models."

BASIC_LATIN = {
    "en", "ceb", "ha", "so", "tlh", "id", "haw", "la", "sw", "eu",
    "nr", "nso", "zu", "xh", "ss", "st", "tn", "ts"
}
EXTENDED_LATIN = {
    "cs", "af", "pl", "hr", "ro", "sk", "sl", "tr", "hu", "az",
    "et", "sq", "ca", "es", "fr", "de", "nl", "it", "da", "is", "nb", "sv",
    "fi", "lv", "pt", "ve", "lt", "tl", "cy", "vi"
}
ALL_LATIN = BASIC_LATIN.union(EXTENDED_LATIN)
CYRILLIC = {"ru", "uk", "kk", "uz", "mn", "sr", "mk", "bg", "ky"}
ARABIC = {"ar", "fa", "ps", "ur"}
DEVANAGARI = {"hi", "ne"}
PT = {"pt_BR", "pt_PT"}

# NOTE mn appears twice, once for mongolian script and once for CYRILLIC
SINGLETONS = [
    ("Armenian", "hy"),
    ("Hebrew", "he"),
    ("Bengali", "bn"),
    ("Gurmukhi", "pa"),
    ("Greek", "el"),
    ("Gujarati", "gu"),
    ("Oriya", "or"),
    ("Tamil", "ta"),
    ("Telugu", "te"),
    ("Kannada", "kn"),
    ("Malayalam", "ml"),
    ("Sinhala", "si"),
    ("Thai", "th"),
    ("Lao", "lo"),
    ("Tibetan", "bo"),
    ("Burmese", "my"),
    ("Georgian", "ka"),
    ("Mongolian", "mn-Mong"),
    ("Khmer", "km"),
]

NAME_MAP = {
    "ab": "Abkhazian",
    "af": "Afrikaans",
    "ar": "Arabic",
    "az": "Azeri",
    "be": "Byelorussian",
    "bg": "Bulgarian",
    "bn": "Bengali",
    "bo": "Tibetan",
    "br": "Breton",
    "ca": "Catalan",
    "ceb": "Cebuano",
    "cs": "Czech",
    "cy": "Welsh",
    "da": "Danish",
    "de": "German",
    "el": "Greek",
    "en": "English",
    "eo": "Esperanto",
    "es": "Spanish",
    "et": "Estonian",
    "eu": "Basque",
    "fa": "Farsi",
    "fi": "Finnish",
    "fo": "Faroese",
    "fr": "French",
    "fy": "Frisian",
    "gd": "Scots Gaelic",
    "gl": "Galician",
    "gu": "Gujarati",
    "ha": "Hausa",
    "haw": "Hawaiian",
    "he": "Hebrew",
    "hi": "Hindi",
    "hr": "Croatian",
    "hu": "Hungarian",
    "hy": "Armenian",
    "id": "Indonesian",
    "is": "Icelandic",
    "it": "Italian",
    "ja": "Japanese",
    "ka": "Georgian",
    "kk": "Kazakh",
    "km": "Cambodian",
    "ko": "Korean",
    "ku": "Kurdish",
    "ky": "Kyrgyz",
    "la": "Latin",
    "lt": "Lithuanian",
    "lv": "Latvian",
    "mg": "Malagasy",
    "mk": "Macedonian",
    "ml": "Malayalam",
    "mn": "Mongolian",
    "mr": "Marathi",
    "ms": "Malay",
    "nd": "Ndebele",
    "ne": "Nepali",
    "nl": "Dutch",
    "nn": "Nynorsk",
    "no": "Norwegian",
    "nso": "Sepedi",
    "pa": "Punjabi",
    "pl": "Polish",
    "ps": "Pashto",
    "pt": "Portuguese",
    "pt_PT": "Portuguese (Portugal)",
    "pt_BR": "Portuguese (Brazil)",
    "ro": "Romanian",
    "ru": "Russian",
    "sa": "Sanskrit",
    "sh": "Serbo-Croatian",
    "sk": "Slovak",
    "sl": "Slovene",
    "so": "Somali",
    "sq": "Albanian",
    "sr": "Serbian",
    "sv": "Swedish",
    "sw": "Swahili",
    "ta": "Tamil",
    "te": "Telugu",
    "th": "Thai",
    "tl": "Tagalog",
    "tlh": "Klingon",
    "tn": "Setswana",
    "tr": "Turkish",
    "ts": "Tsonga",
    "tw": "Twi",
    "uk": "Ukrainian",
    "ur": "Urdu",
    "uz": "Uzbek",
    "ve": "Venda",
    "vi": "Vietnamese",
    "xh": "Xhosa",
    "zh": "Chinese",
    "zh_TW": "Traditional Chinese (Taiwan)",
    "zu": "Zulu",
}

IANA_MAP = {
    "ab": 12026,
    "af": 40,
    "ar": 26020,
    "az": 26030,
    "be": 11890,
    "bg": 26050,
    "bn": 26040,
    "bo": 26601,
    "br": 1361,
    "ca": 3,
    "ceb": 26060,
    "cs": 26080,
    "cy": 26560,
    "da": 26090,
    "de": 26160,
    "el": 26165,
    "en": 26110,
    "eo": 11933,
    "es": 26460,
    "et": 26120,
    "eu": 1232,
    "fa": 26130,
    "fi": 26140,
    "fo": 11817,
    "fr": 26150,
    "fy": 1353,
    "gd": 65555,
    "gl": 1252,
    "gu": 26599,
    "ha": 26170,
    "haw": 26180,
    "he": 26592,
    "hi": 26190,
    "hr": 26070,
    "hu": 26200,
    "hy": 26597,
    "id": 26220,
    "is": 26210,
    "it": 26230,
    "ja": 26235,
    "ka": 26600,
    "kk": 26240,
    "km": 1222,
    "ko": 26255,
    "ku": 11815,
    "ky": 26260,
    "la": 26280,
    "lt": 26300,
    "lv": 26290,
    "mg": 1362,
    "mk": 26310,
    "ml": 26598,
    "mn": 26320,
    "mr": 1201,
    "ms": 1147,
    "ne": 26330,
    "nl": 26100,
    "nn": 172,
    "no": 26340,
    "pa": 65550,
    "pl": 26380,
    "ps": 26350,
    "pt": 26390,
    "ro": 26400,
    "ru": 26410,
    "sa": 1500,
    "sh": 1399,
    "sk": 26430,
    "sl": 26440,
    "so": 26450,
    "sq": 26010,
    "sr": 26420,
    "sv": 26480,
    "sw": 26470,
    "ta": 26595,
    "te": 26596,
    "th": 26594,
    "tl": 26490,
    "tlh": 26250,
    "tn": 65578,
    "tr": 26500,
    "tw": 1499,
    "uk": 26520,
    "ur": 26530,
    "uz": 26540,
    "vi": 26550,
    "zh": 26065,
    "zh_TW": 22,
}

models = {}

try:
    from importlib import import_module
except ImportError:
    import sys

    def import_module(name):
        """Import a module.
        """
        __import__(name)
        return sys.modules[name]

try:
    from collections import namedtuple

    LanguageInfo = namedtuple("LanguageInfo", ["tag", "id", "name"])
except ImportError:
    class LanguageInfo(tuple):
        def __new__(cls, tag, id, name): #@ReservedAssignment
            return tuple.__new__(cls, (tag, id, name))

        def __init__(self, tag, id, name): #@ReservedAssignment
            self.tag = tag
            self.id = id
            self.name = name


class UNKNOWN(str):
    """Unknown language
    """
    def __bool__(self):
        return False


UNKNOWN = UNKNOWN("UNKNOWN")


def guess_language(text: str):
    """Return the ISO 639-1 language code.
    """
    words = WORD_RE.findall(text[:MAX_LENGTH].replace("’", "'"))
    return identify(words, find_runs(words))


def guess_language_info(text: str):
    """Return LanguageInfo(tag, id, name).
    """
    tag = guess_language(text)

    if tag is UNKNOWN:
        return LanguageInfo(UNKNOWN, UNKNOWN, UNKNOWN)

    return LanguageInfo(tag, _get_id(tag), _get_name(tag))


# An alias for guess_language
guess_language_tag = guess_language


def guess_language_id(text: str):
    """Return the language ID.
    """
    return _get_id(guess_language(text))


def guess_language_name(text: str):
    """Return the language name (in English).
    """
    return _get_name(guess_language(text))


def _get_id(tag):
    return IANA_MAP.get(tag, UNKNOWN)


def _get_name(tag):
    return NAME_MAP.get(tag, UNKNOWN)


def find_runs(words):
    """Count the number of characters in each character block.
    """
    run_types = defaultdict(int)

    total_count = 0

    for word in words:
        for char in word:
            block = BLOCKS[ord(char) >> BLOCK_RSHIFT]
            run_types[block] += 1
            total_count += 1

    #pprint(run_types)

    # return run types that used for 40% or more of the string
    # return Basic Latin if found more than 15%
    ## and extended additional latin if over 10% (for Vietnamese)
    relevant_runs = []
    for key, value in run_types.items():
        pct = value * 100 // total_count
        if pct >= 40 or pct >= 15 and key == "Basic Latin":
            relevant_runs.append(key)
        #elif pct >= 10 and key == "Latin Extended Additional":
            #relevant_runs.append(key)

    return relevant_runs


def identify(words, scripts):
    """Identify the language.
    """
    if ("Hangul Syllables" in scripts or "Hangul Jamo" in scripts or
            "Hangul Compatibility Jamo" in scripts or "Hangul" in scripts):
        return "ko"

    if "Greek and Coptic" in scripts:
        return "el"

    if "Kana" in scripts:
        return "ja"

    if ("CJK Unified Ideographs" in scripts or "Bopomofo" in scripts or
            "Bopomofo Extended" in scripts or "KangXi Radicals" in scripts):
# This is in both Ceglowski and Rideout
# I can't imagine why...
#            or "Arabic Presentation Forms-A" in scripts
        return "zh"

    if "Cyrillic" in scripts:
        return check(words, CYRILLIC)

    if ("Arabic" in scripts or "Arabic Presentation Forms-A" in scripts or
            "Arabic Presentation Forms-B" in scripts):
        return check(words, ARABIC)

    if "Devanagari" in scripts:
        return check(words, DEVANAGARI)

    # Try languages with unique scripts
    for block_name, lang_name in SINGLETONS:
        if block_name in scripts:
            return lang_name

    #if "Latin Extended Additional" in scripts:
        #return "vi"

    if "Extended Latin" in scripts:
        latin_lang = check(words, EXTENDED_LATIN)
        if latin_lang == "pt":
            return check(words, PT)
        else:
            return latin_lang

    if "Basic Latin" in scripts:
        return check(words, ALL_LATIN)

    return UNKNOWN


def check_with_all(words, langs):
    """Check what the best match is.
    """
    return check_with_enchant(words, langs) or check_with_models(words, langs)


check = check_with_all


def use_enchant(use_enchant=True):
    """Enable or disable checking with PyEnchant.
    """
    global check
    check = check_with_all if use_enchant else check_with_models


def check_with_models(words, langs):
    """Check against known models.
    """
    sample = " ".join(words)

    if len(sample) < MIN_LENGTH:
        return UNKNOWN

    scores = []
    model = create_ordered_model(sample)  # QMap<int,QString>

    for key in langs:
        lkey = key.lower()

        try:
            known_model = models[lkey]
        except KeyError:
            try:
                known_model = import_module(MODEL_ROOT + lkey).model
            except ImportError:
                known_model = None
            models[lkey] = known_model

        if known_model:
            scores.append((distance(model, known_model), key))

    if not scores:
        return UNKNOWN

    # we want the lowest score, less distance = greater chance of match
    #pprint(sorted(scores))
    return min(scores)[1]


def create_ordered_model(content):
    """Create a list of trigrams in content sorted by frequency.
    """
    trigrams = defaultdict(int)  # QHash<QString,int>
    content = content.lower()

    for i in range(len(content) - 2):
        trigrams[content[i:i+3]] += 1

    return sorted(trigrams.keys(), key=lambda k: (-trigrams[k], k))


def distance(model, known_model):
    """Calculate the distance to the known model.
    """
    dist = 0

    for i, value in enumerate(model[:MAX_GRAMS]):
        if value in known_model:
            dist += abs(i - known_model[value])
        else:
            dist += MAX_GRAMS

    return dist


try:
    import enchant
except ImportError:
    warnings.warn("PyEnchant is unavailable", ImportWarning)
    enchant = None

    def check_with_enchant(*args, **kwargs):
        return UNKNOWN
else:
    import locale

    enchant_base_languages_dict = None

    def check_with_enchant(words, languages, threshold=0.7, min_words=1,
                           dictionaries={}):
        """Check against installed spelling dictionaries.
        """
        if len(words) < min_words:
            return UNKNOWN

        best_score = 0
        best_tag = UNKNOWN

        for tag, enchant_tag in get_enchant_base_languages_dict().items():
            if tag not in languages:
                continue
            try:
                d = dictionaries[tag]
            except KeyError:
                d = dictionaries[tag] = enchant.Dict(enchant_tag)
            score = sum([1 for word in words if d.check(word)])
            if score > best_score:
                best_score = score
                best_tag = tag

        if best_score / len(words) < threshold:
            return UNKNOWN

        return best_tag

    def get_enchant_base_languages_dict():
        """Get ordered dictionary of enchant base languages.

        locale_language, then "en", then the rest.
        """
        global enchant_base_languages_dict
        if enchant_base_languages_dict is None:
            def get_language_subtag(tag):
                return tag.split("_")[0]

            languages = OrderedDict()
            locale_language = get_locale_language()
            for tag in sorted(enchant.list_languages()):
                subtag = get_language_subtag(tag)
                if tag == locale_language or subtag not in languages:
                    languages[subtag] = tag
            languages_keys = list(languages.keys())
            languages_items = list(languages.items())
            for tag in ["en", get_language_subtag(locale_language)]:
                try:
                    index = languages_keys.index(tag)
                except ValueError:
                    pass
                else:
                    languages_keys = (
                        [languages_keys[index]] +
                        languages_keys[:index] + languages_keys[index+1:]
                    )
                    languages_items = (
                        [languages_items[index]] +
                        languages_items[:index] + languages_items[index+1:]
                    )
            enchant_base_languages_dict = OrderedDict(languages_items)

        return enchant_base_languages_dict

    def get_locale_language():
        """Get the language code for the current locale setting.
        """
        return locale.getlocale()[0] or locale.getdefaultlocale()[0]


def deprecated(func):
    """This is a decorator which can be used to mark functions
    as deprecated. It will result in a warning being emitted
    when the function is used.
    """
    @functools.wraps(func)
    def new_func(*args, **kwargs):
        warnings.warn(
            "call to deprecated function %s()" % func.__name__,
            category=DeprecationWarning,
            stacklevel=2
        )
        return func(*args, **kwargs)
    return new_func


@deprecated
def guessLanguage(text):
    """Deprecated function - use guess_language() instead.
    """
    return guess_language(decode_text(text))


@deprecated
def guessLanguageTag(text):
    """Deprecated function - use guess_language_tag() instead.
    """
    return guess_language_tag(decode_text(text))


@deprecated
def guessLanguageId(text):
    """Deprecated function - use guess_language_id() instead.
    """
    return guess_language_id(decode_text(text))


@deprecated
def guessLanguageName(text):
    """Deprecated function - use guess_language_name() instead.
    """
    warnings.warn(guessLanguageName.__doc__.strip(), DeprecationWarning, 2)
    return guess_language_name(decode_text(text))


@deprecated
def guessLanguageInfo(text):
    """Deprecated function - use guess_language_info() instead.
    """
    warnings.warn(guessLanguageInfo.__doc__.strip(), DeprecationWarning, 2)
    return guess_language_info(decode_text(text))


def decode_text(text, encoding="utf-8"):
    """Decode text if needed (for deprecated functions).
    """
    if not isinstance(text, str):
        warnings.warn("passing an encoded string is deprecated",
                      DeprecationWarning, 4)
        text = text.decode(encoding)
    return text
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.