Skip to content

Lexicon

GermanLexicon

Bases: Lexicon

Class for retrieving information regarding word frequency and lexicon membership. Concrete implementations might also serve as interfaces for language-specific lexical resources such as word groups and affixes.

Source code in src/limes/analyzers/de/lexicon.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class GermanLexicon(Lexicon):
    """
    Class for retrieving information regarding word frequency and lexicon
    membership. Concrete implementations might also serve as interfaces for
    language-specific lexical resources such as word groups and affixes.
    """

    def __init__(self):
        self._data: RecordTrie | None = None

    def contains(self, word: str) -> bool:
        """Check whether the given word is contained in the lexicon."""
        data = self._ensure_data_loaded()
        return word in data

    def get_frequency(self, word: str) -> int | None:
        """
        Identify how frequent a given word is in the given Lexicon. Please note
        that frequency is described in relative terms, normalized to [0.0, 1.0].
        """
        data = self._ensure_data_loaded()
        match = data.get(word)
        # As the trie returs lists of tuples, we have to unwrap None results.
        if match is not None:
            match = match[0][0]
        return match

    def _ensure_data_loaded(self) -> RecordTrie:
        """
        Load data into cache if it hasn't been loaded before, and return it.
        """
        if self._data is None:
            self._data = self._load_data_file()
        return self._data

    @staticmethod
    def _load_data_file() -> RecordTrie:
        """
        Load lexicon data from the artifact bundled with the library.
        """
        pkg = importlib.resources.files("limes.data.de")
        lexicon_file = pkg.joinpath("lexicon.marisa")
        with importlib.resources.as_file(lexicon_file) as file:
            # We format the counts into little-Endian unsigned longs. This is
            # dictated by how the actual artifact was compiled;
            # see `scripts/compile_lexicon.py`.
            format = "<L"
            trie = RecordTrie(format)
            trie.load(file)
        return trie

contains(word)

Check whether the given word is contained in the lexicon.

Source code in src/limes/analyzers/de/lexicon.py 
18
19
20
21
def contains(self, word: str) -> bool:
    """Check whether the given word is contained in the lexicon."""
    data = self._ensure_data_loaded()
    return word in data

get_frequency(word)

Identify how frequent a given word is in the given Lexicon. Please note that frequency is described in relative terms, normalized to [0.0, 1.0].

Source code in src/limes/analyzers/de/lexicon.py 
23
24
25
26
27
28
29
30
31
32
33
def get_frequency(self, word: str) -> int | None:
    """
    Identify how frequent a given word is in the given Lexicon. Please note
    that frequency is described in relative terms, normalized to [0.0, 1.0].
    """
    data = self._ensure_data_loaded()
    match = data.get(word)
    # As the trie returs lists of tuples, we have to unwrap None results.
    if match is not None:
        match = match[0][0]
    return match