未經允許不可轉載

Kenlm相關知識

Kenlm下載地址
kenlm中文版本訓練語言模型
如何使用kenlm訓練出來的模型C++版本

關於Kenlm模組的使用及C++原始碼說明

載入Kenlm模組命令

[email protected]:~/Documents/kenlm/lm$ ../bin/query -n test.arpa
***

Kenlm模組C++原始碼說明

query的主入口檔案:query_main.cc
query的執行函式檔案:ngram_query.hh
注意:
預設執行的是query_main.cc檔案96行的

Query<ProbingModel>(file, config, sentence_context, show_words);
 

而不是lm/wrappers/nplm.hh,這個封裝檔案是需要NPLM模組的,參考以下程式碼,當時疏忽了在這個地方耽誤了一些時間

#ifdef WITH_NPLM
    } else if (lm::np::Model::Recognize(file)) {
      lm::np::Model model(file);
      if (show_words) {
        Query<lm::np::Model, lm::ngram::FullPrint>(model, sentence_context);
      } else {
        Query<lm::np::Model, lm::ngram::BasicPrint>(model, sentence_context);
      }
#endif
 

關於Model類的繼承關係

• 最基類virtual_interface.hh lm::base::Model
• 次基類facade.hh lm::base::ModelFacade : public Model
• 子類model.hh lm::ngram::GenericModel : public base::ModelFacade<GenericModel<Search, VocabularyT>, State, VocabularyT>

關於cython的簡單說明

cython官網
可以從官網下載最新版本,參考Documentation分類中的Cython Wiki和Cython FAQ瞭解一些知識。

kenlm的python模組封裝

接下來，讓我們進入正題，在kenlm的原始碼中實際上已經提供了python的應用。在kenlm/python資料夾中，那麼為什麼還要再封裝python模組呢，因為kenlm中所帶的python模組僅僅實現了包含<s>和</s>這種情況下的計算分數的方法，而沒有提供不包含這種情況的計算分數的演算法，這就是為什麼要重新封裝python模組的原因。

簡單介紹一下python模組使用的必要步驟

• 安裝kenlm.so模組到python的目錄下，預設直接執行kenlm目錄下的setup.py檔案即可安裝成功sudo python setup.py install --record log。
• 安裝成功後，即可執行python example.py檔案，檢視執行結果。

如何擴充套件kenlm的python模組

接下來，正式進入python擴充套件模組的介紹。kenlm.pxd是cython針對所用到C++類及物件的宣告檔案，kenlm.pyx是真正要編寫的cython功能程式碼，也是未來python所要呼叫的類及方法。使用cython的編譯命令，可以把kenlm.pxd和kenlm.pyx編譯出kenlm.cpp檔案。setup.py檔案會用到編譯出來的kenlm.cpp檔案。

• cython編譯命令cython --cplus kenlm.pyx

擴充套件後的kenlm.pxd檔案

from libcpp.string cimport string

cdef extern from "lm/word_index.hh":
    ctypedef unsigned WordIndex

cdef extern from "lm/return.hh" namespace "lm":
    cdef struct FullScoreReturn:
        float prob
        unsigned char ngram_length

cdef extern from "lm/state.hh" namespace "lm::ngram":
    cdef struct State:
        pass

    ctypedef State const_State "const lm::ngram::State"

cdef extern from "lm/virtual_interface.hh" namespace "lm::base":
    cdef cppclass Vocabulary:
        WordIndex Index(char*)
        WordIndex BeginSentence() 
        WordIndex EndSentence()
        WordIndex NotFound()

    ctypedef Vocabulary const_Vocabulary "const lm::base::Vocabulary"


cdef extern from "lm/model.hh" namespace "lm::ngram":
    cdef cppclass Model:
        const_Vocabulary& GetVocabulary()
        const_State& NullContextState()
        void Model(char* file)
        FullScoreReturn FullScore(const_State& in_state, WordIndex new_word, const_State& out_state)

        void BeginSentenceWrite(void *)
        void NullContextWrite(void *)
        unsigned int Order()
        const_Vocabulary& BaseVocabulary()
        float BaseScore(void *in_state, WordIndex new_word, void *out_state)
        FullScoreReturn BaseFullScore(void *in_state, WordIndex new_word, void *out_state)
        void * NullContextMemory()

擴充套件後的kenlm.pyx檔案

import os

cdef bytes as_str(data):
    if isinstance(data, bytes):
        return data
    elif isinstance(data, unicode):
        return data.encode('utf8')
    raise TypeError('Cannot convert %s to string' % type(data))

cdef int as_in(int &Num):
    (&Num)[0] = 1

cdef class LanguageModel:
    cdef Model* model
    cdef public bytes path
    cdef const_Vocabulary* vocab

    def __init__(self, path):
        self.path = os.path.abspath(as_str(path))
        try:
            self.model = new Model(self.path)
        except RuntimeError as exception:
            exception_message = str(exception).replace('\n', ' ')
            raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
                    from exception
        self.vocab = &self.model.GetVocabulary()

    def __dealloc__(self):
        del self.model

    property order:
        def __get__(self):
            return self.model.Order()
    
    def score(self, sentence):
        cdef list words = as_str(sentence).split()
        cdef State state
        self.model.BeginSentenceWrite(&state)
        cdef State out_state
        cdef float total = 0
        for word in words:
            total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state)
            state = out_state
        total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state)
        return total

    def full_scores(self, sentence):
        cdef list words = as_str(sentence).split()
        cdef State state
        self.model.BeginSentenceWrite(&state)
        cdef State out_state
        cdef FullScoreReturn ret
        cdef float total = 0
        for word in words:
            ret = self.model.BaseFullScore(&state,
                self.vocab.Index(word), &out_state)
            yield (ret.prob, ret.ngram_length)
            state = out_state
        ret = self.model.BaseFullScore(&state,
            self.vocab.EndSentence(), &out_state)
        yield (ret.prob, ret.ngram_length)
    
    def full_scores_n(self, sentence):
        cdef list words = as_str(sentence).split()
        cdef State state
        state = self.model.NullContextState()
        cdef State out_state
        cdef FullScoreReturn ret
        cdef int ovv = 0
        for word in words:
            ret = self.model.FullScore(state,
                self.vocab.Index(word), out_state)
            yield (ret.prob, ret.ngram_length)
            state = out_state

    """""""""""
    """count scores when not included <s> and </s>"""
    """""""""""
    def score_n(self, sentence):
        cdef list words = as_str(sentence).split()
        cdef State state
        state = self.model.NullContextState()
        cdef State out_state
        cdef float total = 0
        for word in words:
            ret = self.model.FullScore(state,
                self.vocab.Index(word), out_state)
            total += ret.prob
            """print(total)"""
            state = out_state
        return total


    def __contains__(self, word):
        cdef bytes w = as_str(word)
        return (self.vocab.Index(w) != 0)

    def __repr__(self):
        return '<LanguageModel from {0}>'.format(os.path.basename(self.path))

    def __reduce__(self):
        return (LanguageModel, (self.path,))