#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author  : Peidong
# @Site    : 
# @File    : eg7.py
# @Software: PyCharm
"""
從文字提取資訊
"""
import nltk
# 讀取語料庫的“訓練”部分的100 個句子的例子
from nltk.corpus import conll2000
print(conll2000.chunked_sents('train.txt')[99])
# # 使用chunk_types 引數選擇
print(conll2000.chunked_sents('train.txt'
 
, chunk_types=['NP'])[99])

# 訪問一個已分塊語料，可以評估分塊器
cp = nltk.RegexpParser("")
test_sents = conll2000.chunked_sents('test.txt', chunk_types=['NP'])
print(cp.evaluate(test_sents))

# 嘗試一個初級的正則表示式分塊器，查詢以名詞短語標記的特徵字母（如CD、DT 和JJ）開頭的標記。
grammar = r"NP: {<[CDJNP].*>+}"
cp = nltk.RegexpParser(grammar)
test_sents = conll2000.chunked_sents('test.txt'
 
, chunk_types=['NP'])
print(cp.evaluate(test_sents))

# 使用unigram 標註器對名詞短語分塊。
class UnigramChunker(nltk.ChunkParserI):
    def __init__(self, train_sents):
        train_data = [[(t,c) for w,t,c in nltk.chunk.tree2conlltags(sent)]
                  for sent in train_sents]
        self.tagger = nltk.UnigramTagger(train_data)
    def 
 
parse(self, sentence):
        pos_tags = [pos for (word,pos) in sentence]
        tagged_pos_tags = self.tagger.tag(pos_tags)
        chunktags = [chunktag for (pos, chunktag) in tagged_pos_tags]
        conlltags = [(word, pos, chunktag) for ((word,pos),chunktag)
                     in zip(sentence, chunktags)]
        return nltk.chunk.conlltags2tree(conlltags)
# # 可以使用CoNLL2000 分塊語料庫訓練它，並測試其效能
test_sents = conll2000.chunked_sents('test.txt', chunk_types=['NP'])
train_sents = conll2000.chunked_sents('train.txt', chunk_types=['NP'])
unigram_chunker = UnigramChunker(train_sents)
print(unigram_chunker.evaluate(test_sents))

postags = sorted(set(pos for sent in train_sents
                     for (word,pos) in sent.leaves()))
print(unigram_chunker.tagger.tag(postags))

# 使用連續分類器對名詞短語分塊
class ConsecutiveNPChunkTagger(nltk.TaggerI):
    def __init__(self, train_sents):
        train_set = []
        for tagged_sent in train_sents:
            untagged_sent = nltk.tag.untag(tagged_sent)
            history = []
            for i, (word, tag) in enumerate(tagged_sent):
                featureset = npchunk_features(untagged_sent, i, history)
                train_set.append( (featureset, tag) )
                history.append(tag)
        self.classifier = nltk.MaxentClassifier.train(train_set, algorithm='megam', trace=0)
    def tag(self, sentence):
        history = []
        for i, word in enumerate(sentence):
            featureset = npchunk_features(sentence, i, history)
            tag = self.classifier.classify(featureset)
            history.append(tag)
        return zip(sentence, history)

class ConsecutiveNPChunker(nltk.ChunkParserI):
        def __init__(self, train_sents):
            tagged_sents = [[((w, t), c) for (w, t, c) in
nltk.chunk.tree2conlltags(sent)]
                            for sent in train_sents]
            self.tagger = ConsecutiveNPChunkTagger(tagged_sents)

        def parse(self, sentence):
            tagged_sents = self.tagger.tag(sentence)
            conlltags = [(w, t, c) for ((w, t), c) in tagged_sents]
            return nltk.chunk.conlltags2tree(conlltags)

# # 定義一個簡單的特徵提取器，它只是提供了當前識別符號的詞性標記
def npchunk_features(sentence, i, history):
    word, pos = sentence[i]
    return {"pos": pos}
train_sents = conll2000.chunked_sents('train.txt', chunk_types=['NP'])
chunker = ConsecutiveNPChunker(train_sents)
print(chunker.evaluate(test_sents))


# 一個分塊器，處理NP，PP，VP 和S
grammar = r"""
NP: {<DT|JJ|NN.*>+} # Chunk sequences of DT, JJ, NN
PP: {<IN><NP>} # Chunk prepositions followed by NP
VP: {<VB.*><NP|PP|CLAUSE>+$} # Chunk verbs and their arguments
CLAUSE: {<NP><VP>} # Chunk NP, VP
"""
cp = nltk.RegexpParser(grammar)
sentence = [("Mary", "NN"), ("saw", "VBD"), ("the", "DT"), ("cat", "NN"),
("sit", "VB"), ("on", "IN"), ("the", "DT"), ("mat", "NN")]
print(cp.parse(sentence))
sentence = [("John", "NNP"), ("thinks", "VBZ"), ("Mary", "NN"),
("saw", "VBD"), ("the", "DT"), ("cat", "NN"), ("sit", "VB"),
("on", "IN"), ("the", "DT"), ("mat", "NN")]
print(cp.parse(sentence))

cp = nltk.RegexpParser(grammar, loop=2)
print(cp.parse(sentence))




# 在NLTK 中，建立了一棵樹，通過給一個節點新增標籤和一個孩子連結串列：
# tree1 = nltk.Tree('NP', ['Alice'])
# print(tree1)
# tree2 = nltk.Tree('NP', ['the', 'rabbit'])
# print(tree2)
# tree3 = nltk.Tree('VP', ['chased', tree2])
# tree4 = nltk.Tree('S', [tree1, tree3])
# print(tree4)
# print(tree4[1])
# print(tree4.leaves())
# print(tree4[1].node)
# print(tree4[1][1][1])
# 遞迴函式遍歷樹
# def traverse(t):
#     try:
#         t.node
#     except AttributeError:
#         print(t,)
#     else:
#         # Now we know that t.node is defined
#         print ('(', t.node,)
#         for child in t:
#             traverse(child)
#         print (')',)
#
# t = nltk.Tree('(S (NP Alice) (VP chased (NP the rabbit)))')
# print(traverse(t))
sent = nltk.corpus.treebank.tagged_sents()[22]
print(nltk.ne_chunk(sent, binary=True))
print(nltk.ne_chunk(sent))