15、Analyzer分析器之中文分析器的擴充套件
阿新 • • 發佈：2018-11-01
其實在第五章節裡已經有介紹過下面的分析器了，只是沒有做例子，今天將下面沒有做過例子分析器進行一個例子說明
paoding： 庖丁解牛最新版在
https://code.google.com/p/paoding/
中最多支援Lucene 3.0，且最新提交的程式碼在 2008-06-03，在svn中最新也是2010年提交，已經過時，不予考慮。
mmseg4j：最新版已從
https://code.google.com/p/mmseg4j/
移至
https://github.com/chenlb/mmseg4j-solr
，支援Lucene 4.10，且在github中最新提交程式碼是2014年6月，從09年～14年一共有：18個版本，也就是一年幾乎有3個大小版本，有較大的活躍度，用了mmseg演算法。
MMseg 的分析器的使用 首先將引用相關的jar包，
主要參考的文件 http://codepub.cn/2016/03/23/Maven-project-integrating-Lucene-Chinese-Segmentation-tools-Jcseg-and-Ansj/
MMseg 的分析器的使用 首先將引用相關的jar包，
<!--mmseg4j 的分析器的使用 -->
<dependency>
<groupId>com.chenlb.mmseg4j</groupId>
<artifactId>mmseg4j-core</artifactId>
<version>1.10.0</version>
</dependency>package mmseg;
import com.chenlb.mmseg4j.*;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
/**
* Created by kangz on 2016/12/19.
*/
public class MMsegAnalyzerTest {
public static void main(String[] args) throws IOException {
String txt = "";
txt = "那個好看的笑容裡面全是悲傷白富美，他在行屍走肉的活著，他的故事悲傷的像一場沒有結局的黑白電影，他是她小說裡的主角， 她懂他，他愛過她，她不知道自己是愛他的的外表，還是愛他的故事，還是愛他身上的那個自己。";
File file = new File("D:\\LucentTest\\luceneIndex2");//詞典的目錄
Dictionary dic = Dictionary.getInstance();//建立詞典例項，與比較老的版本中不相同。不能直接new。 預設讀取的是jar包中 words.dic（可修改其內容）也可指定詞典目錄 可以是 File 也可以是String 的形式
Seg seg = null;
//seg = new SimpleSeg(dic);//簡單的
seg = new ComplexSeg(dic);//複雜的
MMSeg mmSeg = new MMSeg(new StringReader(txt), seg);
Word word = null;
while((word = mmSeg.next())!=null) {
if(word != null) {
System.out.print(word + "|");
}
}
}
}<!--Jcseg 的分析器的使用 -->
<dependency>
<groupId>org.lionsoul</groupId>
<artifactId>jcseg-core</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>org.lionsoul</groupId>
<artifactId>jcseg-analyzer</artifactId>
<version>2.0.1</version>
</dependency>package lexicon;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.junit.Test;
import org.lionsoul.jcseg.analyzer.v5x.JcsegAnalyzer5X;
import org.lionsoul.jcseg.tokenizer.core.JcsegTaskConfig;
import java.io.File;
import java.nio.file.Paths;
/**
* Created by kangz on 2016/12/19.
*/
public class LexiconAnalyzersTest {
@Test
public void test() throws Exception {
//如果不知道選擇哪個Directory的子類，那麼推薦使用FSDirectory.open()方法來開啟目錄 建立一個分析器物件
Analyzer analyzer = new JcsegAnalyzer5X(JcsegTaskConfig.COMPLEX_MODE);
//非必須(用於修改預設配置): 獲取分詞任務配置例項
JcsegAnalyzer5X jcseg = (JcsegAnalyzer5X) analyzer;
JcsegTaskConfig config = jcseg.getTaskConfig();
//追加同義詞, 需要在 jcseg.properties中配置jcseg.loadsyn=1
config.setAppendCJKSyn(true);
//追加拼音, 需要在jcseg.properties中配置jcseg.loadpinyin=1
config.setAppendCJKPinyin(true);
//更多配置, 請檢視 org.lionsoul.jcseg.tokenizer.core.JcsegTaskConfig
/** ------------------------------------------------------------------------ **/
// 開啟索引庫
// 指定索引庫存放的位置
Directory directory = FSDirectory.open(Paths.get("D:\\LucentTest\\luceneIndex"));
//建立一個IndexwriterConfig物件
//第一個引數：lucene的版本，第二個引數：分析器物件
IndexWriterConfig indexWriterConfig=new IndexWriterConfig(analyzer);
//建立一個Indexwriter物件
IndexWriter indexWriter = new IndexWriter(directory, indexWriterConfig);
indexWriter.deleteAll();//清除之前的索引 注: 全部刪除索引， 請慎用。
//讀取檔案資訊
//原始文件存放的目錄
File path = new File("D:\\LucentTest\\luceneFile");
for (File file:path.listFiles()) {
if (file.isDirectory()) continue;
//讀取檔案資訊
//檔名
String fileName = file.getName();
//檔案內容
String fileContent = FileUtils.readFileToString(file);
//檔案的路徑
String filePath = file.getPath();
//檔案的大小
long fileSize = FileUtils.sizeOf(file);
//建立文件物件
Document document = new Document();
//建立域
//三個引數：1、域的名稱2、域的值3、是否儲存 Store.YES：儲存 Store.NO：不儲存
Field nameField = new TextField("name", fileName, Field.Store.YES);
Field contentField = new TextField("content", fileContent, Field.Store.YES);
Field sizeField=new LongPoint("size",fileSize);
Field pathField = new StoredField("path", filePath);
//把域新增到document物件中
document.add(nameField);
document.add(contentField);
document.add(pathField);
document.add(sizeField);
//把document寫入索引庫
indexWriter.addDocument(document);
}
indexWriter.close();
}
//使用查詢
@Test
public void testTermQuery() throws Exception {
//以讀的方式開啟索引庫
Directory directory = FSDirectory.open(Paths.get("D:\\LucentTest\\luceneIndex"));
//建立一個IndexReader
IndexReader indexReader = DirectoryReader.open(directory);
//建立一個IndexSearcher物件
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
//建立一個查詢物件
Query query = new TermQuery(new Term("content", "全文檢索"));
//執行查詢
TopDocs topDocs = indexSearcher.search(query,10);
System.out.println("查詢結果總數量：" + topDocs.totalHits);
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
//取document物件
Document document = indexSearcher.doc(scoreDoc.doc);
System.out.println("得分：" + scoreDoc.score);
//System.out.println(document.get("content"));
System.out.println(document.get("path"));
}
indexReader.close();
}
}<!--ansj 的分析器的使用-->
<dependency>
<groupId>org.ansj</groupId>
<artifactId>ansj_seg</artifactId>
<version>5.0.2</version>
</dependency>
<dependency>
<groupId>org.ansj</groupId>
<artifactId>ansj_lucene5_plug</artifactId>
<version>5.0.3.0</version>
</dependency>#redress dic file path
ambiguityLibrary=src/main/resources/library/ambiguity.dic
#path of userLibrary this is default library
userLibrary=src/main/resources/library/default.dic
#path of crfModel
crfModel=src/main/resources/library/crf.model
#set real name
isRealName=truepackage ansj;
import org.ansj.library.UserDefineLibrary;
import org.ansj.lucene5.AnsjAnalyzer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.junit.Test;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.nio.file.Paths;
import java.util.Date;
/**
* Created by kangz on 2016/12/19.
*/
public class AnsjAnalyzerTest {
/**
* 簡單測試 AnsjAnalyzer的效能及基礎應用
* @throws IOException
*/
@Test
public void test() throws IOException {
Analyzer ca = new AnsjAnalyzer(AnsjAnalyzer.TYPE.index);
Reader sentence = new StringReader(
"全文檢索是將整本書java、整篇文章中的任意內容資訊查找出來的檢索，java。它可以根據需要獲得全文中有關章、節、段、句、詞等資訊，計算機程式通過掃描文章中的每一個詞");
TokenStream ts = ca.tokenStream("sentence", sentence);
System.out.println("start: " + (new Date()));
long before = System.currentTimeMillis();
while (ts.incrementToken()) {
System.out.println(ts.getAttribute(CharTermAttribute.class));
}
ts.close();
long now = System.currentTimeMillis();
System.out.println("time: " + (now - before) / 1000.0 + " s");
}
@Test
public void indexTest() throws IOException, ParseException {
Analyzer analyzer = new AnsjAnalyzer(AnsjAnalyzer.TYPE.index);
Directory directory = FSDirectory.open(Paths.get("D:\\LucentTest\\luceneIndex2"));
IndexWriter iwriter;
UserDefineLibrary.insertWord("蛇藥片", "n", 1000);
// 建立一個IndexWriterConfig 物件
IndexWriterConfig config = new IndexWriterConfig(analyzer);// 建立indexwriter物件
IndexWriter indexWriter = new IndexWriter(directory, config);
// 建立一個文件物件
Document document = new Document();
Field nameField = new TextField("text", "季德勝蛇藥片 10片*6板 ", Field.Store.YES);
nameField.boost();
document.add(nameField);
//寫入索引庫
indexWriter.addDocument(document);
indexWriter.commit();
indexWriter.close();
System.out.println("索引建立完畢");
search(analyzer, directory, "\"季德勝蛇藥片\"");
}
//封裝索引查詢
private void search(Analyzer queryAnalyzer, Directory directory, String queryStr) throws IOException, ParseException {
IndexSearcher isearcher;
DirectoryReader directoryReader = DirectoryReader.open(directory);
// 查詢索引
isearcher = new IndexSearcher(directoryReader);
QueryParser tq = new QueryParser("text", queryAnalyzer);
Query query = tq.parse(queryStr);
System.out.println(query);
TopDocs hits = isearcher.search(query, 5);
System.out.println(queryStr + ":共找到" + hits.totalHits + "條記錄!");
for (int i = 0; i < hits.scoreDocs.length; i++) {
int docId = hits.scoreDocs[i].doc;
Document document = isearcher.doc(docId);
System.out.println(toHighlighter(queryAnalyzer, query, document));
}
}
//
private String toHighlighter(Analyzer analyzer, Query query, Document doc) {
String field = "text";
try {
SimpleHTMLFormatter simpleHtmlFormatter = new SimpleHTMLFormatter("<font color=\"red\">", "</font>");
Highlighter highlighter = new Highlighter(simpleHtmlFormatter, new QueryScorer(query));
TokenStream tokenStream1 = analyzer.tokenStream("text", new StringReader(doc.get(field)));
String highlighterStr = highlighter.getBestFragment(tokenStream1, doc.get(field));
return highlighterStr == null ? doc.get(field) : highlighterStr;
} catch (IOException | InvalidTokenOffsetsException e) {
}
return null;
}
}
主要參考的文件 http://codepub.cn/2016/03/23/Maven-project-integrating-Lucene-Chinese-Segmentation-tools-Jcseg-and-Ansj/
下面是小編的微信轉帳二維碼，小編再次謝謝讀者的支援，小編會更努力的
----請看下方↓↓↓↓↓↓↓
百度搜索 Drools從入門到精通：可下載開源全套Drools教程
深度Drools教程不段更新中：
更多Drools實戰陸續釋出中………
掃描下方二維碼關注公眾號 ↓↓↓↓↓↓↓↓↓↓
