spring boot 下操作ES
阿新 • • 發佈：2019-05-03
當前頁 應用 rto rec tag reat ans lean context
spring boot 下操作ES
1、引入es相關依賴
<dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> <version>5.5.3</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>transport</artifactId> <version>5.5.3</version> </dependency>
2、ES config 配置
import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import java.net.InetAddress; /** * @Auther: yujuan * @Date: 19-4-23 16:24 * @Description: */ @Configuration public class ElasticsearchConfig { private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchConfig.class); /** * elk集群地址 */ @Value("${elasticsearch.ip}") private String hostName; /** * 端口 */ @Value("${elasticsearch.port}") private String port; /** * 集群名稱 */ @Value("${elasticsearch.cluster.name}") private String clusterName; /** * 連接池 */ @Value("${elasticsearch.pool}") private String poolSize; @Bean public TransportClient init() { TransportClient transportClient = null; try { // 配置信息 Settings esSetting = Settings.builder() .put("cluster.name", clusterName) .put("client.transport.sniff", true)//增加嗅探機制，找到ES集群 .put("thread_pool.search.size", Integer.parseInt(poolSize))//增加線程池個數，暫時設為5 .build(); transportClient = new PreBuiltTransportClient(esSetting); InetSocketTransportAddress inetSocketTransportAddress = new InetSocketTransportAddress(InetAddress.getByName(hostName), Integer.valueOf(port)); transportClient.addTransportAddresses(inetSocketTransportAddress); } catch (Exception e) { LOGGER.error("elasticsearch TransportClient create error!!!", e); } return transportClient; } }
3、操作ES
import cn.zhiu.file.server.common.bean.EsPage; import com.alibaba.fastjson.JSONObject; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; /** * @Auther: yujuan * @Date: 19-4-23 16:33 * @Description: */ @Component public class ElasticsearchServer { private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchServer.class); @Autowired private TransportClient transportClient; private static TransportClient client; @PostConstruct public void init() { client = this.transportClient; } /** * 創建索引 * * @param index * @return */ public static boolean createIndex(String index) { if (!isIndexExist(index)) { LOGGER.info("Index is not exits!"); } CreateIndexResponse indexresponse = client.admin().indices().prepareCreate(index).execute().actionGet(); LOGGER.info("執行建立成功？" + indexresponse.isAcknowledged()); return indexresponse.isAcknowledged(); } /** * 刪除索引 * * @param index * @return */ public static boolean deleteIndex(String index) { if (!isIndexExist(index)) { LOGGER.info("Index is not exits!"); } DeleteIndexResponse dResponse = client.admin().indices().prepareDelete(index).execute().actionGet(); if (dResponse.isAcknowledged()) { LOGGER.info("delete index " + index + " successfully!"); } else { LOGGER.info("Fail to delete index " + index); } return dResponse.isAcknowledged(); } /** * 判斷索引是否存在 * * @param index * @return */ public static boolean isIndexExist(String index) { IndicesExistsResponse inExistsResponse = client.admin().indices().exists(new IndicesExistsRequest(index)).actionGet(); if (inExistsResponse.isExists()) { LOGGER.info("Index [" + index + "] is exist!"); } else { LOGGER.info("Index [" + index + "] is not exist!"); } return inExistsResponse.isExists(); } /** * 數據添加，正定ID * * @param jsonObject 要增加的數據 * @param index 索引，類似數據庫 * @param type 類型，類似表 * @param id 數據ID * @return */ public static String addData(JSONObject jsonObject, String index, String type, String id) { IndexResponse response = client.prepareIndex(index, type, id).setSource(jsonObject).get(); LOGGER.info("addData response status:{},id:{}", response.status().getStatus(), response.getId()); return response.getId(); } /** * 數據添加 * * @param jsonObject 要增加的數據 * @param index 索引，類似數據庫 * @param type 類型，類似表 * @return */ public static String addData(JSONObject jsonObject, String index, String type) { return addData(jsonObject, index, type, UUID.randomUUID().toString().replaceAll("-", "").toUpperCase()); } /** * 通過ID刪除數據 * * @param index 索引，類似數據庫 * @param type 類型，類似表 * @param id 數據ID */ public static void deleteDataById(String index, String type, String id) { DeleteResponse response = client.prepareDelete(index, type, id).execute().actionGet(); LOGGER.info("deleteDataById response status:{},id:{}", response.status().getStatus(), response.getId()); } /** * 通過ID 更新數據 * * @param jsonObject 要增加的數據 * @param index 索引，類似數據庫 * @param type 類型，類似表 * @param id 數據ID * @return */ public static void updateDataById(JSONObject jsonObject, String index, String type, String id) { UpdateRequest updateRequest = new UpdateRequest(); updateRequest.index(index).type(type).id(id).doc(jsonObject); client.update(updateRequest); } /** * 通過ID獲取數據 * * @param index 索引，類似數據庫 * @param type 類型，類似表 * @param id 數據ID * @param fields 需要顯示的字段，逗號分隔（缺省為全部字段） * @return */ public static Map<String, Object> searchDataById(String index, String type, String id, String fields) { GetRequestBuilder getRequestBuilder = client.prepareGet(index, type, id); if (StringUtils.isNotEmpty(fields)) { getRequestBuilder.setFetchSource(fields.split(","), null); } GetResponse getResponse = getRequestBuilder.execute().actionGet(); return getResponse.getSource(); } /** * 使用分詞查詢 * * @param index 索引名稱 * @param type 類型名稱,可傳入多個type逗號分隔 * @param fields 需要顯示的字段，逗號分隔（缺省為全部字段） * @param matchStr 過濾條件（xxx=111,aaa=222） * @return */ public static List<Map<String, Object>> searchListData(String index, String type, String fields, String matchStr) { return searchListData(index, type, 0, 0, null, fields, null, false, null, matchStr); } /** * 使用分詞查詢 * * @param index 索引名稱 * @param type 類型名稱,可傳入多個type逗號分隔 * @param fields 需要顯示的字段，逗號分隔（缺省為全部字段） * @param sortField 排序字段 * @param matchPhrase true 使用，短語精準匹配 * @param matchStr 過濾條件（xxx=111,aaa=222） * @return */ public static List<Map<String, Object>> searchListData(String index, String type, String fields, String sortField, boolean matchPhrase, String matchStr) { return searchListData(index, type, 0, 0, null, fields, sortField, matchPhrase, null, matchStr); } /** * 使用分詞查詢 * * @param index 索引名稱 * @param type 類型名稱,可傳入多個type逗號分隔 * @param size 文檔大小限制 * @param fields 需要顯示的字段，逗號分隔（缺省為全部字段） * @param sortField 排序字段 * @param matchPhrase true 使用，短語精準匹配 * @param highlightField 高亮字段 * @param matchStr 過濾條件（xxx=111,aaa=222） * @return */ public static List<Map<String, Object>> searchListData(String index, String type, Integer size, String fields, String sortField, boolean matchPhrase, String highlightField, String matchStr) { return searchListData(index, type, 0, 0, size, fields, sortField, matchPhrase, highlightField, matchStr); } /** * 使用分詞查詢 * * @param index 索引名稱 * @param type 類型名稱,可傳入多個type逗號分隔 * @param startTime 開始時間 * @param endTime 結束時間 * @param size 文檔大小限制 * @param fields 需要顯示的字段，逗號分隔（缺省為全部字段） * @param sortField 排序字段 * @param matchPhrase true 使用，短語精準匹配 * @param highlightField 高亮字段 * @param matchStr 過濾條件（xxx=111,aaa=222） * @return */ public static List<Map<String, Object>> searchListData(String index, String type, long startTime, long endTime, Integer size, String fields, String sortField, boolean matchPhrase, String highlightField, String matchStr) { SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index); if (StringUtils.isNotEmpty(type)) { searchRequestBuilder.setTypes(type.split(",")); } BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); if (startTime > 0 && endTime > 0) { boolQuery.must(QueryBuilders.rangeQuery("processTime") .format("epoch_millis") .from(startTime) .to(endTime) .includeLower(true) .includeUpper(true)); } //搜索的的字段 if (StringUtils.isNotEmpty(matchStr)) { for (String s : matchStr.split(",")) { String[] ss = s.split("="); if (ss.length > 1) { if (matchPhrase == Boolean.TRUE) { boolQuery.must(QueryBuilders.matchPhraseQuery(s.split("=")[0], s.split("=")[1])); } else { boolQuery.must(QueryBuilders.matchQuery(s.split("=")[0], s.split("=")[1])); } } } } // 高亮（xxx=111,aaa=222） if (StringUtils.isNotEmpty(highlightField)) { HighlightBuilder highlightBuilder = new HighlightBuilder(); //highlightBuilder.preTags("<span style='color:red' >");//設置前綴 //highlightBuilder.postTags("</span>");//設置後綴 // 設置高亮字段 highlightBuilder.field(highlightField); searchRequestBuilder.highlighter(highlightBuilder); } searchRequestBuilder.setQuery(boolQuery); if (StringUtils.isNotEmpty(fields)) { searchRequestBuilder.setFetchSource(fields.split(","), null); } searchRequestBuilder.setFetchSource(true); if (StringUtils.isNotEmpty(sortField)) { searchRequestBuilder.addSort(sortField, SortOrder.DESC); } if (size != null && size > 0) { searchRequestBuilder.setSize(size); } //打印的內容 可以在 Elasticsearch head 和 Kibana 上執行查詢 LOGGER.info("\n{}", searchRequestBuilder); SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); long totalHits = searchResponse.getHits().totalHits; long length = searchResponse.getHits().getHits().length; LOGGER.info("共查詢到[{}]條數據,處理數據條數[{}]", totalHits, length); if (searchResponse.status().getStatus() == 200) { // 解析對象 return setSearchResponse(searchResponse, highlightField); } return null; } /** * 使用分詞查詢,並分頁 * * @param index 索引名稱 * @param type 類型名稱,可傳入多個type逗號分隔 * @param currentPage 當前頁 * @param pageSize 每頁顯示條數 * @param startTime 開始時間 * @param endTime 結束時間 * @param fields 需要顯示的字段，逗號分隔（缺省為全部字段） * @param sortField 排序字段 * @param matchPhrase true 使用，短語精準匹配 * @param highlightField 高亮字段 * @param matchStr 過濾條件（xxx=111,aaa=222） * @return */ public static EsPage searchDataPage(String index, String type, int currentPage, int pageSize, long startTime, long endTime, String fields, String sortField, boolean matchPhrase, String highlightField, String matchStr) { SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index); if (StringUtils.isNotEmpty(type)) { searchRequestBuilder.setTypes(type.split(",")); } searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH); // 需要顯示的字段，逗號分隔（缺省為全部字段） if (StringUtils.isNotEmpty(fields)) { searchRequestBuilder.setFetchSource(fields.split(","), null); } //排序字段 if (StringUtils.isNotEmpty(sortField)) { searchRequestBuilder.addSort(sortField, SortOrder.DESC); } BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); if (startTime > 0 && endTime > 0) { boolQuery.must(QueryBuilders.rangeQuery("processTime") .format("epoch_millis") .from(startTime) .to(endTime) .includeLower(true) .includeUpper(true)); } // 查詢字段 if (StringUtils.isNotEmpty(matchStr)) { for (String s : matchStr.split(",")) { String[] ss = s.split("="); if (matchPhrase == Boolean.TRUE) { boolQuery.must(QueryBuilders.matchPhraseQuery(s.split("=")[0], s.split("=")[1])); } else { boolQuery.must(QueryBuilders.matchQuery(s.split("=")[0], s.split("=")[1])); } } } // 高亮（xxx=111,aaa=222） if (StringUtils.isNotEmpty(highlightField)) { HighlightBuilder highlightBuilder = new HighlightBuilder(); //highlightBuilder.preTags("<span style='color:red' >");//設置前綴 //highlightBuilder.postTags("</span>");//設置後綴 // 設置高亮字段 highlightBuilder.field(highlightField); searchRequestBuilder.highlighter(highlightBuilder); } searchRequestBuilder.setQuery(QueryBuilders.matchAllQuery()); searchRequestBuilder.setQuery(boolQuery); // 分頁應用 searchRequestBuilder.setFrom(currentPage).setSize(pageSize); // 設置是否按查詢匹配度排序 searchRequestBuilder.setExplain(true); //打印的內容 可以在 Elasticsearch head 和 Kibana 上執行查詢 LOGGER.info("\n{}", searchRequestBuilder); // 執行搜索,返回搜索響應信息 SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); long totalHits = searchResponse.getHits().totalHits; long length = searchResponse.getHits().getHits().length; LOGGER.debug("共查詢到[{}]條數據,處理數據條數[{}]", totalHits, length); if (searchResponse.status().getStatus() == 200) { // 解析對象 List<Map<String, Object>> sourceList = setSearchResponse(searchResponse, highlightField); return new EsPage(currentPage, pageSize, (int) totalHits, sourceList); } return null; } /** * 高亮結果集 特殊處理 * * @param searchResponse * @param highlightField */ private static List<Map<String, Object>> setSearchResponse(SearchResponse searchResponse, String highlightField) { List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>(); StringBuffer stringBuffer = new StringBuffer(); for (SearchHit searchHit : searchResponse.getHits().getHits()) { searchHit.getSource().put("id", searchHit.getId()); if (StringUtils.isNotEmpty(highlightField)) { System.out.println("遍歷 高亮結果集，覆蓋 正常結果集" + searchHit.getSource()); Text[] text = searchHit.getHighlightFields().get(highlightField).getFragments(); if (text != null) { for (Text str : text) { stringBuffer.append(str.string()); } //遍歷 高亮結果集，覆蓋 正常結果集 searchHit.getSource().put(highlightField, stringBuffer.toString()); } } sourceList.add(searchHit.getSource()); } return sourceList; } }
spring boot 下操作ES