Elasticsearch通過restapi進行資料查詢----java
阿新 • • 發佈：2018-12-31
做日誌監控使用的是ELK一套的東西，然後將日誌搜尋和下載單獨做了一個功能模組，所以就看著官網的API文件，寫了查詢的程式碼。
1.引入elasticsearch相關jar包，一定使用mvn來引入，如果自己去找很難找全的，因為他的包有43個。別問我為啥，因為自己去找包，然後搭建工程。花了挺多時間，一個mvn ，馬上就下載好了。
2.去官網翻翻rest相關的介面，以及使用。
下面是一個搜尋功能的程式碼，挺亂的，
先判斷es中是否有index（索引），然後再進行搜尋條件的拼接。
最後是用的GraphQl進行查詢的。
/** * 獲取所有的index * @return * @throws IOException * @throws SwordBaseCheckedException */ @Service(serviceName = "jkpt_LogSearchService_queryIndex") public List<Map<String,String>> queryIndex(List<Map<String,String>> esinfolist) throws IOException, SwordBaseCheckedException{ //List<String> result = new ArrayList<String>(); /*HttpHost[] tmp1 = new HttpHost[1]; HttpHost tmp2 = new HttpHost("10.23.11.139", 9200,"http"); tmp1[0] = tmp2; RestClient restClient = getLowClient(tmp1);*/ HttpHost[] hostarray = new HttpHost[esinfolist.size()]; int i = 0; for(Map<String,String> esinfo : esinfolist) { HttpHost hosttmp = new HttpHost(esinfo.get("ipaddress"), Integer.parseInt(esinfo.get("esport")),"http"); hostarray[i] = hosttmp; i++; } RestClient restClient = LogsearchUtil.getLowClient(hostarray); Response response = restClient.performRequest("GET", "/_cat/indices?v&h=index", Collections.singletonMap("pretty", "true")); String[] allindexstr = EntityUtils.toString(response.getEntity()).split("\n"); List<Map<String,String>> allindexlist = new ArrayList<Map<String,String>>(); for(String indexstr : allindexstr) { if("index".equals(indexstr) || indexstr.startsWith(".") || indexstr.startsWith("filebeat")) { continue; } Map<String,String> tmp = new HashMap<String,String>(); tmp.put("code", indexstr); tmp.put("caption", indexstr); allindexlist.add(tmp); } //System.out.println("查詢到的內容" + EntityUtils.toString(response.getEntity())); return allindexlist; } /** * 根據條件查詢日誌 * @param querMap * @return * @throws IOException * @throws ParseException * @throws SwordBaseCheckedException */ @Service(serviceName = "jkpt_LogSearchService_queryBySelect") public Map<String,Object> queryBySelect(List<Map<String,String>> esinfolist,Map<String,Object> queryMap) throws ParseException, SwordBaseCheckedException, IOException{ logger.debug("=======================根據條件查詢日誌"); String ifshowAll = null; if(queryMap.get("ifshowAll") != null) { ifshowAll = (String) queryMap.get("ifshowAll"); queryMap.remove("ifshowAll"); } String yhuuid =(String) queryMap.get("yhuuid"); queryMap.remove("yhuuid"); Map<String,Object> result = new HashMap<String,Object>(); SearchRequest searchRequest = new SearchRequest(); //每頁顯示的條數 int showsize = (Integer) queryMap.get("showsize"); queryMap.remove("showsize"); try { List<Map<String, String>> indexlist = queryIndex(esinfolist); if(indexlist == null || indexlist.size() == 0) { logger.debug("======================當前es中無資料"); result.put("loglist", null); result.put("totalHits", 0L); result.put("pagesize", 0L); return result; } //限制搜尋的index String[] indexnamearr = new String[indexlist.size()]; for(int i = 0 ;i<indexlist.size();i++) { Map<String,String> tmp = indexlist.get(i); String indexname = tmp.get("code"); indexnamearr[i] = indexname; } searchRequest = new SearchRequest(indexnamearr); }catch(IOException ioex) { result.put("error", "Connection refused"); return result; } /*MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("logLevel", "Error") .fuzziness(Fuzziness.AUTO) .prefixLength(3) .maxExpansions(10);*/ //上面程式碼的另一種方式 /*MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("logLevel", "ERROR"); matchQueryBuilder.fuzziness(Fuzziness.AUTO); matchQueryBuilder.prefixLength(3); matchQueryBuilder.maxExpansions(10); String[] indexs = {"logstash-2018.09.04","logstash-2018.08.27"}; SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(matchQueryBuilder);*/ //獲取es叢集下的所有主機 HttpHost[] hostarray = new HttpHost[esinfolist.size()]; int i = 0; for(Map<String,String> esinfo : esinfolist) { HttpHost hosttmp = new HttpHost(esinfo.get("ipaddress"), Integer.parseInt(esinfo.get("esport")),"http"); hostarray[i] = hosttmp; i++; } RestHighLevelClient client = LogsearchUtil.getHighClient(hostarray); //SearchRequest searchRequest = new SearchRequest(); //多index搜尋 ----------暫時不適用index進行篩選 /*if(queryMap.get("indexsname") != null) { String indexname = (String)queryMap.get("indexsname"); searchRequest = new SearchRequest(indexname);//index名稱 queryMap.remove("indexsname"); }*/ /*MatchPhraseQueryBuilder mpq1 = QueryBuilders.matchPhraseQuery("logLevel",queryMap.get("logLevel")); MatchPhraseQueryBuilder mpq2 = QueryBuilders.matchPhraseQuery("sessionId",queryMap.get("sessionId")); */ int nowpage = (Integer)queryMap.get("nowpage"); queryMap.remove("nowpage"); String sortway = (String) queryMap.get("sortway"); queryMap.remove("sortway"); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); //日誌級別 if(queryMap.get("logLevel") != null) { String logLevel = (String) queryMap.get("logLevel"); //yymc="{yyname="+yymc+"}"; //boolQueryBuilder.must(QueryBuilders.matchQuery("fields.yyname",yymc)); boolQueryBuilder.must(QueryBuilders.matchQuery("logLevel",logLevel )); queryMap.remove("logLevel"); } //應用名稱 if(queryMap.get("yymc") != null) { String yymc = (String) queryMap.get("yymc"); //yymc="{yyname="+yymc+"}"; boolQueryBuilder.must(QueryBuilders.matchQuery("fields.yyname",yymc)); //boolQueryBuilder.must(QueryBuilders.wildcardQuery("fields.yyname","*"+yymc+"*")); queryMap.remove("yymc"); } //關鍵字模糊查詢處理 if(queryMap.get("keywordsearch") != null) { String keywordsearch = (String) queryMap.get("keywordsearch"); boolQueryBuilder.must(QueryBuilders.matchQuery("message",keywordsearch).minimumShouldMatch("80%")); //boolQueryBuilder.must(QueryBuilders.wildcardQuery("message","*"+keywordsearch+"*")); queryMap.remove("keywordsearch"); } /** * 使用QueryBuilder * termQuery("key", obj) 完全匹配 * termsQuery("key", obj1, obj2..) 一次匹配多個值 * matchQuery("key", Obj) 單個匹配, field不支援萬用字元, 字首具高階特性 * multiMatchQuery("text", "field1", "field2"..); 匹配多個欄位, field有萬用字元忒行 * matchAllQuery(); 匹配所有檔案 */ //查詢資源整合物件 SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); // 查詢在時間區間範圍內的結果 --------------處理時間區間 RangeQueryBuilder rangbuilder = QueryBuilders.rangeQuery("date.keyword"); boolean ifhasstar = false; if(queryMap.get("starttime") != null) { ifhasstar = true; } if(queryMap.get("endtime") != null) { String endtime = (String) queryMap.get("endtime"); //rangbuilder.lte(endtime+",000"); queryMap.remove("endtime"); rangbuilder.to(endtime+",000"); if(!ifhasstar) { SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); String starttime = df.format(new Date(0)); //rangbuilder.gte(starttime+",000"); rangbuilder.from(starttime); }else { String starttime = (String) queryMap.get("starttime")+",000"; rangbuilder.from(starttime); queryMap.remove("starttime"); } //sourceBuilder.query(rangbuilder); boolQueryBuilder.must(rangbuilder); }else { if(ifhasstar) { SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); String endtime = df.format(new Date()); //rangbuilder.lte(endtime+",000"); rangbuilder.to(endtime); String starttime = (String) queryMap.get("starttime")+",000"; rangbuilder.from(starttime); //sourceBuilder.query(rangbuilder); boolQueryBuilder.must(rangbuilder); queryMap.remove("starttime"); }else { SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); String endtime = df.format(new Date()); rangbuilder.to(endtime); String starttime = df.format(new Date(0)); rangbuilder.from(starttime); } } // boolQueryBuilder.must(rangbuilder); Set<String> keyset = queryMap.keySet(); Iterator<String> keyit = keyset.iterator(); while(keyit.hasNext()) { String nowkey = keyit.next(); String str = (String)queryMap.get(nowkey);//得到每個key多對用value的值 if(str != null && !"".equals(str)) { //boolQueryBuilder.must(QueryBuilders.matchQuery(nowkey,str).minimumShouldMatch("80%")); boolQueryBuilder.must(QueryBuilders.wildcardQuery(nowkey,"*"+str+"*" )); } } QueryBuilder queryBuilder = boolQueryBuilder; //分頁檢視功能 sourceBuilder.query(queryBuilder); //後續處理 應該是動態的 sourceBuilder.from((nowpage-1) * 10); if(nowpage == 1000) { showsize = 10; } sourceBuilder.size(showsize); sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS)); //根據日期進行排序 FieldSortBuilder fsb = SortBuilders.fieldSort("date.keyword"); if("ASC".equals(sortway)) { fsb.order(SortOrder.ASC); }else { fsb.order(SortOrder.DESC); } sourceBuilder.sort(fsb); //sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); //將搜尋條件載入到searchRequest中 searchRequest.source(sourceBuilder); //searchRequest.types("log"); //執行查詢操作 //優化 避免將錯誤拋到頁面 進行try-catch SearchResponse searchResponse =null; try { searchResponse = client.search(searchRequest); } catch (Exception e) { // TODO: handle exception logger.debug("===================查詢出錯了。"); return result; } //SearchResponse searchResponse = client.search(searchRequest); SearchHits hits = searchResponse.getHits(); long totalHits = hits.getTotalHits(); logger.debug("=============查詢到的資料有"+totalHits); SearchHit[] searchHits = hits.getHits(); List<Map<String,Object>> loglist = new ArrayList<Map<String,Object>>(); Map<String,String> yyToFbsjq = queryFbsJqIpByYhuuid(yhuuid); if(ifshowAll != null && ifshowAll.equals("Y")) { for (SearchHit hit : searchHits) { /*String type = hit.getType(); String name = hit.getIndex();*/ Map<String, Object> tmp = hit.getSourceAsMap(); /* String message = (String) tmp.get("message"); System.out.println("message內容是" + message); System.out.println("型別是："+ type); System.out.println("Index 名稱是："+ name);*/ @SuppressWarnings("unchecked") Map<String,String> yynamemap = (HashMap)tmp.get("fields"); String content = (String) tmp.get("message"); if(content.contains("\n")) { content = content.replace("\n","<br>"); tmp.put("message", content); } String yyname = yynamemap.get("yyname"); if(yyToFbsjq != null ) { String fbsurl = "http://" + yyToFbsjq.get(yyname)+":8980/#/trace?traceId="; tmp.put("fbsurl",fbsurl); } tmp.put("yyname",yyname); loglist.add(tmp); } }else { for (SearchHit hit : searchHits) { Map<String, Object> tmp = hit.getSourceAsMap(); @SuppressWarnings("unchecked") Map<String,String> yynamemap = (HashMap)tmp.get("fields"); String yyname = yynamemap.get("yyname"); if(yyToFbsjq != null ) { String fbsurl = "http://" + yyToFbsjq.get(yyname)+":8980/#/trace?traceId="; tmp.put("fbsurl",fbsurl); } tmp.put("yyname",yyname); loglist.add(tmp); } }