JAVA操作ES
阿新 • • 發佈：2020-12-23
1. pom.xml
<dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> <version>7.8.1</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> <version>7.8.1</version> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>4.5.12</version> </dependency> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.2.58</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> <version>3.9</version> </dependency> <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> <version>2.6</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-collections4</artifactId> <version>4.3</version> </dependency> <dependency> <groupId>commons-beanutils</groupId> <artifactId>commons-beanutils</artifactId> <version>1.9.3</version> </dependency>
2.EsConfig
package com.example.demo.es; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.impl.nio.reactor.IOReactorConfig; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.springframework.context.annotation.Bean; /** * @ProjectName: pet_data * @Package: com.example.demo.es * @ClassName: EsConfig * @Author: Devin.W.Zhang * @Description: * @Date: 12/21/2020 4:48 PM * @Version: 1.0 */ public class EsConfig { /** * es restful client builder * * @return restful client */ @Bean public RestClientBuilder restClientBuilder() { // 設定IP HttpHost esHost = new HttpHost("localhost", 9200); RestClientBuilder restClientBuilder = RestClient.builder(esHost); // setPassword(restClientBuilder); setTimeout(restClientBuilder); return restClientBuilder; } /** * 設定超時時間 */ private void setTimeout(RestClientBuilder restClientBuilder) { restClientBuilder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { @Override public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder builder) { return builder.setConnectTimeout(3000) .setSocketTimeout(50000); } }); } /** * 設定ES密碼 */ private void setPassword(RestClientBuilder restClientBuilder) { // 設定密碼 CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("userName", "password")); restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { @Override public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { return httpAsyncClientBuilder .setDefaultCredentialsProvider(credentialsProvider) .setDefaultIOReactorConfig( IOReactorConfig.custom() .setIoThreadCount(4) .build() ); } }); } }
3.EsUtils
package com.example.demo.es; import com.alibaba.fastjson.JSON; import org.apache.commons.beanutils.BeanUtils; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import java.io.IOException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; import java.util.Map; /** * @ProjectName: pet_data * @Package: com.example.demo.es * @ClassName: EsUtils * @Author: Devin.W.Zhang * @Description: ES 操作工具類 * @Date: 12/22/2020 10:58 AM * @Version: 1.0 */ public class EsUtils<T> { public static final String INDEX = "product"; private final RestHighLevelClient restHighLevelClient; public EsUtils() { EsConfig esConfig = new EsConfig(); RestClientBuilder restClientBuilder = esConfig.restClientBuilder(); restHighLevelClient = new RestHighLevelClient(restClientBuilder); } /** * 索引是否存在 * * @param indexName 索引名稱 * @return true/false */ public boolean existIndex(String indexName) { GetIndexRequest request = new GetIndexRequest(); request.indices(indexName); try { return restHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); } catch (IOException e) { e.printStackTrace(); return false; } } /** * 建立索引，新版ES插入資料時自動建立 * * @param index 索引 * @return 返回索引建立結果 * @throws IOException exception */ public CreateIndexResponse createIndex(String index) throws IOException { CreateIndexRequest createIndexRequest = new CreateIndexRequest(index); return restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT); } /** * 刪除索引 * * @param index 索引 * @return 返回刪除結果 * @throws IOException exception */ public AcknowledgedResponse deleteIndex(String index) throws IOException { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(index); return restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT); } /** * 通過id獲取資料 * * @param id 主鍵 * @return 返回查詢到的結果 * @throws IOException exception */ public T get(String id, Class<T> classT) throws Exception { System.out.println(String.format("get the data from es, id:%s", id)); GetRequest request = new GetRequest(INDEX, id); GetResponse getResponse = restHighLevelClient.get(request, RequestOptions.DEFAULT); Map<String, Object> result = getResponse.getSourceAsMap(); System.out.println(String.format("the es response:%s", result)); T t = convertMap2Model(result, classT); System.out.println(String.format("change the result to %s: ,the result is :%s", classT, t)); return t; } /** * 新增/修改資料 * * @param t 資料物件 * @param primaryKey 主鍵欄位名 * @return 儲存結果 * @throws Exception exception */ public IndexResponse save(T t, String primaryKey) throws Exception { IndexRequest indexRequest = new IndexRequest(INDEX); indexRequest.source(JSON.toJSONString(t), XContentType.JSON); indexRequest.id(getId(t, primaryKey)); return restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT); } /** * 批量 新增/修改資料 * * @param list 資料list * @param primaryKey 主鍵 * @return 返回儲存結果 * @throws Exception exception */ public BulkResponse batchSave(List<T> list, String primaryKey) throws Exception { BulkRequest bulkRequest = new BulkRequest(); IndexRequest indexRequest; for (T item : list) { indexRequest = new IndexRequest(INDEX); indexRequest.source(JSON.toJSONString(item), XContentType.JSON); indexRequest.id(getId(item, primaryKey)); bulkRequest.add(indexRequest); } return restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT); } /** * 查詢符合查詢條件的所有資料 * * @param queryBuilder queryBuilder * @param orders 排序map * @param tClass class * @return 返回結果list * @throws Exception exception */ public List<T> query(QueryBuilder queryBuilder, Map<String, SortOrder> orders, Class<T> tClass) throws Exception { final int currentPage = 1; final int pageSize = 100000; Object[] obj = pageQuery(currentPage, pageSize, queryBuilder, orders, tClass); return (List<T>) obj[1]; } /** * 分頁查詢 * <p> * 注意當 (currentPage * pageSize >10000) 該分頁查詢會報錯， * 通過postman put方式呼叫 http://127.0.0.1:9200/_all/_settings?preserve_existing=true 設定 {"index.max_result_window" : "1000000"} * 設定成功會返回 {"acknowledged": true} * </p> * * @param currentPage 當前頁 * @param pageSize 每頁條數 * @param queryBuilder queryBuilder * @param orders 排序map * @param tClass class * @return 返回 總條數和資料list * @throws Exception exception */ public Object[] pageQuery(int currentPage, int pageSize, QueryBuilder queryBuilder, Map<String, SortOrder> orders, Class<T> tClass) throws Exception { Object[] result = new Object[2]; long totalCount; List<T> list = new ArrayList<>(); SearchRequest request = new SearchRequest(); request.indices(INDEX); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(queryBuilder); request.source(searchSourceBuilder); //下面的配置為true則如果總條數顯示為實際查詢到的條數，否則最大隻顯示10000 searchSourceBuilder.trackTotalHits(true); //起始 searchSourceBuilder.from((currentPage - 1) * pageSize); //預設最大10000 searchSourceBuilder.size(pageSize); //排序 if (null != orders) { for (Map.Entry<String, SortOrder> order : orders.entrySet()) { searchSourceBuilder.sort(order.getKey(), order.getValue()); } } SearchResponse res = restHighLevelClient.search(request, RequestOptions.DEFAULT); totalCount = res.getHits().getTotalHits().value; System.out.println(String.format("totalHits: %s", totalCount)); if (res.getHits().getHits() != null) { SearchHit[] hits = res.getHits().getHits(); for (SearchHit hit : hits) { Map<String, Object> resultOne = hit.getSourceAsMap(); T t = convertMap2Model(resultOne, tClass); list.add(t); } } result[0] = totalCount; result[1] = list; return result; } /*下面是一些輔助方法*/ /** * 通過反射獲取主鍵的值 * * @param t 物件 * @param primaryKey 主鍵欄位名稱 * @return 返回主鍵的值 * @throws Exception exception */ private String getId(T t, String primaryKey) throws Exception { Class<?> aClass = t.getClass(); String methodName = "get" + (primaryKey.charAt(0) + "").toUpperCase() + primaryKey.substring(1); Method method = aClass.getDeclaredMethod(methodName); return (String) method.invoke(t); } private T convertMap2Model(Map<String, Object> map, Class<T> classT) throws Exception { Object o = classT.newInstance(); BeanUtils.populate(o, map); return (T) o; } /** * 關閉連線 */ public void close() { try { if (restHighLevelClient != null) { restHighLevelClient.close(); } } catch (Exception e) { e.printStackTrace(); } } }
4.測試資料物件ProductModel
package com.example.demo.es; import java.math.BigDecimal; import java.util.Random; import java.util.UUID; /** * @ProjectName: pet_data * @Package: com.example.demo.es * @ClassName: ProductModel * @Author: Devin.W.Zhang * @Description: * @Date: 12/22/2020 10:21 AM * @Version: 1.0 */ public class ProductModel { public static final String PRIMARY_KEY = "productId"; private String categoryId; private String categoryName; private String productId; private String productName; private String productDesc; private BigDecimal productPrice; private Integer productCount; private long updateTime; public ProductModel() { this.categoryId = UUID.randomUUID().toString(); this.productId = UUID.randomUUID().toString(); Random random = new Random(); int r = random.nextInt(10000); if ((r % 9) == 0) { this.categoryName = "貓"; } else if ((r % 8) == 0) { this.categoryName = "狗"; } else if ((r % 7) == 0) { this.categoryName = "寵物主糧"; } else if (r % 6 == 0) { this.categoryName = "醫療保健"; } else if (r % 5 == 0) { this.categoryName = "家居日用"; } else if (r % 4 == 0) { this.categoryName = "寵物玩具"; } else if (r % 3 == 0) { this.categoryName = "寵物出行"; } else { this.categoryName = "寵物洗護"; } int rr = random.nextInt(1000); this.productName = this.categoryName + (System.currentTimeMillis() + rr); this.productDesc = this.productName + "...."; this.productPrice = BigDecimal.valueOf(r); this.productCount = r; this.updateTime = System.currentTimeMillis(); } public String getCategoryId() { return categoryId; } public void setCategoryId(String categoryId) { this.categoryId = categoryId; } public String getCategoryName() { return categoryName; } public void setCategoryName(String categoryName) { this.categoryName = categoryName; } public String getProductId() { return productId; } public void setProductId(String productId) { this.productId = productId; } public String getProductName() { return productName; } public void setProductName(String productName) { this.productName = productName; } public String getProductDesc() { return productDesc; } public void setProductDesc(String productDesc) { this.productDesc = productDesc; } public BigDecimal getProductPrice() { return productPrice; } public void setProductPrice(BigDecimal productPrice) { this.productPrice = productPrice; } public Integer getProductCount() { return productCount; } public void setProductCount(Integer productCount) { this.productCount = productCount; } public long getUpdateTime() { return updateTime; } public void setUpdateTime(long updateTime) { this.updateTime = updateTime; } @Override public String toString() { return "ProductModel{" + "categoryId='" + categoryId + '\'' + ", categoryName='" + categoryName + '\'' + ", productId='" + productId + '\'' + ", productName='" + productName + '\'' + ", productDesc='" + productDesc + '\'' + ", productPrice=" + productPrice + ", productCount=" + productCount + ", updateTime=" + updateTime + '}'; } }
5.測試case
package com.example.demo.es; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.sort.SortOrder; import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; /** * @ProjectName: pet_data * @Package: com.example.demo.es * @ClassName: EsTestCase * @Author: Devin.W.Zhang * @Description: * @Date: 12/22/2020 10:32 AM * @Version: 1.0 */ public class EsTestCase { public static void main(String[] args) throws Exception { //儲存單條 saveOne(); //批量儲存 batchSave(); String remark = "貓"; BigDecimal productPriceStart = null; BigDecimal productPriceEnd = null; Integer productCount = 1; //查詢所有 query(remark, productPriceStart, productPriceEnd, productCount); //分頁查詢 int currentPage = 1; int pageSize = 20; Map<String, SortOrder> orders = new HashMap<>(); orders.put("updateTime", SortOrder.DESC); pageQuery(currentPage, pageSize, orders, remark, productPriceStart, productPriceEnd, productCount); } /** * 儲存單條 * * @throws Exception */ public static void saveOne() throws Exception { EsUtils esUtils = new EsUtils<ProductModel>(); ProductModel productModel = new ProductModel(); IndexResponse save = esUtils.save(productModel, ProductModel.PRIMARY_KEY); String productId = productModel.getProductId(); getOne(productId); esUtils.close(); } /** * 查詢單條 * * @param id * @throws Exception */ public static void getOne(String id) throws Exception { EsUtils esUtils = new EsUtils<ProductModel>(); ProductModel productModel = (ProductModel) esUtils.get(id, ProductModel.class); System.out.println(productModel); esUtils.close(); } /** * 批量儲存 * * @throws Exception */ private static void batchSave() throws Exception { EsUtils esUtils = new EsUtils<ProductModel>(); List<ProductModel> list = new ArrayList<>(); for (int i = 0; i < 1000000; i++) { ProductModel productModel = new ProductModel(); list.add(productModel); if (list.size() == 10000) { BulkResponse bulkItemResponses = esUtils.batchSave(list, ProductModel.PRIMARY_KEY); System.out.println(bulkItemResponses); list = new ArrayList<>(); } } if (list.size() > 0) { BulkResponse bulkItemResponses = esUtils.batchSave(list, ProductModel.PRIMARY_KEY); System.out.println(bulkItemResponses); } esUtils.close(); } /** * 查詢含有關鍵字 價格在區間內的 ，有貨的商品 * * @param remark 關鍵字 ("categoryName", "productName", "productDesc" 這三個欄位含有remark欄位 ) * @param productPriceStart productPrice >= price start * @param productPriceEnd productPrice < price end * @param productCount >= 庫存 */ public static void query(String remark, BigDecimal productPriceStart, BigDecimal productPriceEnd, Integer productCount) throws Exception { QueryBuilder queryBuilder = builderParams(remark, productPriceStart, productPriceEnd, productCount); EsUtils esUtils = new EsUtils<ProductModel>(); List<ProductModel> list = esUtils.query(queryBuilder, null, ProductModel.class); System.out.println(String.format("get the result count:%s", list.size())); for (ProductModel productModel : list) { System.out.println(productModel); } esUtils.close(); } /** * 分頁查詢 * * @param currentPage 當前頁 * @param pageSize 每頁條數 * @param orders 排序map * @param remark 查詢關鍵字 * @param productPriceStart 價格過濾開始 * @param productPriceEnd 價格過濾結束 * @param productCount 庫存 >=productCount * @throws Exception exception */ public static void pageQuery(int currentPage, int pageSize, Map<String, SortOrder> orders, String remark, BigDecimal productPriceStart, BigDecimal productPriceEnd, Integer productCount) throws Exception { QueryBuilder queryBuilder = builderParams(remark, productPriceStart, productPriceEnd, productCount); EsUtils esUtils = new EsUtils<ProductModel>(); Object[] objects = esUtils.pageQuery(currentPage, pageSize, queryBuilder, orders, ProductModel.class); List<ProductModel> list = (List<ProductModel>) objects[1]; long totalCount = (long) objects[0]; System.out.println(String.format("get the total count:%s", totalCount)); int pageCount = (int) Math.ceil((totalCount / (pageSize * 1.00))); System.out.println(String.format("page count :%s", pageCount)); for (ProductModel productModel : list) { System.out.println(productModel); } esUtils.close(); } /** * 構造查詢條件 * * @param remark 關鍵字 * @param productPriceStart 價格開始 * @param productPriceEnd 價格結束 * @param productCount 庫存數量 * @return 返回構造Query * @throws Exception query */ private static QueryBuilder builderParams(String remark, BigDecimal productPriceStart, BigDecimal productPriceEnd, Integer productCount) throws Exception { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); if (StringUtils.isNotEmpty(remark)) { boolQueryBuilder.must(QueryBuilders.multiMatchQuery(remark, "categoryName", "productName", "productDesc")); } if (productPriceStart != null) { boolQueryBuilder.must(QueryBuilders.rangeQuery("productPrice").gte(productPriceStart)); } if (productPriceEnd != null) { boolQueryBuilder.must(QueryBuilders.rangeQuery("productPrice").lt(productPriceEnd)); } if (productCount != null) { boolQueryBuilder.must(QueryBuilders.rangeQuery("productCount").gte(productCount)); } return boolQueryBuilder; } }