實時電商數倉（二十四）之實時計算（三）日活處理模組（三）日活資料查詢介面
阿新 • • 發佈：2020-09-14
日活資料查詢介面
1訪問路徑
|
總數 |
http://publisher:8070/realtime-total?date=2019-02-01 |
|
分時統計 |
http://publisher:8070/realtime-hour?id=dau&date=2019-02-01 |
2要求資料格式
|
總數 |
[{"id":"dau","name":"新增日活","value":1200}, {"id":"new_mid","name":"新增裝置","value":233}] |
|
分時統計 |
{"yesterday":{"11":383,"12":123,"17":88,"19":200 }, "today":{"12":38,"13":1233,"17":123,"19":688 }} |
3搭建釋出工程
工程目錄
4 配置檔案
4.1 pom.xml
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-parent</artifactId> <version>2.1.15.RELEASE</version> <relativePath/> <!-- lookup parent from repository --> </parent> <groupId>com.atguigu.gmall0105</groupId> <artifactId>gmall0105-publisher</artifactId> <version>0.0.1-SNAPSHOT</version> <name>gmall0105-publisher</name> <description>Demo project forView CodeSpring Boot</description> <properties> <java.version>1.8</java.version> </properties> <dependencies> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-elasticsearch</artifactId> </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> <version>3.8.1</version> </dependency> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.2.68</version> </dependency> <dependency> <groupId>io.searchbox</groupId> <artifactId>jest</artifactId> <version>5.3.3</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </exclusion> </exclusions> </dependency> <dependency> <groupId>net.java.dev.jna</groupId> <artifactId>jna</artifactId> <version>4.5.2</version> </dependency> <dependency> <groupId>org.codehaus.janino</groupId> <artifactId>commons-compiler</artifactId> <version>2.7.8</version> </dependency> <!-- https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch --> <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> <version>2.4.6</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> <optional>true</optional> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-jdbc</artifactId> </dependency> <dependency> <groupId>org.mybatis.spring.boot</groupId> <artifactId>mybatis-spring-boot-starter</artifactId> <version>1.3.4</version> </dependency> <dependency> <groupId>ru.yandex.clickhouse</groupId> <artifactId>clickhouse-jdbc</artifactId> <version>0.1.55</version> </dependency> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.47</version> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-maven-plugin</artifactId> </plugin> </plugins> </build> </project>
4.2 application.properties
server.port=8070 spring.elasticsearch.jest.uris=http://hdp1:9200,http://hdp2:9200 spring.datasource.driver-class-name=com.mysql.jdbc.Driver spring.datasource.url=jdbc:mysql://hdp1/gmall0105_rs?characterEncoding=utf-8&useSSL=false spring.datasource.username=root spring.datasource.password=123123 #spring.datasource.driver-class-name=ru.yandex.clickhouse.ClickHouseDriver #spring.datasource.url=jdbc:clickhouse://hdp1:8123/test0105 mybatis.mapperLocations=classpath:mapper/*.xml mybatis.configuration.map-underscore-to-camel-case=true
5程式碼部分
5.1 程式碼結構
|
控制層 |
PublisherController |
實現介面的web釋出 |
|
服務層 |
PublisherService |
資料業務查詢interface |
|
PublisherServiceImpl |
業務查詢的實現類 |
|
|
主程式 |
GmallPublisherApplication |
增加掃描包 |
5.2 controller層
package com.atguigu.gmall0105.publisher.controller; import com.alibaba.fastjson.JSON; import com.atguigu.gmall0105.publisher.service.ClickHouseService; import com.atguigu.gmall0105.publisher.service.EsService; import org.apache.commons.lang3.time.DateUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.math.BigDecimal; import java.math.RoundingMode; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; @RestController public class PublisherController { @Autowired EsService esService; @Autowired ClickHouseService clickHouseService; @RequestMapping(value = "realtime-total",method = RequestMethod.GET) public String realtimeTotal(@RequestParam("date") String dt){ List<Map<String,Object>> rsList=new ArrayList<>(); Map<String,Object> dauMap = new HashMap(); dauMap.put("id","dau"); dauMap.put("name","新增日活"); Long dauTotal=0L; try { dauTotal = esService.getDauTotal(dt); }catch ( Exception e){ e.printStackTrace(); } if(dauTotal!=null){ dauMap.put("value",dauTotal); }else { dauMap.put("value",0L); } rsList.add(dauMap); Map<String,Object> newMidMap = new HashMap(); newMidMap.put("id","new_mid"); newMidMap.put("name","新增裝置"); newMidMap.put("value",233); rsList.add(newMidMap); Map<String,Object> orderAmountMap = new HashMap(); orderAmountMap.put("id","order_amount"); orderAmountMap.put("name","新增交易額"); BigDecimal orderAmount = clickHouseService.getOrderAmount(dt); orderAmountMap.put("value",orderAmount); rsList.add(orderAmountMap); return JSON.toJSONString(rsList); } @GetMapping("realtime-hour") public String realtimeHour(@RequestParam("id") String id ,@RequestParam("date") String dt){ if(id.equals("dau")){ Map dauHourMapTD = esService.getDauHour(dt); String yd = getYd(dt); Map dauHourMapYD = esService.getDauHour(yd); Map<String,Map<String,Long>> rsMap=new HashMap<>(); rsMap.put("yesterday",dauHourMapYD); rsMap.put("today",dauHourMapTD); return JSON.toJSONString(rsMap); }else if(id.equals("order_amount")){ Map orderAmountHourMapTD = clickHouseService.getOrderAmountHour(dt); String yd = getYd(dt); Map orderAmountHourMapYD = clickHouseService.getOrderAmountHour(yd); Map<String,Map<String,BigDecimal>> rsMap=new HashMap<>(); rsMap.put("yesterday",orderAmountHourMapYD); rsMap.put("today",orderAmountHourMapTD); return JSON.toJSONString(rsMap); }else{ return null; } } private String getYd(String today){ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); try { Date todayDate = dateFormat.parse(today); Date ydDate = DateUtils.addDays(todayDate, -1); return dateFormat.format(ydDate); } catch (ParseException e) { e.printStackTrace(); throw new RuntimeException("日期格式不正確"); } } }
5.3 service層
package com.atguigu.gmall0105.publisher.service; import java.math.BigDecimal; import java.util.List; import java.util.Map; public interface EsService { public Long getDauTotal(String date); public Map getDauHour(String date); }
5.4 service層實現類
package com.atguigu.gmall0105.publisher.service.impl; import com.atguigu.gmall0105.publisher.service.EsService; import io.searchbox.client.JestClient; import io.searchbox.core.Search; import io.searchbox.core.SearchResult; import io.searchbox.core.search.aggregation.TermsAggregation; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @Service public class EsServiceImpl implements EsService { @Autowired JestClient jestClient; @Override public Long getDauTotal(String date) { String indexName = "gmall0105_dau_info_"+date+"-query"; SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(new MatchAllQueryBuilder()); Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(indexName).addType("_doc").build(); try { SearchResult searchResult = jestClient.execute(search); return searchResult.getTotal(); } catch (IOException e) { e.printStackTrace(); throw new RuntimeException("es 查詢異常"); } } @Override public Map getDauHour(String date) { String indexName = "gmall0105_dau_info_"+date+"-query"; //構造查詢語句 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); TermsBuilder aggBuilder = AggregationBuilders.terms("groupby_hr").field("hr").size(24); searchSourceBuilder.aggregation(aggBuilder); Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(indexName).addType("_doc").build(); try { SearchResult searchResult = jestClient.execute(search); //封裝返回結果 Map<String,Long> aggMap=new HashMap<>(); if(searchResult.getAggregations().getTermsAggregation("groupby_hr")!=null){ List<TermsAggregation.Entry> buckets = searchResult.getAggregations().getTermsAggregation("groupby_hr").getBuckets(); for (TermsAggregation.Entry bucket : buckets) { aggMap.put( bucket.getKey(),bucket.getCount()); } } return aggMap; } catch (IOException e) { e.printStackTrace(); throw new RuntimeException("es 查詢異常"); } } }
6 搭建視覺化工程進行對接
拷貝前端工程，啟動主程式啟動。