springboot 使用clickhouse實時大資料分析引擎的方法
阿新 • • 發佈:2018-12-12
宣告:
因專案中使用clickhouse引擎這裡springboot使用的方式是jdbc方式連線,這種方式的好處是可以使用clickhouse 自帶的fetch方法批量從clickhouse中獲取資料,對於大量資料的下載來說,比較好
因為如果全部拿到記憶體中處理,大量資料會有記憶體溢位的結果
如果批量多次請求資料庫對於資料庫查詢等也不靠譜,所有直接使用clickhouse jdbc連線來滿足這種情況,不使用mybatis等框架來管理,這裡根據大家不同的需求酌情參考即可
使用方式:
第一步:加入clickhouse jar包依賴
<!--clickhouse--> <dependency> <groupId>ru.yandex.clickhouse</groupId> <artifactId>clickhouse-jdbc</artifactId> <version>0.1.40</version> </dependency>
第二步:配置資料庫連線屬性配置檔案,yml方式 此處僅作為引數,不連線任何驅動
clickhouse:
address: jdbc:clickhouse://172.20.xxx.xxx:8123
username: default
password: xxx
db: marketing
socketTimeout: 600000
第三步:新增資料庫連線操作util工具類
import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import net.sf.json.JSONObject; import ru.yandex.clickhouse.ClickHouseConnection; import ru.yandex.clickhouse.ClickHouseDataSource; import ru.yandex.clickhouse.settings.ClickHouseProperties; import java.sql.*; import java.util.*; /** * @Description: * @Date 2018/11/12 */ @Slf4j @Component public class ClickHouseUtil { private static String clickhouseAddress; private static String clickhouseUsername; private static String clickhousePassword; private static String clickhouseDB; private static Integer clickhouseSocketTimeout; @Value("${clickhouse.address}") public void setClickhouseAddress(String address) { ClickHouseUtil.clickhouseAddress = address; } @Value("${clickhouse.username}") public void setClickhouseUsername(String username) { ClickHouseUtil.clickhouseUsername = username; } @Value("${clickhouse.password}") public void setClickhousePassword(String password) { ClickHouseUtil.clickhousePassword = password; } @Value("${clickhouse.db}") public void setClickhouseDB(String db) { ClickHouseUtil.clickhouseDB = db; } @Value("${clickhouse.socketTimeout}") public void setClickhouseSocketTimeout(Integer socketTimeout) { ClickHouseUtil.clickhouseSocketTimeout = socketTimeout; } public static Connection getConn() { ClickHouseConnection conn = null; ClickHouseProperties properties = new ClickHouseProperties(); properties.setUser(clickhouseUsername); properties.setPassword(clickhousePassword); properties.setDatabase(clickhouseDB); properties.setSocketTimeout(clickhouseSocketTimeout); ClickHouseDataSource clickHouseDataSource = new ClickHouseDataSource(clickhouseAddress,properties); try { conn = clickHouseDataSource.getConnection(); return conn; } catch (SQLException e) { e.printStackTrace(); } return null; } public static List<JSONObject> exeSql(String sql){ log.info("cliockhouse 執行sql:" + sql); Connection connection = getConn(); try { Statement statement = connection.createStatement(); ResultSet results = statement.executeQuery(sql); ResultSetMetaData rsmd = results.getMetaData(); List<JSONObject> list = new ArrayList(); while(results.next()){ JSONObject row = new JSONObject(); for(int i = 1;i<=rsmd.getColumnCount();i++){ row.put(rsmd.getColumnName(i),results.getString(rsmd.getColumnName(i))); } list.add(row); } return list; } catch (SQLException e) { e.printStackTrace(); } return null; } }
第四步:Test簡單使用執行sql查詢資料
import com.renrenche.databus.common.ClickHouseUtil; import com.renrenche.databus.common.Result; import com.renrenche.databus.domain.logdata.fem.FemParam; import com.renrenche.databus.service.fem.FemMainService; import net.sf.json.JSONObject; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; import java.util.List; /** * @Auther: qixin * @Date: 2018/12/11 15:05 * @Description: */ @RunWith(SpringRunner.class) @SpringBootTest public class SemTest { @Test public void getFrsDataTest(){ System.out.println("******************"); String sql="select * from marketing.sem_campaign_real_time_report"; List<JSONObject> result= ClickHouseUtil.exeSql(sql); System.out.println("******************"); } }
執行完畢列印結果檢視即可,
fetch方法之後再補充
感謝觀看