HBase學習筆記 （伍）- Phoenix & Sqoop
阿新 • • 發佈：2019-01-10
文章目錄
Phoenix簡介
- 構建在Apache HBase之上的一個SQL中間層
- 可以在Apache HBase上執行SQL查詢,效能強勁
- 較完善的查詢支援,支援二級索引,查詢效率較高
Phoenix優勢
- Put the SQL back in NoSQL, 程式設計師熟知SQL語句
- 具有完整ACID事務功能的標準SQL和JDBC API的強大功能
- 完全可以和其他Hadoop產品例如Spark, Hive, Pig, Flume以及MapReduce集
Phoenix vs Hive
為什麼phoenix快
- 通過HBase協處理器,在服務端進行操作,從而最大限度的減少客戶端和伺服器的資料傳輸
- 通過定製的過濾器對資料進行處理
- 使用本地的HBase Api而不是通過MapReduce框架,這樣能最大限度的降低啟動成本
Phoenix功能特性
- 二級索引：拋棄之前的只能根據rowkey做索引，通過過濾器刪選資料；自由根據索引的列或者表示式形成備用的行鍵更方便的進行資料的查詢
- 多租戶：通過多租戶表和租戶專用的連結使使用者只能訪問自己的資料
- 使用者定義函式：自己實現DF；像select一樣使用
- 行時間戳列：可以將hbase每行的時間戳對映成Phoenix的一個列
- 分頁查詢：標準的sql分頁語法
- 檢視：標準sql語法（Phoenix可以使用多個虛擬表共享底層的物理表）
Phoenix安裝
- 下載對應版本的安裝包
http://phoenix.apache.org/download.html - 配置與現有HBase叢集整合重新
解壓後如下操作
如果是分散式則應拷貝到每個reginserver下
cp phoenix-core-4.14.1-HBase-1.2.jar ../hbase-1.2.0-cdh5.14.4/lib/
cp phoenix-4.14.1-HBase-1.2-server.jar ../hbase-1.2.0-cdh5.14.4/lib/
- 啟動HBase環境,並測試環境是否可以正常使用
前置條件啟動HDFS
啟動hbase
最後啟動phoenix（bin/./sqlline.py）
Phoenix實戰：shell命令操作Phoenix
進入shell
建立表
插入資料phoenix插入是upsert與普通sql不同
插入列
插入資料
複雜sql語句
Phoenix實戰：java jdbc操作Phoenix
引入依賴
<dependencies>
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core</artifactId>
<version>4.13.1-HBase-1.2</version>
</dependency>
</dependencies>
package com.kun.bigdata.phoenix.test;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
public class PhoenixTest {
public static void main(String[] args) throws Exception {
Class.forName("org.apache.phoenix.jdbc.PhoenixDriver");
Connection connection = DriverManager.getConnection("jdbc:phoenix:localhost:2181");
PreparedStatement statement = connection.prepareStatement("select * from PERSON");
ResultSet resultSet = statement.executeQuery();
while (resultSet.next()) {
System.out.println(resultSet.getString("NAME"));
}
statement.close();
connection.close();
}
}
通過mybatis操作Phoenix
引入依賴
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
<version>1.4.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
<version>1.4.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jetty</artifactId>
<version>1.4.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<version>1.4.2.RELEASE</version>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core</artifactId>
<version>4.13.1-HBase-1.2</version>
</dependency>
</dependencies>
新增配置檔案
applilcation.properties
mybatis.config-location=mybatis-config.xml
datasource.jdbcUrl=jdbc:phoenix:localhost:2181
datasource.driverClassName=org.apache.phoenix.jdbc.PhoenixDriver
datasource.maxPoolSize=20
datasource.minIdle=2
datasource.validationTimeout=300000
datasource.idleTimeout=600000
datasource.connectionTestQuery=select 1+1
mybatis.mapperLocations=
log4j.properties
log4j.rootLogger=INFO
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
mybatis-config.xml
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration
PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-config.dtd">
<configuration>
<settings>
<setting name="cacheEnabled" value="true"/>
<setting name="lazyLoadingEnabled" value="true"/>
<setting name="multipleResultSetsEnabled" value="true"/>
<setting name="useColumnLabel" value="true"/>
<setting name="useGeneratedKeys" value="false"/>
<setting name="autoMappingBehavior" value="PARTIAL"/>
<setting name="autoMappingUnknownColumnBehavior" value="WARNING"/>
<setting name="defaultExecutorType" value="SIMPLE"/>
<setting name="defaultStatementTimeout" value="25"/>
<setting name="defaultFetchSize" value="100"/>
<setting name="safeRowBoundsEnabled" value="false"/>
<setting name="mapUnderscoreToCamelCase" value="false"/>
<setting name="localCacheScope" value="SESSION"/>
<setting name="jdbcTypeForNull" value="OTHER"/>
<setting name="lazyLoadTriggerMethods" value="equals,clone,hashCode,toString"/>
</settings>
</configuration>
UserInfo實體類
package com.kun.bigdata.phoenix.mybatis.test;
public class UserInfo {
private int id;
private String name;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
HikariDataSourceFactory 相當於配置了一個數據庫連線池
package com.kun.bigdata.phoenix.mybatis.test.mybatis;
import org.apache.ibatis.datasource.unpooled.UnpooledDataSourceFactory;
import com.zaxxer.hikari.HikariDataSource;
public class HikariDataSourceFactory extends UnpooledDataSourceFactory {
public HikariDataSourceFactory() {
this.dataSource = new HikariDataSource();
}
}
package com.kun.bigdata.phoenix.mybatis.test.mybatis;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.util.Set;
import javax.sql.DataSource;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.ResourceLoader;
@Configuration
@MapperScan(basePackages = PhoenixDataSourceConfig.PACKAGE,
sqlSessionFactoryRef = "PhoenixSqlSessionFactory")
public class PhoenixDataSourceConfig {
static final String PACKAGE = "com.kun.bigdata.phoenix.**";
@Bean(name = "PhoenixDataSource")
@Primary
public DataSource phoenixDataSource() throws IOException {
ResourceLoader loader = new DefaultResourceLoader();
InputStream inputStream = loader.getResource("classpath:application.properties")
.getInputStream();
Properties properties = new Properties();
properties.load(inputStream);
Set<Object> keys = properties.keySet();
Properties dsProperties = new Properties();
for (Object key : keys) {
if (key.toString().startsWith("datasource")) {
dsProperties.put(key.toString().replace("datasource.", ""), properties.get(key));
}
}
HikariDataSourceFactory factory = new HikariDataSourceFactory();
factory.setProperties(dsProperties);
inputStream.close();
return factory.getDataSource();
}
@Bean(name = "PhoenixSqlSessionFactory")
@Primary
public SqlSessionFactory phoenixSqlSessionFactory(
@Qualifier("PhoenixDataSource") DataSource dataSource) throws Exception {
SqlSessionFactoryBean factoryBean = new SqlSessionFactoryBean();
factoryBean.setDataSource(dataSource);
ResourceLoader loader = new DefaultResourceLoader();
String resource = "classpath:mybatis-config.xml";
factoryBean.setConfigLocation(loader.getResource(resource));
factoryBean.setSqlSessionFactoryBuilder(new SqlSessionFactoryBuilder());
return factoryBean.getObject();
}
}
dao層
package com.kun.bigdata.phoenix.mybatis.test.dao;
import java.util.List;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.ResultMap;
import org.apache.ibatis.annotations.Select;
import com.kun.bigdata.phoenix.mybatis.test.UserInfo;
@Mapper
public interface UserInfoMapper {
@Insert("upsert into USER_INFO (ID,NAME) VALUES (#{user.id},#{user.name})")
public void addUser(@Param("user") UserInfo userInfo);
@Delete("delete from USER_INFO WHERE ID=#{userId}")
public void deleteUser(@Param("userId") int userId);
@Select("select * from USER_INFO WHERE ID=#{userId}")
@ResultMap("userResultMap")
public UserInfo getUserById(@Param("userId") int userId);
@Select("select * from USER_INFO WHERE NAME=#{userName}")
@ResultMap("userResultMap")
public UserInfo getUserByName(@Param("userName") String userName);
@Select("select * from USER_INFO")
@ResultMap("userResultMap")
public List<UserInfo> getUsers();
}
測試
package com.kun.bigdata.phoenix.mybatis.test;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.PropertySource;
import org.springframework.test