Sharding-JDBC+MyBatis分庫分表
阿新 • • 發佈：2020-09-09
隨著專案功能越來越多業務越來越複雜，資料庫儲存的資料逐漸龐大，當mysql單表儲存資料過千萬的時候，對該表的操作變得緩慢，這時候就需要通過分庫分表對資料庫優化。
水平分庫：是把同一個表的資料按一定規則拆到不同的資料庫中，每個庫可以放在不同的伺服器上。
- 解決了單庫大資料，高併發的效能瓶頸
- 提高了系統的穩定性及可用性
水平分表：是在同一個資料庫內，把同一個表的資料按一定規則拆到多個表中。
- 優化單一表資料量過大而產生的效能問題
- 避免IO爭搶並減少鎖表的機率
本文采用 Sharding-JDBC 分庫分表，它是一個優秀的開源框架，使用客戶端直連資料庫，以 jar 包形式提供服務，無需額外部署和依賴，可理解為增強版的 JDBC 驅動，完全相容 JDBC 和各種 ORM 框架。
- 適用於任何基於 JDBC 的 ORM 框架，如：JPA, Hibernate, Mybatis, Spring JDBC Template 或直接使用 JDBC。
- 支援任何第三方的資料庫連線池，如：DBCP, C3P0, BoneCP, Druid, HikariCP 等。
- 支援任意實現 JDBC 規範的資料庫，目前支援 MySQL，Oracle，SQLServer，PostgreSQL 以及任何遵循 SQL92 標準的資料庫。
一、建庫建表
CREATE DATABASE ds_0; USE ds_0; DROP TABLE IF EXISTS `user_0`; CREATE TABLE `user_0` ( `id` bigint(64) NOT NULL, `city` varchar(20) NOT NULL, `name` varchar(20) NOT NULL, `sex` tinyint(4) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; DROP TABLE IF EXISTS `user_1`; CREATE TABLE `user_1` ( `id` bigint(64) NOT NULL, `city` varchar(20) NOT NULL, `name` varchar(20) NOT NULL, `sex` tinyint(4) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE DATABASE ds_1; USE ds_1; DROP TABLE IF EXISTS `user_0`; CREATE TABLE `user_0` ( `id` bigint(64) NOT NULL, `city` varchar(20) NOT NULL, `name` varchar(20) NOT NULL, `sex` tinyint(4) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; DROP TABLE IF EXISTS `user_1`; CREATE TABLE `user_1` ( `id` bigint(64) NOT NULL, `city` varchar(20) NOT NULL, `name` varchar(20) NOT NULL, `sex` tinyint(4) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
二、建立SpringBoot專案，引入依賴
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>io.shardingjdbc</groupId>
<artifactId>sharding-jdbc-core</artifactId>
<version>2.0.3</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.44</version>
</dependency>
三、建立mapper檔案、類、實體
UserMapper.xml
<?xml version="1.0" encoding="UTF-8" ?> <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> <mapper namespace="law.mapper.UserMapper"> <resultMap id="baseResultMap" type="law.entity.User"> <result column="id" property="id"/> <result column="city" property="city"/> <result column="name" property="name"/> <result column="sex" property="sex"/> </resultMap> <insert id="addUser" parameterType="law.entity.User"> INSERT INTO user (id, city, name, sex) VALUES (#{id}, #{city}, #{name}, #{sex}) </insert> <select id="list" resultMap="baseResultMap" parameterType="java.util.ArrayList"> SELECT * FROM user <where> id in ( <foreach collection="list" item="id" index="index" separator=","> #{id} </foreach> ) </where> </select> <select id="query" resultMap="baseResultMap"> SELECT * FROM user order by id asc </select> </mapper>
package law.mapper; import law.entity.User; import org.springframework.stereotype.Repository; import java.util.List; @Repository public interface UserMapper { void addUser(User user); List<User> list(List<Long> ids); List<User> query(); }
package law.entity; import lombok.Data; import java.io.Serializable; @Data public class User implements Serializable { private static final long serialVersionUID = -1205226416664488559L; private Long id; private String city; private String name; private Long sex; }
四、載入資料庫連結、建立分庫分表策略
package law.configs;
import com.alibaba.druid.pool.DruidDataSource;
import io.shardingjdbc.core.api.config.ShardingRuleConfiguration;
import io.shardingjdbc.core.api.config.TableRuleConfiguration;
import io.shardingjdbc.core.api.config.strategy.StandardShardingStrategyConfiguration;
import io.shardingjdbc.core.jdbc.core.datasource.ShardingDataSource;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.SqlSessionTemplate;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import javax.sql.DataSource;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
@Configuration
@MapperScan(basePackages = "law.mapper", sqlSessionTemplateRef = "testSqlSessionTemplate")
public class DataSourceConfig {
/**
* 配置資料來源
*/
@Bean(name = "shardingDataSource")
DataSource getShardingDataSource() throws SQLException {
ShardingRuleConfiguration shardingRuleConfig = new ShardingRuleConfiguration();
shardingRuleConfig.getTableRuleConfigs().add(userTableRuleConfiguration());
shardingRuleConfig.getTableRuleConfigs().add(caipiaoTableRuleConfiguration());
return new ShardingDataSource(shardingRuleConfig.build(createDataSourceMap()));
}
/**
* 設定單表的分庫分表策略
* 虛擬表與實際表的對映關係
*/
@Bean
TableRuleConfiguration userTableRuleConfiguration() {
TableRuleConfiguration configuration = new TableRuleConfiguration();
configuration.setLogicTable("user");
configuration.setActualDataNodes("ds_${0..1}.user_${0..1}");
configuration.setDatabaseShardingStrategyConfig(new StandardShardingStrategyConfiguration("id", DatabaseShardingAlgorithm.class.getName()));
configuration.setTableShardingStrategyConfig(new StandardShardingStrategyConfiguration("sex", DatabaseShardingAlgorithm.class.getName()));
return configuration;
}
/**
* 設定不分庫分表的對映關係
*/
@Bean
TableRuleConfiguration caipiaoTableRuleConfiguration() {
TableRuleConfiguration configuration = new TableRuleConfiguration();
configuration.setLogicTable("caipiao");
configuration.setActualDataNodes("wms.caipiao");
return configuration;
}
/**
* 手動配置事務管理器
*/
@Bean
public DataSourceTransactionManager transactitonManager(DataSource shardingDataSource) {
return new DataSourceTransactionManager(shardingDataSource);
}
@Bean
@Primary
public SqlSessionFactory sqlSessionFactory(DataSource shardingDataSource) throws Exception {
SqlSessionFactoryBean bean = new SqlSessionFactoryBean();
bean.setDataSource(shardingDataSource);
bean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath:law/mapper/*.xml"));
return bean.getObject();
}
@Bean
@Primary
public SqlSessionTemplate testSqlSessionTemplate(SqlSessionFactory sqlSessionFactory) throws Exception {
return new SqlSessionTemplate(sqlSessionFactory);
}
/**
* 載入資料庫
*/
private Map<String, DataSource> createDataSourceMap() {
Map<String, DataSource> result = new HashMap<>();
result.put("ds_0", createDataSource("ds_0"));
result.put("ds_1", createDataSource("ds_1"));
result.put("wms", createDataSource("wms"));
return result;
}
private DataSource createDataSource(final String dataSourceName) {
DruidDataSource result = new DruidDataSource();
result.setDriverClassName(com.mysql.jdbc.Driver.class.getName());
result.setUrl(String.format("jdbc:mysql://localhost:3306/%s", dataSourceName) + "?useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&zeroDateTimeBehavior=convertToNull&useSSL=false");
result.setUsername("root");
result.setPassword("123456");
return result;
}
}
package law.configs; import io.shardingjdbc.core.api.algorithm.sharding.PreciseShardingValue; import io.shardingjdbc.core.api.algorithm.sharding.standard.PreciseShardingAlgorithm; import java.util.Collection;
/**
*分庫分表演算法，分為2個庫每個庫2個表
*以id欄位分庫，用id值除2取餘，分配到對應庫
*以sex欄位分表，用sex值除2取餘，分配到對應表
*/ public class DatabaseShardingAlgorithm implements PreciseShardingAlgorithm<Long> { @Override public String doSharding(Collection<String> collection, PreciseShardingValue<Long> preciseShardingValue) { for (String each : collection) { if (each.endsWith(Long.parseLong(preciseShardingValue.getValue().toString()) % 2 + "")) { return each; } } throw new IllegalArgumentException(); } }
五、啟動測試
package law.controller; import com.google.common.collect.Lists; import law.entity.Lottery; import law.entity.User; import law.mapper.LotteryMapper; import law.mapper.UserMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import java.util.List; @RestController public class UserController { @Autowired private UserMapper userMapper; @Autowired private LotteryMapper lotteryMapper; @Transactional @RequestMapping("/addUsers") public Object addUsers() { for (long i = 1; i < 10; i++) { User user = new User(); user.setId(i); user.setCity("北京"); user.setName("張" + i); user.setSex((long) 0); userMapper.addUser(user); } // //測試事務 // if (1 == 1) { // throw new RuntimeException(); // } for (long i = 11; i < 20; i++) { User user = new User(); user.setId(i); user.setCity("成都"); user.setName("劉" + i); user.setSex((long) 1); userMapper.addUser(user); } return "success"; }
//測試單表 @RequestMapping("/addLotteries") public Object addLotteries() { for (int i = 1; i < 3; i++) { Lottery lottery = new Lottery(); lottery.setCpNum(i); lottery.setCpValue("票" + i); lotteryMapper.addLottery(lottery); } return "success"; } @RequestMapping("/selectByIds") public Object selectByIds() { List<Long> ids = Lists.newArrayList(); ids.add(1L); ids.add(3L); return userMapper.list(ids); } @RequestMapping("/selectSort") public Object selectSort() { return userMapper.query(); } }
package law; import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.transaction.annotation.EnableTransactionManagement; @MapperScan(value = {"law.mapper"}) @EnableTransactionManagement(proxyTargetClass = true) @SpringBootApplication public class Demo { public static void main(String[] args) { SpringApplication.run(Demo.class, args); } }