一般來說，資料庫的資料過多，查詢效率就很慢，這時候我們如果把表分庫到不同的資料庫，這時候訪問速度就會快很多，如果並且採用多執行緒去訪問的話，查詢速度也會提高的更快，我這裡是執行記憶體8核電腦進行測試的單個訪問mysql資料庫的200萬資料，時間在8秒左右，但是把表分到10個數據庫裡面，每張表20萬，總共也是200萬，並且採用多執行緒訪問，這裡只需要3-5秒左右，下面是程式碼：

jar包

資料庫：這裡我們知道我是把所有資料庫都放到一個一個伺服器裡面，當我們有兩個伺服器，相當於每個伺服器是5個數據庫，訪問效率會更高，如果在往上加，效率就會更低，但是伺服器成本也高。

1. /**
2. *
3. */
4. package com.liuchaojun;
6. /**
7. * @author liuchaojun 資料庫訪問物件。用於插入資料。
8. * @date 2018-3-20 上午10:06:11
9. */
10. public class DemoDAO
11. {
12. private int a;
13. private String b;
14. private int c;
16. public int getA()
17. {
18. return a;
19. }
20. public void setA(int a)
21. {
22. this.a = a;
23. }
24. public String getB()
25. {
26. return b;
27. }
28. public void setB(String b)
29. {
30. this.b = b;
31. }
32. public int getC()
33. {
34. return c;
35. }
36. public void setC(int c)
37. {
38. this.c = c;
39. }
40. }

1. /*
2. */
3. package com.liuchaojun;
5. /**
6. * @author liuchaojun 儲存資料庫標識。每個執行緒由獨立的物件儲存
7. * @date 2018-3-20 上午10:05:13
8. */
9. public class DBIndetifier
10. {
11. private static ThreadLocal<String> dbKey = new ThreadLocal<String>();
13. public static void setDBKey(final String dbKeyPara)
14. {
15. dbKey.set(dbKeyPara);
16. }
18. public static String getDBKey()
19. {
20. return dbKey.get();
21. }
22. }

1. /**
2. *
3. */
4. package com.liuchaojun;
6. import java.lang.reflect.Method;
8. /**
9. * 資料庫訪問任務定義。將每一個對資料庫訪問的請求包裝為一個任務物件，放到任務管理中， 然後等待任務執行完成，取出執行結果。
10. *
11. * @author liuchaojun
12. */
13. public class DBTask implements Runnable {
14. // 操作資料庫標識，用於指定訪問的資料庫。與spring配置檔案中的資料動態資料來源定義一致。
15. private final String dbKey;
17. // mybatis資料庫訪問物件
18. private final Object dbAccessObject;
20. // mysbatis資料庫訪問方法名稱，用於反射呼叫
21. private final String methodName;
23. // 儲存可變引數的值
24. private final Object[] paraArray;
26. // 儲存可變引數型別
27. @SuppressWarnings("rawtypes")
28. private final Class[] paraClassArray;
30. // 資料庫操作結果。查詢操作返回查詢結果; 插入、刪除、修改操作返回null。
31. private Object operateResult;
33. // 操作資料庫丟擲的異常資訊
34. private Exception exception;
36. // 標識任務是否已經執行
37. private boolean finish;
39. /**
40. * 建構函式
41. *
42. * @param dbKey
43. * 資料庫標識
44. * @param dbAccessObject
45. * 資料庫訪問物件
46. * @param methodName
47. * 資料庫訪問方法名稱
48. * @param paraArray
49. * 引數列表
50. */
51. public DBTask(final String dbKey, final Object dbAccessObject,
52. final String methodName, final Object... paraArray) {
53. this.dbKey = dbKey;
54. this.dbAccessObject = dbAccessObject;
55. this.methodName = methodName;
56. this.paraArray = paraArray;
57. finish = false;
58. exception = null;
60. paraClassArray = new Class[paraArray.length];
61. for (int index = 0; index < paraArray.length; ++index) {
62. paraClassArray[index] = paraArray[index].getClass();
63. }
65. operateResult = null;
66. }
68. /**
69. *
70. * 任務執行函式
71. *
72. */
73. @Override
74. public void run() {
75. try {
76. DBIndetifier.setDBKey(dbKey);
77. Method method = dbAccessObject.getClass().getMethod(methodName,
78. paraClassArray);
80. // 查詢操作返回查詢結果; 插入、刪除、修改操作返回null
81. operateResult = method.invoke(dbAccessObject, paraArray);
82. } catch (Exception e) {
83. exception = e;
84. e.printStackTrace();
85. }
87. finish = true;
88. }
90. /**
91. *
92. * 返回操作結果。查詢操作返回查詢結果; 插入、刪除、修改操作返回null
93. *
94. * @return 操作結果
95. */
96. public Object getRetValue() {
97. return operateResult;
98. }
100. /**
101. * 丟擲資料庫操作異常
102. *
103. * @return 異常
104. */
105. public Exception getException() {
106. return exception;
107. }
109. /**
110. *
111. * 返回任務是否已執行
112. *
113. * @return 標記
114. */
115. public boolean isFinish() {
116. return finish;
117. }
118. }

1. /**
2. *
3. */
4. package com.liuchaojun;
6. import java.util.concurrent.ArrayBlockingQueue;
7. import java.util.concurrent.ThreadPoolExecutor;
8. import java.util.concurrent.TimeUnit;
10. /**
11. * @author liuchaojun 資料庫訪問任務管理。將資料庫訪問任務放到執行緒池中執行。
12. * @date 2018-3-20 上午10:07:58
13. */
14. public class DBTaskMgr {
15. private static class DBTaskMgrInstance {
16. public static final DBTaskMgr instance = new DBTaskMgr();
17. }
19. public static DBTaskMgr instance() {
20. return DBTaskMgrInstance.instance;
21. }
23. private ThreadPoolExecutor pool;
25. public DBTaskMgr() {
26. pool = new ThreadPoolExecutor(10, 50, 60, TimeUnit.SECONDS,
27. new ArrayBlockingQueue<Runnable>(10000),
28. new ThreadPoolExecutor.CallerRunsPolicy());
29. }
31. public void excute(Runnable task) {
32. pool.execute(task);
33. }
34. }

1. /**
2. *
3. */
4. package com.liuchaojun;
6. import java.io.Serializable;
8. /**
9. * @author liuchaojun 對映結果定義
10. * @date 2018-3-20 上午10:06:39
11. */
12. public class DemoResult implements Serializable
13. {
14. /**
15. * Comment for <code>serialVersionUID</code><br>
16. *
17. */
18. private static final long serialVersionUID = -413001138792531448L;
19. private DemoDAO demoDAO;
21. /**
22. * @return the demoDAO
23. */
24. public DemoDAO getDemoDAO() {
25. return demoDAO;
26. }
28. /**
29. * @param demoDAO the demoDAO to set
30. */
31. public void setDemoDAO(DemoDAO demoDAO) {
32. this.demoDAO = demoDAO;
33. }
36. }

1. package com.liuchaojun;
3. import java.util.List;
5. /**
6. * @author liuchaojun Mybatis 對映服務實現
7. * @date 2018-3-20 上午09:50:26
8. */
9. public class DemoServiceImpl implements IDemoService
10. {
11. private IDemo idemo = null;
13. public void setIdemo(IDemo idemo) {
14. this.idemo = idemo;
15. }
17. @Override
18. public void insertDemo(DemoDAO demo)
19. {
20. idemo.insertDemo(demo);
21. }
23. @Override
24. public List<Integer> selectGroup()
25. {
26. return idemo.selectGroup();
27. }
28. }

1. package com.liuchaojun;
3. import org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource;
5. /**
6. * @author liuchaojun 動態資料來源。可根據不同的資料索引連線不同的資料庫
7. * @date 2018-3-20 上午10:05:48
8. */
9. public class DynamicDataSource extends AbstractRoutingDataSource
10. {
11. @Override
12. public Object determineCurrentLookupKey()
13. {
14. return DBIndetifier.getDBKey();
15. }
16. }

1. package com.liuchaojun;
3. import java.util.List;
5. /**
6. * @author liuchaojun Mybatis 對映介面
7. * @date 2018-3-20 上午09:44:03
8. */
9. public interface IDemo
10. {
11. public void insertDemo(DemoDAO demo);
12. public List<Integer> selectGroup();
13. }

1. package com.liuchaojun;
3. import java.util.List;
5. /**
6. * @author liuchaojun Mybatis 對映服務介面
7. * @date 2018-3-20 上午09:45:34
8. */
9. public interface IDemoService
10. {
11. public void insertDemo(DemoDAO demo);
12. public List<Integer> selectGroup();
13. }

1. package com.liuchaojun;
3. import java.text.SimpleDateFormat;
4. import java.util.ArrayList;
5. import java.util.Date;
6. import java.util.List;
8. import org.springframework.context.ApplicationContext;
9. import org.springframework.context.support.ClassPathXmlApplicationContext;
11. /**
12. * @author liuchaojun
13. * @date 2018-3-20 上午10:09:12
14. */
15. public class TestMain {
16. /**
17. * 測試程式碼
18. *
19. * @param args
20. */
21. public static void main(String[] args) {
22. ApplicationContext context = new ClassPathXmlApplicationContext(
23. "cfg/spring.xml");
24. IDemoService service1 = (IDemoService) context.getBean("iDemoService");
26. // 建立任務物件
27. DBTask task1 = new DBTask("test1", service1, "selectGroup");
28. DBTask task2 = new DBTask("test2", service1, "selectGroup");
29. DBTask task3 = new DBTask("test3", service1, "selectGroup");
30. DBTask task4 = new DBTask("test4", service1, "selectGroup");
31. DBTask task5 = new DBTask("test5", service1, "selectGroup");
32. DBTask task6 = new DBTask("test6", service1, "selectGroup");
33. DBTask task7 = new DBTask("test7", service1, "selectGroup");
34. DBTask task8 = new DBTask("test8", service1, "selectGroup");
35. DBTask task9 = new DBTask("test9", service1, "selectGroup");
36. DBTask task10 = new DBTask("test10", service1, "selectGroup");
37. DBTask task_single = new DBTask("test_single", service1, "selectGroup");
39. // 單個執行緒直接查詢單表
40. SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
41. DBTaskMgr.instance().excute(task_single);
42. System.out.println("開始查詢test_single表中的200萬資料："+ format.format(new Date()));
43. while (true) {
44. if (!task_single.isFinish()) {
45. try {
46. Thread.sleep(1000);
47. } catch (InterruptedException e) {
48. e.printStackTrace();
49. }
50. } else {
51. break;
52. }
53. }
54. System.out.println("結束查詢test_single表中的200萬資料結束:"+ format.format(new Date()));
56. // 多個執行緒查詢多個數據庫中的表資料
57. List<DBTask> taskList = new ArrayList<DBTask>();
58. taskList.add(task1);
59. taskList.add(task2);
60. taskList.add(task3);
61. taskList.add(task4);
62. taskList.add(task5);
63. taskList.add(task6);
64. taskList.add(task7);
65. taskList.add(task8);
66. taskList.add(task9);
67. taskList.add(task10);
68. for (DBTask task : taskList) {
69. DBTaskMgr.instance().excute(task);
70. }
71. System.out.println("開始查詢10個數據庫中的t_test_table表中的20萬資料表："+ format.format(new Date()));
72. while (true) {
73. int success = 0;
74. for (DBTask task : taskList) {
75. if (!task.isFinish()) { // 執行執行緒run方法這裡為false,沒有執行擇true
76. continue;
77. } else {
78. ++success;
79. }
80. }
81. if (success == 10) {
82. break;
83. }
84. }
85. System.out.println("結束查詢10個數據庫中的t_test_table表中的20萬資料表："+ format.format(new Date()));
86. }
87. }

demoMapper.xml

1. <?xml version="1.0" encoding="UTF-8" ?>
2. <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
3. <mapper namespace="com.liuchaojun.IDemo">
4. <insert id="insertDemo" parameterType="com.liuchaojun.DemoDAO">
5. insert into t_test_table(a, b, c) values(#{a}, #{b}, #{c});
6. </insert>
8. <resultMap id="demoDAO" type="com.liuchaojun.DemoDAO">
9. <id column="a" property="a" />
10. <result column="b" property="b" />
11. <result column="c" property="c" />
12. </resultMap>
15. <select id="selectGroup" resultMap="demoDAO">
16. select * from t_test_table
17. </select>
18. </mapper>

mybatis.xml

1. <?xml version="1.0" encoding="UTF-8" ?>
2. <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
3. <mapper namespace="com.liuchaojun.IDemo">
4. <insert id="insertDemo" parameterType="com.liuchaojun.DemoDAO">
5. insert into t_test_table(a, b, c) values(#{a}, #{b}, #{c});
6. </insert>
8. <resultMap id="demoDAO" type="com.liuchaojun.DemoDAO">
9. <id column="a" property="a" />
10. <result column="b" property="b" />
11. <result column="c" property="c" />
12. </resultMap>
15. <select id="selectGroup" resultMap="demoDAO">
16. select * from t_test_table
17. </select>
18. </mapper>

spring.xml

1. <?xml version="1.0" encoding="UTF-8" ?>
2. <!DOCTYPE configuration PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
3. "http://mybatis.org/dtd/mybatis-3-config.dtd">
4. <configuration>
5. <mappers>
6. <mapper resource="cfg/demoMapper.xml"/>
7. </mappers>
8. </configuration>

1. <?xml version="1.0" encoding="UTF-8"?>
2. <beans xmlns="http://www.springframework.org/schema/beans"
3. xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4. xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd">
6. <bean id="dataSource_1" class="org.apache.commons.dbcp.BasicDataSource">
7. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
8. <property name="url" value="jdbc:mysql://localhost:3306/test1"></property>
9. <property name="username" value="root"></property>
10. <property name="password" value="root"></property>
11. <property name="maxActive" value="100"></property>
12. <property name="maxIdle" value="30"></property>
13. <property name="maxWait" value="500"></property>
14. <property name="defaultAutoCommit" value="true"></property>
15. </bean>
17. <bean id="dataSource_2" class="org.apache.commons.dbcp.BasicDataSource">
18. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
19. <property name="url" value="jdbc:mysql://localhost:3306/test2"></property>
20. <property name="username" value="root"></property>
21. <property name="password" value="root"></property>
22. <property name="maxActive" value="100"></property>
23. <property name="maxIdle" value="30"></property>
24. <property name="maxWait" value="500"></property>
25. <property name="defaultAutoCommit" value="true"></property>
26. </bean>
28. <bean id="dataSource_3" class="org.apache.commons.dbcp.BasicDataSource">
29. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
30. <property name="url" value="jdbc:mysql://localhost:3306/test3"></property>
31. <property name="username" value="root"></property>
32. <property name="password" value="root"></property>
33. <property name="maxActive" value="100"></property>
34. <property name="maxIdle" value="30"></property>
35. <property name="maxWait" value="500"></property>
36. <property name="defaultAutoCommit" value="true"></property>
37. </bean>
39. <bean id="dataSource_4" class="org.apache.commons.dbcp.BasicDataSource">
40. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
41. <property name="url" value="jdbc:mysql://localhost:3306/test4"></property>
42. <property name="username" value="root"></property>
43. <property name="password" value="root"></property>
44. <property name="maxActive" value="100"></property>
45. <property name="maxIdle" value="30"></property>
46. <property name="maxWait" value="500"></property>
47. <property name="defaultAutoCommit" value="true"></property>
48. </bean>
50. <bean id="dataSource_5" class="org.apache.commons.dbcp.BasicDataSource">
51. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
52. <property name="url" value="jdbc:mysql://localhost:3306/test5"></property>
53. <property name="username" value="root"></property>
54. <property name="password" value="root"></property>
55. <property name="maxActive" value="100"></property>
56. <property name="maxIdle" value="30"></property>
57. <property name="maxWait" value="500"></property>
58. <property name="defaultAutoCommit" value="true"></property>
59. </bean>
61. <bean id="dataSource_6" class="org.apache.commons.dbcp.BasicDataSource">
62. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
63. <property name="url" value="jdbc:mysql://localhost:3306/test6"></property>
64. <property name="username" value="root"></property>
65. <property name="password" value="root"></property>
66. <property name="maxActive" value="100"></property>
67. <property name="maxIdle" value="30"></property>
68. <property name="maxWait" value="500"></property>
69. <property name="defaultAutoCommit" value="true"></property>
70. </bean>
73. <bean id="dataSource_7" class="org.apache.commons.dbcp.BasicDataSource">
74. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
75. <property name="url" value="jdbc:mysql://localhost:3306/test7"></property>
76. <property name="username" value="root"></property>
77. <property name="password" value="root"></property>
78. <property name="maxActive" value="100"></property>
79. <property name="maxIdle" value="30"></property>
80. <property name="maxWait" value="500"></property>
81. <property name="defaultAutoCommit" value="true"></property>
82. </bean>
84. <bean id="dataSource_8" class="org.apache.commons.dbcp.BasicDataSource">
85. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
86. <property name="url" value="jdbc:mysql://localhost:3306/test8"></property>
87. <property name="username" value="root"></property>
88. <property name="password" value="root"></property>
89. <property name="maxActive" value="100"></property>
90. <property name="maxIdle" value="30"></property>
91. <property name="maxWait" value="500"></property>
92. <property name="defaultAutoCommit" value="true"></property>
93. </bean>
95. <bean id="dataSource_9" class="org.apache.commons.dbcp.BasicDataSource">
96. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
97. <property name="url" value="jdbc:mysql://localhost:3306/test9"></property>
98. <property name="username" value="root"></property>
99. <property name="password" value="root"></property>
100. <property name="maxActive" value="100"></property>
101. <property name="maxIdle" value="30"></property>
102. <property name="maxWait" value="500"></property>
103. <property name="defaultAutoCommit" value="true"></property>
104. </bean>
106. <bean id="dataSource_10" class="org.apache.commons.dbcp.BasicDataSource">
107. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
108. <property name="url" value="jdbc:mysql://localhost:3306/test10"></property>
109. <property name="username" value="root"></property>
110. <property name="password" value="root"></property>
111. <property name="maxActive" value="100"></property>
112. <property name="maxIdle" value="30"></property>
113. <property name="maxWait" value="500"></property>
114. <property name="defaultAutoCommit" value="true"></property>
115. </bean>
117. <bean id="dataSource_single" class="org.apache.commons.dbcp.BasicDataSource">
118. <property name="driverClassName" value="com.mysql.jdbc.Driver"></property>
119. <property name="url" value="jdbc:mysql://localhost:3306/test_single"></property>
120. <property name="username" value="root"></property>
121. <property name="password" value="root"></property>
122. <property name="maxActive" value="100"></property>
123. <property name="maxIdle" value="30"></property>
124. <property name="maxWait" value="500"></property>
125. <property name="defaultAutoCommit" value="true"></property>
126. </bean>
128. <bean id="dataSource" class="com.liuchaojun.DynamicDataSource">
129. <property name="targetDataSources">
130. <map>
131. <entry key="test1" value-ref="dataSource_1"/>
132. <entry key="test2" value-ref="dataSource_2"/>
133. <entry key="test3" value-ref="dataSource_3"/>
134. <entry key="test4" value-ref="dataSource_4"/>
135. <entry key="test5" value-ref="dataSource_5"/>
136. <entry key="test6" value-ref="dataSource_6"/>
137. <entry key="test7" value-ref="dataSource_7"/>
138. <entry key="test8" value-ref="dataSource_8"/>
139. <entry key="test9" value-ref="dataSource_9"/>
140. <entry key="test10" value-ref="dataSource_10"/>
141. <entry key="test_single" value-ref="dataSource_single"/>
142. </map>
143. </property>
144. </bean>
146. <bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
147. <property name="configLocation" value="classpath:cfg/mybatis.xml"></property>
148. <property name="dataSource" ref="dataSource" />
149. </bean>
151. <bean id="iDemo" class="org.mybatis.spring.mapper.MapperFactoryBean">
152. <property name="mapperInterface" value="com.liuchaojun.IDemo"></property>
153. <property name="sqlSessionFactory" ref="sqlSessionFactory"></property>
154. </bean>
156. <bean id="iDemoService" class="com.liuchaojun.DemoServiceImpl">
157. <property name="idemo" ref="iDemo"></property>
158. </bean>
160. </beans>

資料表的批量插入資料儲存過程：
DROP PROCEDURE IF EXISTS pro_test1_insert;
DELIMITER //
CREATE PROCEDURE pro_test1_insert(cnt int)
BEGIN
DECLARE i int DEFAULT 0;
START TRANSACTION;
WHILE i<cnt
DO
INSERT INTO t_test_table(b,c) VALUES(i,'168');
set i=i+1;
end WHILE;
COMMIT;
end //
DELIMITER;

CALL pro_test1_insert(1000000);

1. <span style="color:#454545">
2. </span>

專案原始碼資源下載地址：https://download.csdn.net/download/qq_27026603/10298566