【資料庫效能測試實戰】測試不同分頁儲存過程在10w，100w以及1000w資料量下面的表現
阿新 • • 發佈：2018-12-18
前言
資料庫的效能與每一行程式碼息息相關，所以，每次寫程式碼可以考慮一下在不同級別的資料量下面測試一下效能。
本文參考了：
Postgresql生成大量測試資料
以及
準備測試用資料
此次測試我們將分別用10w，100w以及1000w級別的表來測試，下面先建立一波資料表。
-- 測試用資料表，分別為10w，100w以及1000w級別。表結構基本一致。 create table tbl_test_10w ( id serial primary key, "name" varchar(200) null, "groupid" integer null, create_time timestamp(0) without time zone ); insert into tbl_test_10w (name,groupid,create_time) select md5(n::varchar||random()::varchar) as "name", (floor(RANDOM()*(100))::int%15) as groupid, ('2018-5-1'::date + trunc(random()*100)::integer +' 00:22:22'::time + (trunc(random()*3600*24)||' second')::interval) as create_time from generate_series(0,100000) n; select count(*) from tbl_test_10w; -- 100w資料量 create table tbl_test_100w ( id serial primary key, "name" varchar(200) null, "groupid" integer null, create_time timestamp(0) without time zone ); insert into tbl_test_100w (name,groupid,create_time) select md5(n::varchar||random()::varchar) as "name", (floor(RANDOM()*(100))::int%15) as groupid, ('2018-5-1'::date + trunc(random()*100)::integer +' 00:22:22'::time + (trunc(random()*3600*24)||' second')::interval) as create_time from generate_series(0,1000000) n; select count(*) from tbl_test_100w; -- 1000w資料量 create table tbl_test_1000w ( id serial primary key, "name" varchar(200) null, "groupid" integer null, create_time timestamp(0) without time zone ); insert into tbl_test_1000w (name,groupid,create_time) select md5(n::varchar||random()::varchar) as "name", (floor(RANDOM()*(100))::int%15) as groupid, ('2018-5-1'::date + trunc(random()*100)::integer +' 00:22:22'::time + (trunc(random()*3600*24)||' second')::interval) as create_time from generate_series(0,10000000) n; select count(*) from tbl_test_1000w;
接下來，看看這些記錄佔用空間：
select pg_size_pretty(pg_relation_size('tbl_test_1000w'));
效能測試工具
待會需要用到：top free vmstat iostat【可能會用】等工具，用的是centos7系統，
還有，可以參考：
效能測試
先說一下渣機，阿里雲上面的配置：
然後再說明一下測試程式碼，首先，全域性配置 config：
-- 這是全域性配置 create or replace function config( ) returns jsonb language 'plpgsql' IMMUTABLE as $body$ declare job_manager_config constant jsonb := '{ "pager":{ "pageSize":20 } }'; BEGIN return job_manager_config; END; $body$; -- 快速獲取配置項。 create or replace function config( variadic path_array text[] ) returns text LANGUAGE 'plpgsql' IMMUTABLE as $body$ begin return jsonb_extract_path_text(config(),variadic path_array); end; $body$;
針對系統的自定義型別：
/*** 系統基本型別。 ***/ -- 2018-10-28 drop type sys_types_pager_outline; create type sys_type_pager_outline as ( "pageIndex" integer, "pageSize" integer, "total" integer, "totalPages" integer, "beginIndex" integer, "state" boolean, "stateCode" integer, "message" varchar(600) ); drop type sys_type_operation_outline; create type sys_type_operation_outline as ( "state" boolean, "stateCode" integer, "message" varchar(600) );
分頁偏移量計算：
/****系統基本函式**/
-- 計算分頁的偏移還有總頁碼等資訊。
create or replace function sys_func_cal_pager(
in pageIndex integer,
in pageSize integer,
in totalSize integer
)
returns sys_type_pager_outline
as
$body$
declare pagerOutLine sys_type_pager_outline;
begin
-- 初始化分頁基本資料。
pagerOutLine."pageIndex":=1;
pagerOutLine."pageSize":=config('pager','pageSize');
pagerOutLine."total":=0;
pagerOutLine."totalPages":=0;
pagerOutLine."beginIndex":=0;
if pageIndex > 0 then
pagerOutLine."pageIndex":=pageIndex;
end if;
if pageSize > 0 then
pagerOutLine."pageSize"=pageSize;
end if;
if totalSize > 0 then
pagerOutLine."total":=totalSize;
-- 計算總頁碼
if pagerOutLine."total" % pagerOutLine."pageSize" = 0 then
pagerOutLine."totalPages" := pagerOutLine."total"/pagerOutLine."pageSize";
else
pagerOutLine."totalPages" := (pagerOutLine."total"-(pagerOutLine."total"%pagerOutLine."pageSize"))/pagerOutLine."pageSize"+1;
end if;
-- 計算當前頁碼是不是超出正常水平
if pagerOutLine."pageIndex" > pagerOutLine."totalPages" then
pagerOutLine."pageIndex" := pagerOutLine."totalPages";
end if;
-- 計算offset
pagerOutLine."beginIndex"=(pagerOutLine."pageIndex"-1) * pagerOutLine."pageSize";
else
pagerOutLine."total":=0;
pagerOutLine."totalPages":=0;
pagerOutLine."pageIndex":=1;
pagerOutLine."beginIndex":=0;
end if;
return pagerOutLine;
end;
$body$ language plpgsql volatile;
好了，接下來才是正主，幾個測試用分頁儲存過程：
/*********為了可以測試各個數量級的效能，請屆時將tbl_test_1000w這個名稱換成100w以及10w級別的。*************/
/***
*
* 測試用分頁儲存過程，其中利用了臨時表儲存中間記錄集
*
***/
CREATE OR REPLACE FUNCTION public.test_case_search_pager_using_temp_table(
in pageindex integer,
in pagesize integer,
cnd_eq_id integer,
cnd_min_groupid integer,
cnd_max_groupid integer,
cnd_begin_create_time timestamp without time zone,
cnd_end_create_time timestamp without time zone,
outline_cursor refcursor,
records_cursor refcursor)
RETURNS setof refcursor
AS $BODY$
declare pagerOutLine sys_type_pager_outline;
declare totalSize integer;
declare tmpStr varchar(400);
begin
-- 根據條件查詢列表，然後存放到臨時表裡面去。
create temp table "temp_tbl4pager"
on commit drop
as select * from
(
select * from tbl_test_1000w
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end
and
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end
and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
) tbl_middle;
select count(*) into totalSize from temp_tbl4pager;
pagerOutLine:=sys_func_cal_pager(pageIndex,pageSize,totalSize);
-- raise notice '好了，看看pagerOutline的資料：';
-- raise notice '看看pagerOutLine裡面的資料：%，真奇怪，那麼，totalSize 是：%',pagerOutLine,totalSize;
-- 定義返回的狀態。
pagerOutLine."state":=true;
pagerOutLine."stateCode":=0;
pagerOutLine."message":='';
-- 返回相關資料。
open outline_cursor for select pagerOutLine;
open records_cursor for select * from temp_tbl4pager limit pagerOutLine."pageSize" offset pagerOutLine."beginIndex";
-- 返回資料
return next outline_cursor;
return next records_cursor;
end;
$BODY$ LANGUAGE plpgsql volatile;
/***
*
* 測試用分頁儲存過程，其中使用了cte
*
***/
CREATE OR REPLACE FUNCTION public.test_case_search_pager_using_cte(
in pageindex integer,
in pagesize integer,
cnd_eq_id integer,
cnd_min_groupid integer,
cnd_max_groupid integer,
cnd_begin_create_time timestamp without time zone,
cnd_end_create_time timestamp without time zone,
outline_cursor refcursor,
records_cursor refcursor)
RETURNS setof refcursor
AS $BODY$
declare pagerOutLine sys_type_pager_outline;
declare totalSize integer;
declare tmpStr varchar(400);
begin
-- 首先獲取記錄總數
with tbl_middle as (
select * from tbl_test_1000w
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end
and
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end
and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
)
select count(*) into totalSize from tbl_middle;
-- 然後計算beginIndex，totalPages
pagerOutLine:=sys_func_cal_pager(pageIndex,pageSize,totalSize);
-- raise notice '好了，看看pagerOutline的資料：';
-- raise notice '看看pagerOutLine裡面的資料：%，真奇怪，那麼，totalSize 是：%',pagerOutLine,totalSize;
-- 定義返回的狀態。
pagerOutLine."state":=true;
pagerOutLine."stateCode":=0;
pagerOutLine."message":='';
-- 返回相關資料。
open outline_cursor for select pagerOutLine;
open records_cursor for select * from
(
select * from tbl_test_1000w
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end
and
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end
and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
) tbl_middle
limit pagerOutLine."pageSize" offset pagerOutLine."beginIndex";
-- 返回資料
return next outline_cursor;
return next records_cursor;
end;
$BODY$ LANGUAGE plpgsql volatile;
/***
*
* 測試用分頁儲存過程，原始版本，不用cte等等特性。
*
***/
CREATE OR REPLACE FUNCTION public.test_case_search_pager_origin(
in pageindex integer,
in pagesize integer,
cnd_eq_id integer,
cnd_min_groupid integer,
cnd_max_groupid integer,
cnd_begin_create_time timestamp without time zone,
cnd_end_create_time timestamp without time zone,
outline_cursor refcursor,
records_cursor refcursor)
RETURNS setof refcursor
AS $BODY$
declare pagerOutLine sys_type_pager_outline;
declare totalSize integer;
declare tmpStr varchar(400);
begin
-- 首先獲取記錄總數
select count(*) into totalSize from
(
select * from tbl_test_1000w
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end
and
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end
and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
) tbl_middle
;
-- 然後計算beginIndex，totalPages
pagerOutLine:=sys_func_cal_pager(pageIndex,pageSize,totalSize);
-- raise notice '好了，看看pagerOutline的資料：';
-- raise notice '看看pagerOutLine裡面的資料：%，真奇怪，那麼，totalSize 是：%',pagerOutLine,totalSize;
-- 定義返回的狀態。
pagerOutLine."state":=true;
pagerOutLine."stateCode":=0;
pagerOutLine."message":='';
-- 返回相關資料。
open outline_cursor for select pagerOutLine;
open records_cursor for select * from
(
select * from tbl_test_1000w
where 1=1
and
case when cnd_eq_id is not null then "id"=cnd_eq_id else 1=1 end
and
case when cnd_min_groupid is not null then "groupid">cnd_min_groupid else 1=1 end
and
case when cnd_max_groupid is not null then "groupid"<cnd_max_groupid else 1=1 end
and
case when cnd_begin_create_time is not null then "create_time">cnd_begin_create_time else 1=1 end
and
case when cnd_end_create_time is not null then "create_time"< cnd_end_create_time else 1=1 end
) tbl_middle
limit pagerOutLine."pageSize" offset pagerOutLine."beginIndex";
-- 返回資料
return next outline_cursor;
return next records_cursor;
end;
$BODY$ LANGUAGE plpgsql volatile;
先測試1000w條記錄的資料庫的讀取時間…
首先，用兩個ssh連線到阿里雲伺服器。。嗯，每個視窗分別準備一個命令：
-- 每一秒繪製一次記憶體使用情況，持續60次。採用mb單位顯示。
vmstat 1 60 -S M
以及：
-- 列出所有postgres使用者持有的程序，每次檢測間隔1s，批次執行模式方便監測，對了 -c表示要顯示完整的命令。
top -b -c -d 1 -U postgres
如下圖：
然後準備一段呼叫程式碼：
begin;
select test_case_search_pager_using_temp_table(
2, -- pageIndex
399, -- pageSize
null, -- equal id
7,-- min group id
19, -- max group id
null,-- begin create time
null,-- end create time
'outline_crs_temp_table',
'records_crs_temp_table');
fetch all in outline_crs_temp_table;
fetch all in records_crs_temp_table;
end;
然後，先執行top和vmstat進行監測，再執行sql程式碼，你會發現如下結果：
1000w的排序足足用了5秒多，搞不好會上6秒的。。。嗯。。。。
然後，檢視監測情況：
有一段時間，記憶體是急劇消耗的，最低空閒記憶體降到115mb ，
而cpu方面的 監測圖例有：
又多又雜的跟蹤，很難用肉眼來判斷。。
而我們注意到，20522這個pid似乎是用得最多，壓根就是我們執行的pid，於是，我們可以直接改改命令，新增pid指定，方便監控。
改為：
-- 注意，-U -p這些沒辦法同時使用。。
top -b -c -d 1 -p 20522
然後再嘗試一次。。
得到結果：
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
4 0 0 1060 7 400 0 0 12 46 9 10 2 1 96 0 0
0 0 0 1060 7 400 0 0 0 40 369 1008 1 0 99 0 0
0 0 0 1059 7 400 0 0 0 76 388 1042 2 2 96 0 0
14 0 0 971 7 489 0 0 65048 240 783 1033 18 7 74 1 0
1 0 0 612 7 848 0 0 252456 58000 1781 738 70 29 0 1 0
2 0 0 296 7 1164 0 0 222092 114824 1634 815 62 25 0 13 0
4 0 0 130 7 1330 0 0 62176 114832 1571 834 73 21 0 6 0
3 0 0 103 1 1172 0 0 4 40 1400 727 65 35 0 0 0
1 0 0 1094 1 371 0 0 24 192 1161 823 29 51 19 1 0
4 0 0 1090 1 375 0 0 4712 52 458 1083 2 1 85 12 0
而top的結果是：
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 405988 133680 120628 S 0.0 3.4 0:18.31 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 23:30:16 up 47 days, 12:01, 4 users, load average: 0.10, 0.72, 0.68
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 1.0 us, 1.0 sy, 0.0 ni, 98.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 1085684 free, 2378848 used, 417520 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1107460 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 405988 133680 120628 S 0.0 3.4 0:18.31 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 23:30:17 up 47 days, 12:01, 4 users, load average: 0.09, 0.71, 0.68
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 5.9 us, 2.9 sy, 0.0 ni, 91.2 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 1085684 free, 2378848 used, 417520 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1107460 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 405988 133940 120888 R 4.0 3.5 0:18.35 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:30:18 up 47 days, 12:01, 4 users, load average: 0.09, 0.71, 0.68
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 69.7 us, 28.3 sy, 0.0 ni, 0.0 id, 2.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 697940 free, 2378356 used, 805756 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1106284 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 405988 133940 120888 R 96.0 3.5 0:19.31 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:30:19 up 47 days, 12:01, 4 users, load average: 0.09, 0.71, 0.68
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 66.0 us, 26.0 sy, 0.0 ni, 0.0 id, 8.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 697940 free, 2378356 used, 805756 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1106284 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 405988 133940 120888 R 88.1 3.5 0:20.20 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:30:20 up 47 days, 12:01, 4 users, load average: 0.09, 0.71, 0.68
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 71.0 us, 23.0 sy, 0.0 ni, 0.0 id, 6.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 697940 free, 2378356 used, 805756 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1106284 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 405988 133940 120888 R 88.0 3.5 0:21.08 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:30:21 up 47 days, 12:01, 4 users, load average: 0.09, 0.71, 0.68
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 67.6 us, 27.5 sy, 0.0 ni, 0.0 id, 4.9 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 125756 free, 2444088 used, 1312208 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1037680 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 478612 199908 120892 R 91.1 5.1 0:22.00 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:30:22 up 47 days, 12:01, 4 users, load average: 0.96, 0.88, 0.73
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 38.4 us, 60.6 sy, 0.0 ni, 1.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 125756 free, 2444088 used, 1312208 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1037680 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 405972 133928 120892 S 88.0 3.4 0:22.88 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 23:30:23 up 47 days, 12:01, 4 users, load average: 0.96, 0.88, 0.73
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 2.0 us, 1.0 sy, 0.0 ni, 84.2 id, 12.9 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 125756 free, 2444088 used, 1312208 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1037680 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 405972 133928 120892 S 0.0 3.4 0:22.88 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
嗯，臨時表的耗時是5秒以上。。。
下面來看看cte的，兩次查詢，不用臨時表，看看效率。
測試程式碼：
begin;
-- select test_case_search_pager_using_temp_table(
select test_case_search_pager_using_temp_table(
2, -- pageIndex
399, -- pageSize
null, -- equal id
7,-- min group id
19, -- max group id
null,-- begin create time
null,-- end create time
'outline_crs_temp_table',
'records_crs_temp_table');
fetch all in outline_crs_temp_table;
fetch all in records_crs_temp_table;
end;
其他跟之前的一樣。
可以看到：
還是5秒多，那麼看看其他結果：
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
5 0 0 1125 0 342 0 0 13 47 9 0 2 1 96 0 0
0 0 0 1124 0 342 0 0 0 116 419 1000 4 2 93 1 0
0 0 0 1124 0 342 0 0 0 40 341 976 1 1 98 0 0
0 0 0 1124 0 342 0 0 0 72 374 1013 2 1 97 0 0
0 0 0 1124 0 342 0 0 0 40 361 994 1 0 99 0 0
3 1 0 1041 0 426 0 0 61536 56 712 992 16 6 78 0 0
1 0 0 670 0 797 0 0 261788 53388 1708 731 71 28 0 1 0
1 0 0 321 0 1146 0 0 242284 106792 1746 804 69 27 0 4 0
1 0 0 167 0 1300 0 0 63980 113120 1334 934 58 19 0 23 0
2 0 0 145 0 1105 0 0 0 92 1362 743 65 35 0 0 0
0 0 0 1129 0 337 0 0 256 48 1057 789 27 45 27 0 0
0 0 0 1130 0 337 0 0 0 40 350 966 2 0 98 0 0
0 0 0 1129 0 337 0 0 0 140 349 954 2 1 97 0 0
0 0 0 1130 0 337 0 0 668 40 373 986 1 0 98 1 0
0 0 0 1129 0 338 0 0 248 40 372 1007 2 1 97 0 0
0 0 0 1129 0 338 0 0 0 40 331 958 1 0 99 0 0
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 406156 135212 121984 S 0.0 3.5 0:55.59 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 23:52:44 up 47 days, 12:23, 4 users, load average: 0.05, 0.46, 0.62
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 1.0 us, 0.0 sy, 0.0 ni, 99.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 1151460 free, 2379272 used, 351320 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1120732 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 406156 135212 121984 S 0.0 3.5 0:55.59 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 23:52:45 up 47 days, 12:23, 4 users, load average: 0.05, 0.46, 0.62
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 15.0 us, 6.0 sy, 0.0 ni, 79.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 1151460 free, 2379272 used, 351320 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1120732 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 406156 135212 121984 R 18.0 3.5 0:55.77 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:52:46 up 47 days, 12:24, 4 users, load average: 0.05, 0.46, 0.62
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 71.0 us, 28.0 sy, 0.0 ni, 0.0 id, 1.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 693076 free, 2378228 used, 810748 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1106416 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 406156 135212 121984 R 97.0 3.5 0:56.75 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:52:47 up 47 days, 12:24, 4 users, load average: 0.21, 0.48, 0.63
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 69.0 us, 27.0 sy, 0.0 ni, 0.0 id, 4.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 693076 free, 2378228 used, 810748 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1106416 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 406156 135212 121984 R 93.0 3.5 0:57.68 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:52:48 up 47 days, 12:24, 4 users, load average: 0.21, 0.48, 0.63
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 59.0 us, 18.0 sy, 0.0 ni, 0.0 id, 23.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 693076 free, 2378228 used, 810748 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1106416 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 406156 135212 121984 R 73.3 3.5 0:58.42 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:52:49 up 47 days, 12:24, 4 users, load average: 0.21, 0.48, 0.63
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 65.3 us, 34.7 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 101368 free, 2591208 used, 1189476 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 890824 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 626256 347796 122140 R 97.0 9.0 0:59.39 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:52:50 up 47 days, 12:24, 4 users, load average: 0.21, 0.48, 0.63
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 28.3 us, 46.5 sy, 0.0 ni, 25.3 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 101368 free, 2591208 used, 1189476 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 890824 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 406156 135368 122140 S 65.3 3.5 1:00.05 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 23:52:51 up 47 days, 12:24, 4 users, load average: 0.21, 0.48, 0.63
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 2.0 us, 0.0 sy, 0.0 ni, 98.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 101368 free, 2591208 used, 1189476 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 890824 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 406156 135368 122140 S 0.0 3.5 1:00.05 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
差距沒多大啊。。
下面看看不用cte的。。
begin;
-- select test_case_search_pager_using_temp_table(
-- select test_case_search_pager_using_temp_table(
select test_case_search_pager_origin(
2, -- pageIndex
399, -- pageSize
null, -- equal id
7,-- min group id
19, -- max group id
null,-- begin create time
null,-- end create time
'outline_crs_temp_table',
'records_crs_temp_table');
fetch all in outline_crs_temp_table;
fetch all in records_crs_temp_table;
end;
然後測試：
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
4 0 0 456 4 1006 0 0 13 47 9 0 2 1 96 0 0
0 0 0 455 4 1006 0 0 0 40 389 1066 1 1 98 0 0
0 0 0 456 4 1006 0 0 0 52 368 1022 1 1 98 0 0
2 0 0 456 4 1006 0 0 0 40 291 924 1 1 98 0 0
0 0 0 456 4 1006 0 0 0 100 346 991 1 0 99 0 0
0 0 0 456 4 1006 0 0 0 40 304 942 1 1 98 0 0
0 0 0 455 4 1006 0 0 0 40 361 976 2 1 97 0 0
3 0 0 452 4 1008 0 0 1892 0 1230 3788 48 32 17 3 0
1 0 0 451 4 1008 0 0 0 80 1420 4396 60 40 0 0 0
1 0 0 452 4 1008 0 0 0 108 1406 4228 62 38 0 0 0
9 0 0 446 4 1009 0 0 0 112 1341 3003 68 32 0 0 0
13 0 0 447 4 1009 0 0 0 40 1272 1069 83 17 0 0 0
3 0 0 452 4 1008 0 0 0 40 1369 3571 64 36 0 0 0
2 0 0 452 4 1008 0 0 0 40 1390 4355 59 41 0 0 0
1 0 0 453 4 1008 0 0 0 40 1406 4523 56 44 0 0 0
4 0 0 452 4 1008 0 0 148 176 1406 4359 58 42 0 0 0
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 409836 139740 125028 S 0.0 3.6 1:01.77 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 23:59:31 up 47 days, 12:30, 4 users, load average: 1.72, 0.58, 0.56
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 80.8 us, 19.2 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 463120 free, 2381500 used, 1037432 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1104724 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 410172 139740 125028 R 16.0 3.6 1:01.93 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 23:59:32 up 47 days, 12:30, 4 users, load average: 1.90, 0.63, 0.58
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 79.2 us, 19.8 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
KiB Mem : 3882052 total, 463096 free, 2381364 used, 1037592 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1104708 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 409836 140176 125464 S 14.0 3.6 1:02.07 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 23:59:33 up 47 days, 12:30, 4 users, load average: 1.90, 0.63, 0.58
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 58.0 us, 40.0 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 2.0 si, 0.0 st
KiB Mem : 3882052 total, 463096 free, 2381364 used, 1037592 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1104708 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 409836 140176 125464 S 1.0 3.6 1:02.08 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
這裡我先解釋一下為什麼記憶體會突然之間降低了這麼多一直沒見恢復的。。。是不是洩露了？
不是的。。因為剛才中間第三個儲存過程報錯了，結果空閒記憶體一下子降了這麼多。。我也很無奈的。
報錯時候執行的在這裡：
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
4 0 0 1056 3 407 0 0 13 47 9 0 2 1 96 0 0
0 0 0 1056 3 407 0 0 0 40 359 1004 1 0 99 0 0
0 0 0 1056 3 407 0 0 0 68 349 994 2 1 97 0 0
0 0 0 1056 3 407 0 0 0 40 337 979 1 0 99 0 0
10 5 0 866 4 592 0 0 189020 104 1017 1185 27 10 54 9 0
4 3 0 494 4 964 0 0 380656 60 1727 1230 72 17 0 11 0
0 0 0 457 4 1005 0 0 42972 40 1048 957 53 7 35 5 0
0 0 0 457 4 1005 0 0 0 56 347 987 2 1 97 0 0
0 0 0 457 4 1005 0 0 0 40 316 949 1 1 98 0 0
0 0 0 457 4 1005 0 0 0 120 319 958 1 0 99 0 0
0 0 0 457 4 1005 0 0 0 40 323 969 1 1 98 0 0
0 0 0 457 4 1005 0 0 0 40 324 956 2 0 98 0 0
0 0 0 457 4 1005 0 0 0 76 309 947 1 1 98 0 0
0 0 0 457 4 1005 0 0 0 40 320 955 1 2 97 0 0
0 0 0 457 4 1005 0 0 0 96 324 963 0 0 100 0 0
0 0 0 458 4 1005 0 0 0 40 343 978 2 1 97 0 0
0 0 0 458 4 1005 0 0 0 40 348 973 1 1 98 0 0
0 0 0 458 4 1005 0 0 0 40 349 989 1 0 99 0 0
從一千多降到460m左右。。不見恢復。。
好了，說說第三次的結論。。
1.7s。。。。額。。是臨時表還有cte的三分一消耗。。而且，記憶體不減，cpu消耗也不高。這是開玩笑嘛？
下面來調整搜尋條件還有先進行一下分析看看結果。
好了，調整後還是分析還是認為1.7s，那麼執行實際情況，監測：
begin;
-- select test_case_search_pager_using_temp_table(
-- select test_case_search_pager_using_temp_table(
select test_case_search_pager_origin(
7, -- pageIndex
319, -- pageSize
null, -- equal id
1,-- min group id
11, -- max group id
null,-- begin create time
null,-- end create time
'outline_crs_temp_table',
'records_crs_temp_table');
fetch all in outline_crs_temp_table;
fetch all in records_crs_temp_table;
end;
結果：
兩秒鐘！結果還是遠遠低於cte還有臨時表！
其他表現：
r b swpd free buff cache si so bi bo in cs us sy id wa st
6 0 0 445 7 1011 0 0 13 47 9 1 2 1 96 0 0
1 0 0 445 7 1011 0 0 0 40 1389 4451 62 38 0 0 0
3 0 0 445 7 1011 0 0 0 148 1393 4545 58 42 0 0 0
3 0 0 445 7 1011 0 0 0 40 1383 4555 58 42 0 0 0
2 0 0 445 7 1011 0 0 0 40 1382 4518 58 42 0 0 0
4 0 0 440 7 1011 0 0 0 116 1256 2036 79 21 0 0 0
6 0 0 439 7 1011 0 0 0 40 1250 1078 87 13 0 0 0
2 0 0 445 7 1011 0 0 0 124 1373 4330 62 38 0 0 0
6 0 0 445 7 1011 0 0 0 92 1392 4516 59 41 0 0 0
1 0 0 445 7 1011 0 0 0 0 1370 4343 62 38 0 0 0
4 0 0 445 7 1011 0 0 0 40 1344 2263 83 17 0 0 0
3 0 0 446 7 1011 0 0 0 40 1383 4305 57 43 0 0 0
2 0 0 446 7 1011 0 0 0 40 1383 4315 57 43 0 0 0
top - 00:07:18 up 47 days, 12:38, 4 users, load average: 4.03, 1.81, 1.09
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 58.6 us, 40.4 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
KiB Mem : 3882052 total, 456592 free, 2382092 used, 1043368 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1102968 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 410084 141056 125984 S 0.0 3.6 1:02.44 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 00:07:19 up 47 days, 12:38, 4 users, load average: 4.03, 1.81, 1.09
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 72.3 us, 26.7 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
KiB Mem : 3882052 total, 456592 free, 2382092 used, 1043368 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1102968 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 410420 141320 126108 R 8.9 3.6 1:02.53 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 00:07:20 up 47 days, 12:38, 4 users, load average: 4.03, 1.81, 1.09
Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie
%Cpu(s): 86.0 us, 14.0 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 3882052 total, 456592 free, 2382092 used, 1043368 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1102968 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 410420 141320 126108 R 18.0 3.6 1:02.71 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) SELECT
top - 00:07:21 up 47 days, 12:38, 4 users, load average: 4.03, 1.81, 1.09
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 68.6 us, 30.4 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
KiB Mem : 3882052 total, 456460 free, 2382068 used, 1043524 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1102844 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
20522 postgres 20 0 410084 141396 126184 S 6.9 3.6 1:02.78 postgres: dbuser dev_op_system xxx.xxxxx.xxxx.xxxx(29661) idle
top - 00:07:22 up 47 days, 12:38, 4 users, load average: 4.43, 1.93, 1.14
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
%Cpu(s): 59.6 us, 39.4 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
KiB Mem : 3882052 total, 456460 free, 2382068 used, 1043524 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1102844 avail Mem
記憶體消耗很少，不到20%的cpu利用率。。額，我這種渣機器竟然能夠拖動千萬級別表的除了主鍵以外其他地方沒有加索引的任意條件查詢！
這個簡直是逆天了。
而使用臨時表還有cte的。。。感覺就應了一句話： 特性一時爽，全家火葬場。。。
100w及10w級別的已經不想測試了，因為。。。1000w級別的資料也能查詢這麼快了。。嗯，我何必再跟小兒科斤斤計較。。