mongo索引批量在postgres裡面重建
阿新 • • 發佈：2020-08-12
一些etl工具從mongo同步到pg庫裡面的資料，多數能處理這種異構的索引，我自己找了一種不那麼複雜的方法：
(1)寫一個js,在mongo匯出源庫所有索引資訊：
1 root@ip-10-10-30-211:~# cat mongo_idx.js 2 var collectionList = db.getCollectionNames(); 3 for(var index in collectionList){ 4 var collection = collectionList[index]; 5 var cur = db.getCollection(collection).getIndexes();6 if(cur.length == 1){ 7 continue; 8 } 9 for(var index1 in cur){ 10 var next = cur[index1]; 11 if(next["key"]["_id"] == '1'){ 12 continue; 13 } 14 print("(\""+collection+"\")" +","+JSON.stringify(next.key))}}
然後匯出的顯示格式為：
#mongo mongo-xxxx-pri.xxxx.space:3717/databasename -usyncuser -pxxxx mongo_idx.js > monidx.json
#cat monidx.json ("mg_result_all"),{"login":1,"brokerid":1,"usertype":1} ("mg_result_all"),{"publishfrozentime":1,"usertype":1,"accountstatus":1,"accounttype":1,"equity":1,"period_trade":1} ("mg_result_all"),{"rate_ss_profit_balance_close":-1} ("mg_result_brsymday"),{"login":1,"brokerid":1} ("mg_result_brsymday"),{"login":1,"brokerid":1,"close_date":1} ("mg_result_day"),{"login":1,"brokerid":1,"close_date":1} ("mg_result_follall"),{"login":1,"brokerid":1} ("mg_result_follall"),{"masteraccount":1,"masterbrokerid":1} ("mg_result_follall"),{"login":1,"brokerid":1,"usertype":1,"masteraccount":1,"masterbrokerid":1,"masterusertype":1} ("mg_result_follall"),{"masteraccount":1,"masterbrokerid":1,"close_weekofyear":1} ("mg_result_follday"),{"login":1,"brokerid":1,"usertype":1,"masteraccount":1,"masterbrokerid":1,"masterusertype":1} ("mg_result_follday"),{"login":1,"brokerid":1,"close_date":1} ("mg_result_follday"),{"masteraccount":1,"masterbrokerid":1,"close_date":1} ("mg_result_follday"),{"login":1,"brokerid":1,"masteraccount":1,"masterbrokerid":1,"close_date":1} ("mg_result_follday"),{"login":1,"brokerid":1,"masteraccount":1,"masterbrokerid":1,"close_weekofyear":1} ("mg_result_follweekofyear"),{"login":1,"brokerid":1,"standardsymbol":1,"close_weekofyear":1} ("mg_result_hour"),{"login":1,"brokerid":1} ("mg_result_month"),{"login":1,"brokerid":1,"close_month":1} ("mg_result_nusymall"),{"standardsymbol":1} ("mg_result_order"),{"login":1,"brokerid":1,"ticket":1} ("mg_result_sorted_all"),{"statName":1,"updatets":1} ("mg_result_sorted_follweekofyear"),{"statName":1,"updatets":1,"close_weekofyear":1} ("mg_result_sorted_follweekofyear"),{"statName":1,"updatets":1,"close_weekofyear":1,"login":1,"brokerid":1} ("mg_result_sorted_weekofyear"),{"statName":1,"updatets":1,"close_weekofyear":1} ("mg_result_sorted_weekofyear"),{"statName":1,"updatets":1,"close_weekofyear":1,"login":1,"brokerid":1} ("mg_result_symall"),{"login":1,"brokerid":1,"standardsymbol":1} ("mg_result_symall"),{"login":1,"brokerid":1,"money_profit_close_max":1} ("mg_result_symday"),{"login":1,"brokerid":1,"close_date":1} ("mg_result_symday"),{"login":1,"brokerid":1,"standardsymbol":1,"close_date":1} ("mg_result_symmonth"),{"login":1,"brokerid":1,"standardsymbol":1} ("mg_result_symmonth"),{"login":1,"brokerid":1,"standardsymbol":1,"close_month":1} ("mg_result_symweekofyear"),{"login":1,"brokerid":1,"standardsymbol":1,"close_weekofyear":1} ("mg_result_week"),{"login":1,"brokerid":1} ("mg_result_weekofyear"),{"login":1,"brokerid":1}
（2）把這個檔案匯入postgres資料庫存為一個欄位，然後寫一個sql,生成postggres語法的建立索引語句：
1 select concat('create index idx_', split_part(translate(idx_name,'(){}:-1"',''),',',1),'_',floor(random()*(25-10)+10),' on ',split_part(translate(idx_name,'(){}:-1"',''),',',1) 2 ,'(',ltrim(SUBSTRING(translate(idx_name,'(){}:-1',''),strpos(translate(idx_name,'(){}:-1',''),','),500),','), 3 ')',';') from devops.mongo_index_ddl；
生成的建立索引為：
1 create index idx_mg_result_all_12 on mg_result_all(""publishfrozentime"",""usertype"",""accountstatus"",""accounttype"",""equity"",""period_trade""); 2 create index idx_mg_result_all_23 on mg_result_all(""rate_ss_profit_balance_close""); 3 create index idx_mg_result_brsymday_21 on mg_result_brsymday(""login"",""brokerid""); 4 create index idx_mg_result_brsymday_12 on mg_result_brsymday(""login"",""brokerid"",""close_date""); 5 create index idx_mg_result_day_11 on mg_result_day(""login"",""brokerid"",""close_date"");" 6 create index idx_mg_result_follall_12 on mg_result_follall(""login"",""brokerid""); 7 create index idx_mg_result_follall_17 on mg_result_follall(""masteraccount"",""masterbrokerid""); 8 create index idx_mg_result_follall_13 on mg_result_follall(""login"",""brokerid"",""usertype"",""masteraccount"",""masterbrokerid"",""masterusertype"");
....
最後，如果有需要可以做成指令碼，只執行一次就好。