百萬資料excel匯出打包下載
阿新 • • 發佈:2019-01-11
前端框架easyUI,後端java,使用poi匯出資料暫存到伺服器,然後打包下載。
用於解決大資料一次性匯出到一個excel記憶體溢位問題和請求超時504問題
1,前端請求js
$$.openProcessingDialog(); //開啟等待動畫
$.ajax({
type: "POST",dataType: "JSON",
url:exportUrl,
data: {search_condition:JSON.stringify(params)},
success: function(data){
$$.closeProcessingDialog();//關閉等待動畫
if(data.errorCode==0){
//執行下載
$(formSearchTemplate).form('submit', {
url : "/kywdop/actions/tool/download.do?actionId=tool_downloadFile",
onSubmit : function(param) {
param.FileName = data.msg;
}
})
}else {
alert(data.msg);
}
}
});
2,打包請求
與業務相關操作,資料是模擬的假資料,可以通過改變迴圈變數測試大資料的匯出\
區分資料量匯出不同格式資料,\
如果totnum<100w條,匯出一個excel檔案\
如果totnum>100w條,每10w條匯出到一個excel,打成壓縮包下載
@RequestMapping(value="/exportShiperrdtl")
public CIPResponseMsg testExport(CIPReqParameter parameter,HttpServletRequest request,HttpServletResponse response){
CIPResponseMsg res = new CIPResponseMsg();
//模擬資料總數
int totnum = 2200000;
//每個excel最大行數
int maxrow = 1000000;
long start = System.currentTimeMillis();
//頁數,每頁一個單獨的excel檔案
int page = totnum%maxrow==0?totnum/maxrow:totnum/maxrow+1;
/** 1.建立臨時資料夾 */
//String rootPath = request.getSession().getServletContext().getRealPath("/");
String rootPath = "D:\\u02\\real_file\\kywdop\\";
if(page>1){
File temDir = new File(rootPath + UUID.randomUUID().toString().replaceAll("-", ""));
if(!temDir.exists()){
temDir.mkdirs();
}
String filepath = temDir.getPath();
/** 2.生成需要下載的檔案,存放在臨時資料夾內 */
List<File> srcfile = new ArrayList<File>();
//填充資料行
for(int i=1;i<=page;i++){
SXSSFWorkbook wb = new SXSSFWorkbook();
//新建sheet頁
Sheet sheet = wb.createSheet();
//設定表頭
Row row = sheet.createRow(0);
CellStyle style = wb.createCellStyle();
style.setAlignment(CellStyle.ALIGN_CENTER);
Cell cell = row.createCell((short) 0);
cell.setCellValue("姓名");
cell.setCellStyle(style);
cell = row.createCell((short) 1);
cell.setCellValue("單位");
cell.setCellStyle(style);
cell = row.createCell((short) 2);
cell.setCellValue("科室");
cell.setCellStyle(style);
cell = row.createCell((short) 3);
cell.setCellValue("註冊日期");
cell.setCellStyle(style);
//模擬資料
for (int p = 1; p< maxrow; p++) {
row = sheet.createRow(p);
row.createCell((short) 0).setCellValue("大名"+p);
row.createCell((short) 1).setCellValue("單位"+p);
row.createCell((short) 2).setCellValue("科室"+p);
row.createCell((short) 3).setCellValue((new Date()).toString());
}
String fileName = "\\使用者資訊" + i+".xls";
FileOutputStream fos = null;
try {
fos = new FileOutputStream(filepath+fileName);
wb.write(fos);
fos.flush();
fos.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
res.msg="伺服器錯誤";
res.errorCode=1;
return res;
} catch (IOException e) {
e.printStackTrace();
res.msg="伺服器錯誤";
res.errorCode=1;
return res;
}
srcfile.add(new File(filepath+fileName));
}
/** 3.呼叫工具類,生成zip壓縮包 */
String zipfilepath = rootPath+"excel.zip"; //壓縮包檔案路徑
try {
FileOutputStream fos2 = new FileOutputStream(new File(zipfilepath));
ZipUtils.toZip(srcfile, fos2);
} catch (IOException e) {
e.printStackTrace();
}
/** 4.刪除臨時檔案和資料夾 */
File[] listFiles = temDir.listFiles();
for (int i = 0; i < listFiles.length; i++) {
listFiles[i].delete();
}
temDir.delete();
res.msg=zipfilepath;
}else{
SXSSFWorkbook wb = new SXSSFWorkbook();
//新建sheet頁
Sheet sheet = wb.createSheet();
//設定表頭
Row row = sheet.createRow(0);
CellStyle style = wb.createCellStyle();
style.setAlignment(CellStyle.ALIGN_CENTER);
Cell cell = row.createCell((short) 0);
cell.setCellValue("姓名");
cell.setCellStyle(style);
cell = row.createCell((short) 1);
cell.setCellValue("單位");
cell.setCellStyle(style);
cell = row.createCell((short) 2);
cell.setCellValue("科室");
cell.setCellStyle(style);
cell = row.createCell((short) 3);
cell.setCellValue("註冊日期");
cell.setCellStyle(style);
//模擬資料
for (int p = 1; p< maxrow; p++) {
row = sheet.createRow(p);
row.createCell((short) 0).setCellValue("大名"+p);
row.createCell((short) 1).setCellValue("單位"+p);
row.createCell((short) 2).setCellValue("科室"+p);
row.createCell((short) 3).setCellValue((new Date()).toString());
}
String fileName = "使用者資訊.xls";
FileOutputStream fos = null;
try {
fos = new FileOutputStream(rootPath+fileName);
wb.write(fos);
fos.flush();
fos.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
res.msg="伺服器錯誤";
res.errorCode=1;
return res;
} catch (IOException e) {
e.printStackTrace();
res.msg="伺服器錯誤";
res.errorCode=1;
return res;
}
res.msg=rootPath+fileName;
}
long end = System.currentTimeMillis();
log.info("匯出耗時:"+(end-start)+" ms");
res.errorCode=0;
return res;
}
百萬資料每個excel匯出多少條可以根據自己業務資料除錯,每個excel匯出越少,分的excel個數越多。
以上程式碼我替換為真實資料後的測試
效能測試
70w資料 每頁10w –> 174977ms
70w資料 每頁20w –> 154977ms
70w資料 每頁30w –> 163430ms
70w資料 每頁40w –> 141162ms
70w資料 每頁50w –> 180974ms
3,下載通用controller
通用的檔案下載只需要傳入伺服器的檔案全路徑即可下載,不區分檔案是壓縮包還是excel
/**
* 檔案下載
* @param request
* @param response
* @throws IOException
*/
@RequestMapping(value="/download")
public void templateDownload(HttpServletRequest request, HttpServletResponse response) throws IOException{
String path = request.getParameter("FileName");
File file = new File(path);
String fileName = StringUtils.substringAfterLast(path,"\\");
if (request.getHeader("User-Agent").toUpperCase().indexOf("MSIE") > 0) { //IE瀏覽器
fileName = URLEncoder.encode(fileName, "UTF-8");
} else { //非IE瀏覽器
fileName = new String(fileName.getBytes("UTF-8"), "ISO8859-1");
}
response.setCharacterEncoding("utf-8");
response.setHeader("Content-Disposition", "attachment;fileName="+fileName);
//輸入流寫入輸出流
InputStream inputStream = new FileInputStream(file);
OutputStream ouputStream = response.getOutputStream();
byte b[] = new byte[1024];
int n ;
//迴圈讀取 !=-1讀取完畢
while((n = inputStream.read(b)) != -1){
//寫入到輸出流 從0讀取到n
ouputStream.write(b,0,n);
}
//關閉流、釋放資源
ouputStream.close();
inputStream.close();
}