POI(excel)匯出優化
阿新 • • 發佈：2019-02-01
寄語：第一次接觸百萬級別匯出，不知道極限，嘗試過百度，經歷過絕望，記憶體溢位，等了一萬年，不出來檔案….
1.要明白 極限：
excel 不同版本的行極限:
excel 2007 及以上：1048576
2003 : 65536 行
不考慮列：因為最少256列夠用
2：什麼會導致記憶體溢位：
1.迴圈量過大
2.建立重複實體過多
3.map 新增多隻
4.實體未釋放
3：怎麼檢測是什麼過大：
jconsole
4: 看過無數篇部落格，感覺敘述都沒用，但其實是業務不對，但其實是能組裝業務的
比如說：
- poi匯出100w的excel需要多久：測試了二十列100w行匯出 一分半鐘就能匯出來。
- 那為什麼在系統裡會如此慢： 列印日誌，查時間，發現mongo 查詢 很慢，而且還易報錯，比如說16mb的bson，這個會讓做大資料很麻煩。但你查詢的過程 想到分出去多少萬去查詢，這個很難，比如說一百萬 你分十次 可以，但分五次比十次要測測哪個快，我本人選擇的是二十萬，因為減少連線，可以快速的拼裝資料，而且，利用計算，使流持久化，等我寫夠一百萬在讓流釋放，如果沒寫夠，讓他一直存在。
- 這樣我每二十萬就手動控制寫出到workbook 裡，加快了以後刷出100萬的迴圈。也從而減少了拼裝集合。
/**
* @param tableID 表id
* @param tableName 表名稱
* @param excel程式碼：
/**
* @param entities 欄位集合，用來拆分表頭，和解析時間型別
* @param results 資料集
* @param excelName 表名
* @param request 請求頭
* @param response 請求頭
* @param isZip 是否是zip檔案
* @param path path是匯出的路徑
* @param fooIf 是用來判別是第幾個excel的
* @param hasNext 區別是否還需要拼裝zip裡的一個excel
* @param workbook 帶有未組合好的excel
* @param zipOutStream 帶有未組合好的壓縮流
* @param countIndex 第幾個檔名字首
* @return
* @author liuyu
* date 2018/8/29
**/
public static HashMap<String, Object> exportExcel(List<FieldEO> entities, List<DBObject> results,
String excelName, HttpServletRequest request,
HttpServletResponse response, boolean isZip, String path,
int fooIf, boolean hasNext, Workbook workbook,
ZipOutputStream zipOutStream, int countIndex) throws IOException {
HashMap<String, Object> returnMap = new HashMap<>();
if (isZip) {
File zipFile = null;
XSSFWorkbook xssfWorkbook = null;
ByteArrayOutputStream outputStream = null;
try {
if (workbook == null) {
xssfWorkbook = new XSSFWorkbook();
workbook = new SXSSFWorkbook(xssfWorkbook);
workbook.createSheet();
}
createSheet(entities, results, workbook, fooIf);
int lastRowNum = workbook.getSheetAt(0).getLastRowNum();
if (!hasNext || lastRowNum >= 1000000) {
zipFile = new File(path + File.separator + excelName);
if (!zipFile.exists()) {
if (!zipFile.getParentFile().exists()) {
boolean mkdirs = zipFile.getParentFile().mkdirs();
}
boolean newFile = zipFile.createNewFile();
}
if (zipOutStream == null) {
zipOutStream = new ZipOutputStream(new FileOutputStream(zipFile));
}
outputStream = new ByteArrayOutputStream();
try {
workbook.write(outputStream);
} catch (IOException e) {
e.printStackTrace();
logger.error(e.getMessage());
} finally {
((SXSSFWorkbook) workbook).dispose();
workbook.close();
}
int count;
ByteArrayInputStream zipTemp = null;
zipTemp = new ByteArrayInputStream(outputStream.toByteArray());
try {
ZipEntry zipEntry = new ZipEntry(countIndex + ".xlsx");
zipOutStream.putNextEntry(zipEntry);
while ((count = zipTemp.read()) != -1) {
zipOutStream.write(count);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
zipOutStream.closeEntry();
zipTemp.close();
outputStream.close();
}
countIndex++;
//先關閉源頭
xssfWorkbook = new XSSFWorkbook();
workbook = new SXSSFWorkbook(xssfWorkbook);
workbook.createSheet();
}
if (hasNext) {
returnMap.put(Constant.EXECEL_EXPORT_WORK_STREAM, workbook);
returnMap.put(Constant.EXECEL_EXPORT_ZIP_STREAM, zipOutStream);
returnMap.put(Constant.EXECEL_EXPORT_ZIP_COUNT, countIndex);
return returnMap;
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (!hasNext && workbook != null) {
workbook.close();
}
if (!hasNext && zipOutStream != null) {
zipOutStream.close();
}
}
} else {
XSSFWorkbook xssfWorkbook = null;
ServletOutputStream outputStream = null;
try {
xssfWorkbook = new XSSFWorkbook();
workbook = new SXSSFWorkbook(xssfWorkbook);
workbook.createSheet();
createSheet(entities, results, workbook, 0);
String rtn = ContentDispositionUtils.handler(request, excelName);
response.setCharacterEncoding("UTF-8");
response.addHeader("Content-disposition", "attachment;" + rtn);
outputStream = response.getOutputStream();
workbook.write(outputStream);
outputStream.flush();
} catch (IOException e) {
e.printStackTrace();
logger.error("exception:" + e);
} finally {
if (outputStream != null) {
outputStream.close();
}
if (workbook != null) {
workbook.close();
}
if (xssfWorkbook != null) {
xssfWorkbook.close();
}
}
}
return null;
}
/**
* @param entities 欄位集合
* @param results 結果集
* @param workbook 工作流
* @param fooIf 用來判定是否需要生成表頭
* @return
* @author liuyu
* date 2018/8/29
**/
private static void createSheet(List<FieldEO> entities, List<DBObject> results, Workbook workbook, int fooIf) {
Sheet sheet;
sheet = workbook.getSheetAt(0);
// style.setShrinkToFit(true);
//設定行內居中
// style.setAlignment(HorizontalAlignment.CENTER);
//使用這個可以優化公式。
// XSSFFormulaEvaluator evaluator = new XSSFFormulaEvaluator((XSSFWorkbook) workbook);
Map<String, String> fieldEOMap = entities.stream().collect(Collectors.toMap(FieldEO::getFieldNameZH,
FieldEO::getFieldName));
Map<String, Integer> fieldNameAndType = entities.stream().collect(Collectors.toMap(FieldEO::getFieldName,
FieldEO::getFieldType));
if (fooIf == 0 || fooIf % 1000000 == 0) {
results.add(0, new BasicDBObject(fieldEOMap));
}
List<String> fieldList = new ArrayList<>(fieldEOMap.values());
int lastRowNum = sheet.getLastRowNum();
int count = lastRowNum;
if (count != 0) {
count = count + 1;
}
for (int i = 0; i < results.size(); i++) {
final DBObject paramsMap = results.get(i);
Row row = sheet.createRow(count + i);
if ((lastRowNum + i) == 0) {
Set entries = paramsMap.toMap().entrySet();
Iterator iterator = entries.iterator();
for (int j = 0; iterator.hasNext(); j++) {
Map.Entry<String, Object> next = (Map.Entry<String, Object>) iterator.next();
//設定單元格的值
Object value = next.getValue();
Object key = next.getKey();
Cell cell = row.createCell(j);
cell.setCellValue(key + "(" + value + ")\t");
}
} else {
for (int z = 0; z < fieldList.size(); z++) {
Cell cell = row.createCell(z);
String key = fieldList.get(z);
Object value = paramsMap.get(key);
if (fieldNameAndType.get(key).equals(FieldTypeEnum.TIME.getId())) {
value = DateUtils.dateToString((Date) value, DateUtils.YYYY_MM_DD_HH_MM_SS_EN);
cell.setCellType(CellType.STRING);
} else if (fieldNameAndType.get(key).equals(FieldTypeEnum.DATE.getId())) {
value = DateUtils.dateToString((Date) value, DateUtils.YYYY_MM_DD_EN);
cell.setCellType(CellType.STRING);
}
if (value == null) {
value = "";
}
cell.setCellValue(value + "");
}
}
}
}