JavaWEB_POI匯出大量資料excel(50萬左右)
POI 對大資料量的匯出是一個難點,這裡博主總結了兩種處理方式:
方式一,使用高版本的POI,具有記憶體自動釋放的特點。
方式二,將資料匯出到多個Excel中,並進行壓縮處理,上傳到伺服器中。
方式一:
POI之前的版本不支援大資料量處理,如果資料過多則經常報OOM錯誤,有時候調整JVM大小效果也不是太好。3.8版本的POI新出來了SXSSFWorkbook,可以支援大資料量的操作,只是SXSSFWorkbook只支援.xlsx格式,不支援.xls格式。
3.8版本的POI對excel的匯出操作,一般只使用HSSFWorkbook以及SXSSFWorkbook,import junit.framework.Assert;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
public static void main(String[] args) throws Throwable {
Workbook wb = new SXSSFWorkbook(100); // keep 100 rows in memory, exceeding rows will be flushed to disk
Sheet sh = wb.createSheet();
for(int rownum = 0; rownum < 100000; rownum++){
Row row = sh.createRow(rownum);
for(int cellnum = 0; cellnum < 10; cellnum++){
Cell cell = row.createCell(cellnum);
String address = new CellReference(cell).formatAsString();
cell.setCellValue(address); }
}
FileOutputStream out = new FileOutputStream("/temp/sxssf.xlsx");
wb.write(out);
out.close();
}
以前還用xml來處理,現在3.8以上就好辦了。
方式二:
說明:我的電腦 2.0CPU 2G記憶體 能夠十秒鐘匯出 20W 條資料 ,12.8M的excel內容壓縮後2.68M
我們知道在POI匯出Excel時,資料量大了,很容易導致記憶體溢位。由於Excel 一個sheet允許的最大行數是65536這時我們想到分sheet進行匯出;但是這種情況也不能解決記憶體溢位的問題。畢竟資料還是一次性在記憶體中進行儲存的。這時我們想是不是可以匯出多個excel呢?下面我就嘗試著按照匯出多個excel
首先:我們要確定資料量有多大,然後確定一個excel匯出多少條資料,這樣就可以確定匯出的Excel的數量,於是我們就可以迴圈的匯出excel並儲存在任意的臨時目錄中。去這樣如果記憶體不夠的話虛擬機器就會去進行回收已經儲存的excel在記憶體中的空間。
假設我們我們已經成功的生成了多個excel,這時我們怎麼把這N個excel文件傳到客戶端呢?其實一個一個的傳也未嘗不可,但是考慮那樣對使用者來說體驗不夠好,再次多個檔案在網路上傳輸也比較慢。我們可以考慮對生成的幾個檔案進行壓縮,然後傳到客戶端。
總結一下
第一、分批次生成excel
第二、壓縮後到客戶端
下面我把我的一個小例項貼上供大家參考
第一、Person.java 普通javabean
Javabean程式碼- package bean;
- /**
- *
- * @author http://javaflex.iteye.com/
- *
- */
- public class Person {
- private Integer id;
- private String name;
- private String address;
- private String tel;
- private Double money=0.0;
- public Double getMoney() {
- return money;
- }
- public void setMoney(Double money) {
- this.money = money;
- }
- public Person(Integer id, String name, String address, String tel,Double money) {
- super();
- this.id = id;
- this.name = name;
- this.address = address;
- this.tel = tel;
- this.money=money;
- }
- public Integer getId() {
- return id;
- }
- public void setId(Integer id) {
- this.id = id;
- }
- public String getName() {
- return name;
- }
- public void setName(String name) {
- this.name = name;
- }
- public String getAddress() {
- return address;
- }
- public void setAddress(String address) {
- this.address = address;
- }
- public String getTel() {
- return tel;
- }
- public void setTel(String tel) {
- this.tel = tel;
- }
- }
第二、PersonService模擬業務邏輯迴圈生成100023個Person物件
模擬業務邏輯程式碼- package service;
- import java.util.ArrayList;
- import java.util.List;
- import bean.Person;
- /**
- *
- * @author http://javaflex.iteye.com/
- *
- */
- public class PersonService {
- public static List getPerson(){
- List<Person> list =new ArrayList<Person>();
- for(int i=0;i<100320;i++){
- list.add(new Person(i,"zhangsan"+i,"北京"+i,"13214587632",123123.12+i));
- }
- return list;
- }
- }
第三、業務處理Servlet
操作servlet程式碼- package servlet;
- import java.io.File;
- import java.io.FileInputStream;
- import java.io.FileOutputStream;
- import java.io.IOException;
- import java.io.OutputStream;
- import java.text.SimpleDateFormat;
- import java.util.ArrayList;
- import java.util.Date;
- import java.util.List;
- import javax.servlet.ServletException;
- import javax.servlet.http.HttpServlet;
- import javax.servlet.http.HttpServletRequest;
- import javax.servlet.http.HttpServletResponse;
- import org.apache.poi.hssf.usermodel.HSSFWorkbook;
- import org.apache.poi.hssf.util.CellRangeAddress;
- import org.apache.poi.ss.usermodel.Cell;
- import org.apache.poi.ss.usermodel.CellStyle;
- import org.apache.poi.ss.usermodel.Row;
- import org.apache.poi.ss.usermodel.Sheet;
- import org.apache.poi.ss.usermodel.Workbook;
- import bean.Person;
- import service.PersonService;
- /**
- *
- * @author http://javaflex.iteye.com/
- *
- */
- public class PersonServlet extends HttpServlet {
- private String fileName;
- public PersonServlet() {
- super();
- }
- public void destroy() {
- super.destroy(); // Just puts "destroy" string in log
- // Put your code here
- }
- public void doGet(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
- // 檔名獲取
- Date date = new Date();
- SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");
- String f = "Person-" + format.format(date);
- this.fileName = f;
- setResponseHeader(response);
- OutputStream out = null;
- try {
- out = response.getOutputStream();
- List<Person> list = PersonService.getPerson();
- toExcel(list,request,10000,f,out);
- } catch (IOException e1) {
- e1.printStackTrace();
- } finally {
- try {
- out.flush();
- out.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- /** 設定響應頭 */
- public void setResponseHeader(HttpServletResponse response) {
- try {
- response.setContentType("application/octet-stream;charset=UTF-8");
- response.setHeader("Content-Disposition", "attachment;filename="
- + java.net.URLEncoder.encode(this.fileName, "UTF-8")
- + ".zip");
- response.addHeader("Pargam", "no-cache");
- response.addHeader("Cache-Control", "no-cache");
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
- public void doPost(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
- doGet(request, response);
- }
- public void init() throws ServletException {
- // Put your code here
- }
- public void toExcel(List<Person> list, HttpServletRequest request,
- int length, String f, OutputStream out) throws IOException {
- List<String> fileNames = new ArrayList();// 用於存放生成的檔名稱s
- File zip = new File(request.getRealPath("/files") + "/" + f + ".zip");// 壓縮檔案
- // 生成excel
- for (int j = 0, n = list.size() / length + 1; j < n; j++) {
- Workbook book = new HSSFWorkbook();
- Sheet sheet = book.createSheet("person");
- double d = 0;// 用來統計
- String file = request.getRealPath("/files") + "/" + f + "-" + j
- + ".xls";
- fileNames.add(file);
- FileOutputStream o = null;
- try {
- o = new FileOutputStream(file);
- // sheet.addMergedRegion(new
- // CellRangeAddress(list.size()+1,0,list.size()+5,6));
- Row row = sheet.createRow(0);
- row.createCell(0).setCellValue("ID");
- row.createCell(1).setCellValue("NAME");
- row.createCell(2).setCellValue("ADDRESS");
- row.createCell(3).setCellValue("TEL");
- row.createCell(4).setCellValue("Money");
- int m = 1;
- for (int i = 1, min = (list.size() - j * length + 1) > (length + 1) ? (length + 1)
- : (list.size() - j * length + 1); i < min; i++) {
- m++;
- Person user = list.get(length * (j) + i - 1);
- Double dd = user.getMoney();
- if (dd == null) {
- dd = 0.0;
- }
- d += dd;
- row = sheet.createRow(i);
- row.createCell(0).setCellValue(user.getId());
- row.createCell(1).setCellValue(user.getName());
- row.createCell(2).setCellValue(user.getAddress());
- row.createCell(3).setCellValue(user.getTel());
- row.createCell(4).setCellValue(dd);
- }
- CellStyle cellStyle2 = book.createCellStyle();
- cellStyle2.setAlignment(CellStyle.ALIGN_CENTER);
- row = sheet.createRow(m);
- Cell cell0 = row.createCell(0);
- cell0.setCellValue("Total");
- cell0.setCellStyle(cellStyle2);
- Cell cell4 = row.createCell(4);
- cell4.setCellValue(d);
- cell4.setCellStyle(cellStyle2);
- sheet.addMergedRegion(new CellRangeAddress(m, m, 0, 3));
- } catch (Exception e) {
- e.printStackTrace();
- }
- try {
- book.write(o);
- } catch (Exception ex) {
- ex.printStackTrace();
- } finally {
- o.flush();
- o.close();
- }
- }
- File srcfile[] = new File[fileNames.size()];
- for (int i = 0, n = fileNames.size(); i < n; i++) {
-