解決使用poi處理execl表格記憶體溢位問題
阿新 • • 發佈:2018-11-02
在平常的開發中會用到處理表格檔案的功能,poi就是一個非常優秀的處理表格的java框架,但是當表格檔案的資料量過大處理過程就會出現堆記憶體溢位的異常,讓人痛苦了兩天,最後在谷歌的幫助下找了一個解決方法,現在把這個方法分享出來希望可以幫助到大家!!
import org.apache.poi.hssf.eventusermodel.*; import org.apache.poi.hssf.eventusermodel.dummyrecord.LastCellOfRowDummyRecord; import org.apache.poi.hssf.eventusermodel.dummyrecord.MissingCellDummyRecord; import org.apache.poi.hssf.model.HSSFFormulaParser; import org.apache.poi.hssf.record.*; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.text.ParseException; import java.util.ArrayList; import java.util.Date; public class XlsToCsv implements HSSFListener { private int minColumns; private POIFSFileSystem fs; private PrintStream output; private int lastRowNumber; private int lastColumnNumber; public long time_cha; public long getTime_cha() { return time_cha; } public void setTime_cha(long time_cha) { this.time_cha = time_cha; } /** * Should we output the formula, or the value it has? */ private boolean outputFormulaValues = true; /** * For parsing Formulas */ private EventWorkbookBuilder.SheetRecordCollectingListener workbookBuildingListener; private HSSFWorkbook stubWorkbook; // Records we pick up as we process private SSTRecord sstRecord; private FormatTrackingHSSFListener formatListener; /** * So we known which sheet we're on */ private int sheetIndex = -1; private BoundSheetRecord[] orderedBSRs; private ArrayList boundSheetRecords = new ArrayList(); // For handling formulas with string results private int nextRow; private int nextColumn; private boolean outputNextStringRecord; private String d = ""; private final static String OUTPUT_CHARSET = "UTF-8"; // private final String OUTPUT_CHARSET = "GBK"; private int count = 0; public static String deal_time = ""; /** * Creates a new XLS -> CSV converter * * @param fs The POIFSFileSystem to process * @param output The PrintStream to output the CSV to * @param minColumns The minimum number of columns to output, or -1 for no minimum */ public XlsToCsv(POIFSFileSystem fs, PrintStream output, int minColumns) { this.fs = fs; this.output = output; this.minColumns = minColumns; } public XlsToCsv(String inputFilePath, String outputFilePath) throws Exception { fs = new POIFSFileSystem(new FileInputStream(inputFilePath)); output = new PrintStream(outputFilePath, OUTPUT_CHARSET); minColumns = -1; } /** * Creates a new XLS -> CSV converter * * @param filename The file to process * @param minColumns The minimum number of columns to output, or -1 for no minimum * @throws IOException * @throws FileNotFoundException */ public XlsToCsv(String filename, int minColumns) throws IOException, FileNotFoundException { this(new POIFSFileSystem(new FileInputStream(filename)), System.out, minColumns); } public XlsToCsv() { } public XlsToCsv(long time_cha) { this.time_cha = time_cha; } /** * Initiates the processing of the XLS file to CSV */ public void process() throws IOException { MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this); formatListener = new FormatTrackingHSSFListener(listener); HSSFEventFactory factory = new HSSFEventFactory(); HSSFRequest request = new HSSFRequest(); if (outputFormulaValues) { request.addListenerForAllRecords(formatListener); } else { workbookBuildingListener = new EventWorkbookBuilder.SheetRecordCollectingListener(formatListener); request.addListenerForAllRecords(workbookBuildingListener); } factory.processWorkbookEvents(request, fs); } /** * Main HSSFListener method, processes events, and outputs the CSV as the * file is processed. */ public void processRecord(Record record) { int thisRow = -1; int thisColumn = -1; String thisStr = null; String sign = ""; switch (record.getSid()) { case BoundSheetRecord.sid: //boundSheetRecords.add(record); break; case BOFRecord.sid: BOFRecord br = (BOFRecord) record; if (br.getType() == BOFRecord.TYPE_WORKSHEET) { // Create sub workbook if required if (workbookBuildingListener != null && stubWorkbook == null) { stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook(); } // Output the worksheet name // Works by ordering the BSRs by the location of // their BOFRecords, and then knowing that we // process BOFRecords in byte offset order sheetIndex++; if (orderedBSRs == null) { orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords); } } break; case SSTRecord.sid: sstRecord = (SSTRecord) record; break; case BlankRecord.sid: BlankRecord brec = (BlankRecord) record; thisRow = brec.getRow(); thisColumn = brec.getColumn(); // thisStr = ""; break; case BoolErrRecord.sid: BoolErrRecord berec = (BoolErrRecord) record; thisRow = berec.getRow(); thisColumn = berec.getColumn(); // thisStr = ""; break; case FormulaRecord.sid: FormulaRecord frec = (FormulaRecord) record; thisRow = frec.getRow(); thisColumn = frec.getColumn(); if (outputFormulaValues) { if (Double.isNaN(frec.getValue())) { // Formula result is a string // This is stored in the next record outputNextStringRecord = true; nextRow = frec.getRow(); nextColumn = frec.getColumn(); } else { thisStr = formatListener.formatNumberDateCell(frec); } } else { // thisStr = '"' + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"'; thisStr = HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()); } break; case StringRecord.sid: if (outputNextStringRecord) { // String for formula StringRecord srec = (StringRecord) record; thisStr = srec.getString(); thisRow = nextRow; thisColumn = nextColumn; outputNextStringRecord = false; } break; case LabelRecord.sid: LabelRecord lrec = (LabelRecord) record; thisRow = lrec.getRow(); thisColumn = lrec.getColumn(); // thisStr = '"' + lrec.getValue() + '"'; thisStr = lrec.getValue(); break; case LabelSSTRecord.sid: LabelSSTRecord lsrec = (LabelSSTRecord) record; thisRow = lsrec.getRow(); thisColumn = lsrec.getColumn(); if (sstRecord == null) { thisStr = '"' + "(No SST Record, can't identify string)" + '"'; // } else if (sstRecord.getString(lsrec.getSSTIndex()).toString().contains("30/03/2018 - 16h34:18.440")) { } else if (sstRecord.getString(lsrec.getSSTIndex()).toString().contains("/")) { // thisStr = '"' + sstRecord.getString(lsrec.getSSTIndex()).toString() + '"'; thisStr = sstRecord.getString(lsrec.getSSTIndex()).toString(); //用-分割得到時間資料的年月日和小時,分鐘秒的資料 String[] splitData = thisStr.split("-"); //格式化年月日的資料 String[] date = splitData[0].split("/"); //格式化小時分鐘秒的資料 String[] time = splitData[1].split(":"); String[] hourAndMount = time[0].split("h"); String hours = hourAndMount[0] + ":" + hourAndMount[1] + ":" + time[1].split("\\.")[0] + "." + time[1].split("\\.")[1]; d = date[date.length - 1].trim() + "-" + date[1] + "-" + date[0]; deal_time = d + " " + hours; thisStr = ""; } else { //處理時間格式 thisStr = sstRecord.getString(lsrec.getSSTIndex()).toString(); // System.out.println("------->" + thisStr); if (thisStr.contains("h")) { String deal_get_time_split[] = thisStr.split(":"); String deal_hour_minute[] = deal_get_time_split[0].split("h"); String deal_seconds_millis[] = deal_get_time_split[1].split(","); // String deal_seonds_millis_ = deal_seconds_millis[0] + "." + deal_seconds_millis[1]; String deal_date = d + " " + deal_hour_minute[0] + ":" + deal_hour_minute[1] + ":" + deal_seconds_millis[0] + "." + deal_seconds_millis[1]; // thisStr = d + " " + deal_hour_minute_ + ":" + deal_seonds_millis_; try { long t = SystemConstants.stdMSsdf.parse(deal_date).getTime() + this.getTime_cha(); thisStr = SystemConstants.stdMSsdf.format(new Date(t)); System.out.println("s===" + thisStr); } catch (ParseException e) { e.printStackTrace(); } } else if (thisStr.contains("Sign") || thisStr.contains("N2")) { //處理訊號的訊號名。包括訊號產生時間和對應的值 if (thisStr.contains("N2")) { thisStr = thisStr; } else { sign += thisStr + "_time," + thisStr + "_value"; thisStr = sign; } } else { thisStr = ""; } } break; case NoteRecord.sid: NoteRecord nrec = (NoteRecord) record; thisRow = nrec.getRow(); thisColumn = nrec.getColumn(); thisStr = '"' + "(TODO)" + '"'; break; case NumberRecord.sid: NumberRecord numrec = (NumberRecord) record; thisRow = numrec.getRow(); thisColumn = numrec.getColumn(); // Format thisStr = formatListener.formatNumberDateCell(numrec); break; case RKRecord.sid: RKRecord rkrec = (RKRecord) record; thisRow = rkrec.getRow(); thisColumn = rkrec.getColumn(); thisStr = '"' + "(TODO)" + '"'; break; default: break; } // Handle new row if (thisRow != -1 && thisRow != lastRowNumber) { lastColumnNumber = -1; } // Handle missing column if (record instanceof MissingCellDummyRecord) { MissingCellDummyRecord mc = (MissingCellDummyRecord) record; thisRow = mc.getRow(); thisColumn = mc.getColumn(); thisStr = ""; } // If we got something to print out, do so if (thisStr != null) { if (thisStr == "") { } else { // System.out.println("thisColumn :" + thisColumn + " thisStr: " + thisStr); if (thisColumn > 0) { //控制那一列不被寫入到csv檔案中,具體以thisStr來決定 if ((thisColumn - 1) % 4 == 0) { thisStr = ""; } else { output.print(','); } } output.print(thisStr); } } // Update column and row count if (thisRow > -1) { lastRowNumber = thisRow; } if (thisColumn > -1) { lastColumnNumber = thisColumn; } // Handle end of row if (record instanceof LastCellOfRowDummyRecord) { if (((LastCellOfRowDummyRecord) record).getRow() > 7) { // Print out any missing commas if needed if (minColumns > 0) { // Columns are 0 based if (lastColumnNumber == -1) { lastColumnNumber = 0; } for (int i = lastColumnNumber; i < (minColumns); i++) { output.print(','); } } // We're onto a new row lastColumnNumber = -1; // End the row output.println(); } } } public static void main(String[] args) throws Exception { long star = System.currentTimeMillis(); System.out.println("開始讀取的時間是======》:: " + star); String inputPath2 = "/home/java/data/fault_database/20180330-第一次試驗-試驗階段.xls"; String outputPath2 = "/home/java/data/fault_database/data.csv"; // XlsToCsv.time_cha=2000; XlsToCsv xls2csv = new XlsToCsv(inputPath2, outputPath2); xls2csv.setTime_cha(2000); xls2csv.process(); System.out.println(XlsToCsv.deal_time); long end = System.currentTimeMillis(); System.out.println("總過用時==========>: " + (end - star)); } }
import com.monitorjbl.xlsx.StreamingReader; import jxl.WorkbookSettings; import jxl.read.biff.BiffException; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import java.io.*; import java.util.*; /** * Created by 李澤華 on 2017/9/7. */ public class BigExcelReaderPoi { private static OutputStream os; private static OutputStreamWriter osw; private static BufferedWriter bw; private static WorkbookSettings ws = new WorkbookSettings(); private static jxl.Workbook wk; private static FileInputStream in; /** * xls 字尾的execl表格檔案內容轉成csv檔案 * (1)使用流的方法讀取execl表格內容,然後處理表格的內容 * (2)獲得第2行資料,處理得到該檔案生成的時間 * (3)處理第4行資料,獲取訊號名,然後把訊號名分別加上"_time"和"_value" * (4)把處理好的資料按照csv檔案格式要求寫入檔案中,在解析車臺數據用用到生成的csv檔案 * @param inputPath * @param outputPath * @return */ public static List<Map<String,String>> xlsConvertToCsv(String inputPath, String outputPath) { List<Map<String,String>> startTime_outFilePath = new ArrayList<Map<String, String>>(); try { long start_time = System.currentTimeMillis(); System.out.println("當前開始時間是=====>:" + start_time); os = new FileOutputStream(new File(outputPath)); osw = new OutputStreamWriter(os, "UTF8"); bw = new BufferedWriter(osw); //載入excel檔案 ws.setLocale(new Locale("en", "EN")); wk = jxl.Workbook.getWorkbook(new File(inputPath), ws); //從工作簿workbook取得每頁sheets String date_ = ""; boolean flag = true; String deal_time = ""; String sign_date = ""; String sign =""; StringBuffer signBuffer = new StringBuffer(); String startTime = ""; Map<String,String> outFilePath_map = new HashMap<String, String>(); outFilePath_map.put("outFilePath",outputPath); startTime_outFilePath.add(outFilePath_map); for (int sheet = 0; sheet < wk.getNumberOfSheets(); sheet++) { jxl.Sheet s = wk.getSheet(sheet); jxl.Cell[] row = null; //從每頁sheet取得每個區塊cell for (int i = 0; i < s.getRows(); i++) { row = s.getRow(i); //獲取開始時間 如:2018-03-30 if (i == 2){ String rowData = row[1].getContents(); //用-分割得到時間資料的年月日和小時,分鐘秒的資料 String[] splitData = rowData.split("-"); //格式化年月日的資料 String[] date = splitData[0].split("/"); //格式化小時分鐘秒的資料 String[] time = splitData[1].split(":"); String[] hourAndMount = time[0].split("h"); String hours = hourAndMount[0] + ":" + hourAndMount[1] + ":" + time[1].split("\\.")[0] + "." + time[1].split("\\.")[1]; deal_time = date[date.length - 1].trim() + "-" + date[1] + "-" + date[0]; startTime = deal_time + " " + hours; } //處理訊號的名的值處理後的值如: // Sign1_time,Sign1_value,Sign2_time,Sign2_value,Sign3_time,Sign3_value,Sign4_time,Sign4_value,Sign5_time,Sign5_value,Sign6_time,Sign6_value,Sign7_time,Sign7_value, if (i == 4){ for (int m = 0;m<row.length; m++){ if (m % 4 == 0){ // sign += row[m].getContents() + "_time," + row[m].getContents() + "_value,"; signBuffer.append(row[m].getContents() + "_time," + row[m].getContents() + "_value,"); sign = signBuffer.toString(); } } bw.write(sign); bw.newLine(); } if ((i>8)&&(row.length > 0)) { //寫入第一列處理後的資料,並加上, String time = ""; String deal_get_time_split[] = row[0].getContents().split(":"); String deal_hour_minute[] = deal_get_time_split[0].split("h"); String deal_hour_minute_ = deal_hour_minute[0] + ":" + deal_hour_minute[1]; String deal_seconds_millis[] = deal_get_time_split[1].split(","); String deal_seonds_millis_ = deal_seconds_millis[0] + "." + deal_seconds_millis[1]; time = deal_time + " " + deal_hour_minute_ + ":" + deal_seonds_millis_; bw.write(time); bw.write(','); bw.write(row[2].getContents()); bw.write(','); for (int j = 1; j < row.length; j++) { // bw.write(','); //取出時間並處理然後寫入新的檔案中 if (j % 4 ==0){ String ti =""; String deal_get_time_split_[] = row[j].getContents().split(":"); String deal_hour_minute1[] = deal_get_time_split_[0].split("h"); String deal_hour_minute_1 = deal_hour_minute1[0] + ":" + deal_hour_minute1[1]; String deal_seconds_millis1[] = deal_get_time_split_[1].split(","); String deal_seonds_millis_1 = deal_seconds_millis1[0] + "." + deal_seconds_millis1[1]; ti = deal_time + " " + deal_hour_minute_1 + ":" + deal_seonds_millis_1; bw.write(ti); bw.write(','); //獲得訊號的值 bw.write(row[j+2].getContents()); bw.write(','); } } bw.newLine(); } } } Map<String,String> startTime_map = new HashMap<String, String>(); startTime_map.put("startTime", startTime); startTime_outFilePath.add(startTime_map); long end_time = System.currentTimeMillis(); System.out.println("用時=======>: " + (end_time -start_time) + "ms"); bw.flush(); bw.close(); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (BiffException e) { e.printStackTrace(); } return startTime_outFilePath; } /** * .xlsx型別的execl表格資料處理成為csv檔案 * @param inputPath * @param outputPath * @return */ public String xlsxConvertToCsv(String inputPath, String outputPath) { long startTime = System.currentTimeMillis(); System.out.println("程式開始時間:" + (startTime) + "ms"); try { os = new FileOutputStream(new File(outputPath)); osw = new OutputStreamWriter(os, "UTF8"); bw = new BufferedWriter(osw); in = new FileInputStream(inputPath); Workbook wk = StreamingReader.builder() .rowCacheSize(100) //快取到記憶體中的行數,預設是10 .bufferSize(4096) //讀取資源時,快取到記憶體的位元組大小,預設是1024 .open(in); //開啟資源,必須,可以是InputStream或者是File,注意:只能開啟XLSX格式的檔案 Sheet sheet = wk.getSheetAt(0); bw.write(sheet.getSheetName()); bw.newLine(); //控制前幾行不被寫入csv檔案 int count = 0; //控制那些列不被寫入csv檔案 int cell_index = 0; //遍歷所有的行 for (Row row : sheet) { //System.out.println("開始遍歷第" + row.getRowNum() + "行資料:"); count++; System.out.println("當前行數是==============》: " + count); if (count < 8) { continue; } //遍歷所有的列 for (Cell cell : row) { cell_index++; System.out.println("當前的列號是==========》: " + cell.getColumnIndex()); String line_data = ""; // System.out.print(cell.getStringCellValue() + " "); if (cell.getStringCellValue().contains("h")) { String deal_get_time_split[] = cell.getStringCellValue().split(":"); String deal_hour_minute[] = deal_get_time_split[0].split("h"); String deal_hour_minute_ = deal_hour_minute[0] + ":" + deal_hour_minute[1]; // String deal_seconds_millis[] = deal_get_time_split[1].split(","); String deal_seconds_millis[] = deal_get_time_split[1].split("\\."); String deal_seonds_millis_ = deal_seconds_millis[0] + "." + deal_seconds_millis[1]; line_data += deal_hour_minute_ + ":" + deal_seonds_millis_; bw.write(line_data); bw.write(','); } else if ((cell.getColumnIndex() + 1) % 2 == 0) { bw.write(line_data); // bw.write(','); } else { bw.write(cell.getStringCellValue()); bw.write(','); } } bw.newLine(); //System.out.println(" "); } long endTime = System.currentTimeMillis(); System.out.println("程式執行時間:" + (endTime - startTime) + "ms"); //輸出程式執行時間 } catch (FileNotFoundException e) { e.printStackTrace(); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { try { os.close(); osw.close(); bw.close(); in.close(); wk.close(); } catch (IOException e) { e.printStackTrace(); } } return outputPath; } //TEST public static void main(String args[]) { // String inputPath2 = "/home/java/data/fault_database/20180330-第一次試驗-試驗階段.xls"; String inputPath2 = "d:/home/java/data/fault_database/20180330-第一次試驗-試驗階段.xls"; // String outputPath2 = "/home/java/data/fault_database/data.csv"; String outputPath2 = "d:/home/java/data/fault_database/fault_databasedata.csv"; String csv = "/home/java/data/fault_database/lizehua_test.csv"; xlsConvertToCsv(inputPath2, outputPath2); } }