文章目錄

• model
• Action
• Service
• Dao
• Mapper
• Thread類
• 測試結果
• 需要優化的地方

• JDK 1.6
• POI 3.9
• 框架 struts2 spring mybatis

model

public class ConsultGovDataModel extends ModelBase {
    private static final long serialVersionUID =
 
 1L;
    /**
     * 政務諮詢資料主鍵
     */
    private String govdataId;
    /**
     * 專案名稱
     */
    private String projectName;
    /**
     * 專案地點
     */
    private String projectAddress;
    /**
     *行業分類
     */
    private String projectIndustry;
    /**
     * 釋出日期
     */
    private String publishDate;
 

    /**
     * 行業
     */
    private String industry;
    /**
     * 產品
     */
    private String product;
    /**
     * 匯入日期
     */
    private String importDate;
    /**
     * 建立時間開始 檢索條件
     */
    private String start_time;
    /**
     * 建立時間結束 檢索條件
     */
    private String end_time;
	
     Get .
 
... set ...

}

Action

public class ConsultGovDataAction extends ActionBase implements ModelDriven<ConsultGovDataModel> {

    private static final long serialVersionUID = 1L;

    /**
     * 上傳檔案
     */
    private File upload;
    /**
     * 上傳檔名稱
     */
    private String uploadFileName;
    /**
     * 上傳檔案的mimeType型別
     */
    private String uploadContentType;

    ConsultGovDataService consultGovDataService;

    ConsultGovDataModel model = new ConsultGovDataModel();
     
Get set ... 

@Override
    public ConsultGovDataModel getModel() {
      return model;
    }


/**
 * 方法名: importExcle
 * 方法描述: 匯入政務諮詢的excle資料
 * 引數 []
 * 返回型別 java.lang.String
 * @throws
 */
public String importExcle() throws Exception {
    String type = getRequest().getParameter("type");
    String fileEnd="xls";
    //獲取檔名,判斷是xls還是xlsx
    Workbook workbook=null;
    ArrayList<ConsultGovDataModel> list = new ArrayList<ConsultGovDataModel>();
    if (StringUtils.isNotBlank(uploadFileName)) {
        if (uploadFileName.endsWith(fileEnd)) {
            //03版本的xls
            workbook= new HSSFWorkbook(new FileInputStream(upload));
        }else {
            //07版本的xlsx
            workbook= new XSSFWorkbook(new FileInputStream(upload));
        }
        //獲取一共有幾個sheet
        int numberOfSheets = workbook.getNumberOfSheets();
        for (int i=0;i<numberOfSheets;i++) {
            //遍歷迴圈sheet 匯入 獲取每一個sheet
            Sheet sheet = workbook.getSheetAt(i);
            //遍歷sheet中的每一行
            for (Row row : sheet) {
                if (row.getRowNum() == 0) {
                    //跳過每一個sheet第一行的標題行
                    continue;
                }
                //讀取到末尾了或者到空行了,跳過空行. (當讀取到最後一行時,會結束迴圈  )
                if (row.getCell(0) == null || StringUtils.isBlank(row.getCell(0).getStringCellValue())) {
                    // 讀取到末尾了, 儲存當前sheet頁的資料到資料庫
                    consultGovDataService.saveBatch(list);
                    list.removeAll(list);
                    break;
                }
                ConsultGovDataModel govDataModel = new ConsultGovDataModel();
                if (row.getCell(0)!=null) {
                    govDataModel.setProjectName(row.getCell(0).getStringCellValue());
                }
                if (row.getCell(1)!=null) {
                    govDataModel.setProjectAddress(row.getCell(1).getStringCellValue());
                }
                if (row.getCell(2)!=null) {
                    govDataModel.setProjectIndustry(row.getCell(2).getStringCellValue());
                }
                if (row.getCell(3)!=null) {
                    govDataModel.setPublishDate(row.getCell(3).getStringCellValue());
                }
                //轉換匯入的日期到時分秒
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                String format = sdf.format(new Date());
                govDataModel.setImportDate(format);
                if ("product".equals(type)) {
                    //選擇的是產品,那麼就把sheet名,存入產品列中
                    govDataModel.setProduct(sheet.getSheetName());
                } else if ("industry".equals(type)) {
                    //選擇的是行業,那麼就把sheet名, 存入行業列中
                    govDataModel.setIndustry(sheet.getSheetName());
                }
                list.add(govDataModel);
            }
            //一個sheet讀取完畢了, 放入dao層進行插入
            consultGovDataService.saveBatch(list);
            list.removeAll(list);
        }
    }
    return "success";
}

Service

public class ConsultGovDataServiceImpl implements ConsultGovDataService {
    ConsultGovdataDao consultGovdataDao;

    public ConsultGovdataDao getConsultGovdataDao() {
        return consultGovdataDao;
    }

    public void setConsultGovdataDao(ConsultGovdataDao consultGovdataDao) {
        this.consultGovdataDao = consultGovdataDao;
    }

       @Override
    public void saveBatch(ArrayList<ConsultGovDataModel> list) throws InterruptedException{
        //一個執行緒處理200條資料
        int count = 200;
        //資料集合大小
        int listSize = list.size();
        //開啟的執行緒數
        int runSize = (listSize / count) + 1;
        //存放每個執行緒的執行資料
        List<ConsultGovDataModel> newlist = null;

        //建立一個執行緒池，數量和開啟執行緒的數量一樣
        //Executors 的寫法
       // ExecutorService executor = Executors.newFixedThreadPool(runSize);
        
        //ThreadPoolExecutor的寫法
        ThreadPoolExecutor executor = new ThreadPoolExecutor(runSize, runSize, 1,
                TimeUnit. SECONDS, new ArrayBlockingQueue<Runnable>(3),
                new ThreadPoolExecutor.DiscardOldestPolicy());
                
        //建立兩個個計數器
        CountDownLatch begin = new CountDownLatch(1);
        CountDownLatch end = new CountDownLatch(runSize);
        //迴圈建立執行緒
        for (int i = 0; i < runSize; i++) {
            //計算每個執行緒執行的資料
            if ((i + 1) == runSize) {
                int startIndex = (i * count);
                int endIndex = list.size();
                newlist = list.subList(startIndex, endIndex);
            } else {
                int startIndex = (i * count);
                int endIndex = (i + 1) * count;
                newlist = list.subList(startIndex, endIndex);
            }
            //執行緒類
            ImportThread mythead = new ImportThread(newlist, begin, end);
            //這裡執行執行緒的方式是呼叫執行緒池裡的executor.execute(mythead)方法。
            executor.execute(mythead);
        }
        begin.countDown();
        end.await();
        //執行完關閉執行緒池
        executor.shutdown();
        //consultGovdataDao.saveBatch(list);
    }

Dao

public interface ConsultGovdataDao {

    void saveBatch(List<ConsultGovDataModel> list);
}

Mapper

<!--批量插入匯入的資料-->
<insert id="saveBatch" parameterType="java.util.List">
    insert all
    <foreach item="item" collection="list" separator="">
        INTO CONSULT_GOVDATA
        (
        PROJECT_NAME, PROJECT_ADDRESS, PROJECT_INDUSTRY, PUBLISH_DATE, INDUSTRY,
        PRODUCT,IMPORT_DATE
        )
        values
        (
        #{item.projectName,jdbcType=VARCHAR},
        #{item.projectAddress,jdbcType=VARCHAR},
        #{item.projectIndustry,jdbcType=VARCHAR},
        to_date(#{item.publishDate,jdbcType=DATE},'yyyy-mm-dd'),
        #{item.industry,jdbcType=VARCHAR},
        #{item.product,jdbcType=VARCHAR},
        to_date(#{item.importDate,jdbcType=DATE},'yyyy-mm-dd hh24:mi:ss')
        )
    </foreach>
    select * from dual
</insert>

Thread類

public class ImportThread implements Runnable {
    public ImportThread() {
    }

    ConsultGovdataDao consultGovdataDao;

    public ConsultGovdataDao getConsultGovdataDao() {
        return consultGovdataDao;
    }

    public void setConsultGovdataDao(ConsultGovdataDao consultGovdataDao) {
        this.consultGovdataDao = consultGovdataDao;
    }

    private List<ConsultGovDataModel> list;
    private CountDownLatch begin;
    private CountDownLatch end;

    /**
     * 方法名: ImportThread
     * 方法描述: 建立個建構函式初始化 list,和其他用到的引數
     * @throws
     */
    public ImportThread(List<ConsultGovDataModel> list, CountDownLatch begin, CountDownLatch end) {
        this.list = list;
        this.begin = begin;
        this.end = end;
    }

    @Override
    public void run() {
        try {
            //執行完讓執行緒直接進入等待
            ConsultGovdataDao consultGovdataDao = (ConsultGovdataDao) ServiceFactory.getService("consultGovdataDao");
            consultGovdataDao.saveBatch(list);
            begin.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            //這裡要主要了，當一個執行緒執行完 了計數要減一不然這個執行緒會被一直掛起
            //這個方法就是直接把計數器減一的
            end.countDown();
        }
    }
}

測試結果

經測試, 2萬條資料插入, 批量插入200到300一次最佳.
原來要四分鐘, 現在為41秒.

需要優化的地方

建立執行緒池的方式為Executors.newFixedThreadPool
需要改進為
ThreadPoolExecutor 的方式建立執行緒
20181120已經優化