目錄

第一種：直接取分頁資料

/**
  * 處理記憶體分頁
  * @param list
  * @param pageNo
  * @param pageSize
  * @return
  */
 private <T> List<T> getPageConfigList(List<T> list, int pageNo, int pageSize){
     List<T> currentPagelist = null;
     if (list != null && !list.isEmpty()) {
         currentPagelist = new
 
 ArrayList<T>();
         int iEnd = 0;
         int iStart = 0;

         iStart = pageSize * (pageNo - 1);
         iEnd = iStart + pageSize;
         if (iEnd > list.size()) {
             iEnd = list.size();
         }
         for (int i = iStart; i < iEnd; i++) {
             currentPagelist.add(list.get(i));
         }

     }
     return
 
 currentPagelist;
 }

第二種：先分頁，再取分頁資料

com.google.common.collect.List實現

CollectionContentUtils.sortList(sList, new String[] {"rtn_urlpathmethod"}, "desc");
List<List<TipConfig>> partition = Lists.partition(sList, length);
page.setData(partition.get((start+length-1)/length));

擴充套件知識

guava使用Lists.partition,Lists.transform小結
有時候我們會遇到分割List,把list分成幾份，或者把list的元素轉換成另一個型別的元素，使用 guava的Lists.partition,Lists.transform可以幫忙我們更加簡單的實現此功能

import com.google.common.base.Function;
import com.google.common.collect.Lists;

import java.util.ArrayList;
import java.util.List;


public class Main {
    public static void main(String[] args) throws Exception {

        List<Long> list = new ArrayList<>();
        list.add(1L);
        list.add(2L);
        list.add(3L);
        list.add(4L);
        list.add(5L);
        list.add(6L);
        list.add(7L);
        list.add(8L);
        list.add(9L);

        List<List<Long>> originalPageList = Lists.partition(list, 3);
        List<String> pageList = Lists.transform(originalPageList, new Function<List<Long>, String>() {
            @Override
            public String apply(List<Long> list) {
                final StringBuffer pageSkuIds = new StringBuffer();
                for(Long info : list) {
                    pageSkuIds.append("AA_").append(info).append(",");
                }
                return pageSkuIds.toString();
            }
        });
        System.out.println(pageList.toString());
    }
}

第三種：遍歷的同時，進行分頁

一次遍歷，邊分頁邊批量插入資料庫 ；適合海量資料分批處理。

@DataSource(value = "master")
public void recordRanking() {
    logger.error("recordRanking start====================================");
    String distributedLockKey = "";//分散式鎖key
    String rankingSizeKey = "";//某個活動的排行榜大小key
    String batchInsertErrorKey = "";//某個活動批次插入報錯key
    String promotionAndRuleKey = "";//時光機活動規則key
    try {
        // 獲取最近一條需要發放大獎的活動（活動結束即可）
        promotionAndRuleKey = "promotionAndRule_promotionType_" + SnsPromotionType.TIMEMACHINE_GAME.getCode();
        SnsPromotionShareRuleVo spsrVo = (SnsPromotionShareRuleVo) CacheUtil.getObject(promotionAndRuleKey);
        if (spsrVo == null) {
            spsrVo = snsPromotionDAO.selectPrixPromotionAndRuleByType(SnsPromotionType.TIMEMACHINE_GAME.getCode());
            CacheUtil.setObject(promotionAndRuleKey, spsrVo, 30);
            if (spsrVo == null) {
                logger.error("沒有已結束需要發放大獎的時光機活動");
                return;
            }
        }
        Long promotionId = spsrVo.getPromotionId();

        distributedLockKey = String.format("distributedLockKey_recordRanking_promotionId_%d", promotionId);
        rankingSizeKey = String.format("rankingSizeKey_recordRanking_promotionId_%d", promotionId);
        batchInsertErrorKey = String.format("batchInsertErrorKey_recordRanking_promotionId_%d", promotionId);

        // 分散式鎖（保證下面邏輯不會在同一時間重複執行）
        if(!CacheUtil.set(distributedLockKey, distributedLockKey, 60*15)){
            Object[] args = {distributedLockKey, promotionId};
            logger.error("未獲取到分散式鎖,key:{},活動id:{}", args);
            return;
        }

        // 先檢視排行榜表中是否已經全部寫入本次活動的資料
        SnsShareRecordRankingExample example = new SnsShareRecordRankingExample();
        example.createCriteria().andPromotionIdEqualTo(promotionId);
        int count = snsShareRecordRankingDAO.countByExample(example);
        String rankingSizeStr = CacheUtil.get(rankingSizeKey);
        if (StringUtils.isNotBlank(rankingSizeStr) && Integer.valueOf(rankingSizeStr) <= count) {
            logger.error("活動id：" + promotionId + "的排行榜資料已經全部寫入臨時表,排行榜數量：" + count + ",請勿重複操作");
            return;
        }

        // 處理插入失敗的資料
        boolean handleFlag = handleBatchInsertErrorList(batchInsertErrorKey, promotionId);
        if (handleFlag) {
            return;
        }

        // 獲取排行榜
        StopWatch sw1 = new StopWatch();
        sw1.start();
        List<SnsShareRecord> ssrList = snsShareRecordDAO.getListOfAccomplishedAndFilteredByPromotionId(promotionId);
        sw1.stop();
        logger.error("snsShareRecordDAO.getListOfAccomplishedAndFilteredByPromotionId耗時：" + sw1.getTotalTimeMillis() + "ms");
        if (CollectionUtils.isEmpty(ssrList)) {
            logger.error("該時光機活動排行榜為空,活動id：" + promotionId);
            return;
        }
        int rankingSize = ssrList.size();
        // 將排行榜大小放入快取
        CacheUtil.set(rankingSizeKey, String.valueOf(rankingSize));
        logger.error("活動id：" + promotionId + "，排行榜數量：" + rankingSize);

        // 分批次寫入
        StopWatch sw2 = new StopWatch();
        sw2.start();
        // n:計數器,m:頁數,l:模數,maxSize:每批處理數量
        int n = 0, m = 0, size = rankingSize, maxSize = 500, l = size%maxSize;
        int flag = (l == 0) ? 0 : 1;
        size = size/maxSize;
        List<SnsShareRecordRanking> ssrrListTemp = new ArrayList<>(maxSize);
        SnsShareRecordRanking ssrrTemp = null;
        for (int i = 0; i < rankingSize; i++) {
            SnsShareRecord ssr = ssrList.get(i);
            ssrrTemp = new SnsShareRecordRanking();
            ssrrTemp.setShareRecordId(ssr.getId());
            ssrrTemp.setPromotionId(ssr.getPromotionId());
            ssrrTemp.setUserId(ssr.getUserId());
            ssrrTemp.setPrixResultState(ssr.getPrixResultState());
            ssrrTemp.setTotalRegisterDuration(ssr.getTotalRegisterDuration());
            ssrrTemp.setRanking(i+1);
            ssrrListTemp.add(ssrrTemp);

            n++;
            //預處理資料不能整除最大行數時,最後幾行資料的處理,n==l表示最後一部分資料;
            if(size == m && n == l) {
                flag = 0;
            }

            /**
             * case1：大於等於最大行數
             * case2：預處理資料不能整除最大行數時,最後幾行資料的處理,flag=0
             * case3：預處理資料量可以整除最大行數,flag=0
             */
            if(n >= maxSize || flag == 0) {
                try {
                    snsShareRecordRankingDAO.batchInsert(ssrrListTemp);
                    Object[] args = {m, ssrrListTemp.size(), rankingSize};
                    logger.error("snsShareRecordRankingDAO.batchInsert 第{}次寫入成功,本次寫入資料量：{},總資料量：{}", args);
                } catch (Exception e) {
                    Object[] args = {m, ssrrListTemp.size(), rankingSize};
                    logger.error("snsShareRecordRankingDAO.batchInsert 第{}次寫入失敗,本次寫入資料量：{},總資料量：{}", args);
                    logger.error("snsShareRecordRankingDAO.batchInsert has error", e.getMessage());

                    // 記錄該批次資料到快取,後面重新插入
                    String field = String.format("promotionId_%d_page_%d", promotionId, m);
                    Integer rankingMix = ssrrListTemp.get(0).getRanking();
                    Integer rankingMax = ssrrListTemp.get(ssrrListTemp.size()-1).getRanking();
                    String range = String.format("%d_%d", rankingMix, rankingMax);
                    CacheUtil.hset(batchInsertErrorKey, field, range);
                } finally {
                    ssrrListTemp.clear();
                    n = 0;
                    m++;
                }
            }
        }
        sw2.stop();
        logger.error("分批次寫入排行榜資料總耗時：" + sw2.getTotalTimeMillis() + "ms");
    } catch (Exception e) {
        logger.error("recordRanking has error", e.getMessage());
    } finally {
        // 釋放分散式鎖
        CacheUtil.del(distributedLockKey);
        logger.error("recordRanking end====================================");
    }
}

/**
 * 處理批次插入失敗的資料
 * @param batchInsertErrorKey
 * @param promotionId
 * @return
 */
private boolean handleBatchInsertErrorList(String batchInsertErrorKey, Long promotionId) {
    logger.error("handleBatchInsertErrorList start==========batchInsertErrorKey：" + batchInsertErrorKey);
    boolean flag = false;

    StopWatch sw1 = new StopWatch();
    sw1.start();

    Map<String, String> batchInsertErrorMap = CacheUtil.hgetAll(batchInsertErrorKey);
    if (MapUtils.isNotEmpty(batchInsertErrorMap)) {
        // 將批量插入失敗的資料重新插入一次
        for (Map.Entry<String, String> entry : batchInsertErrorMap.entrySet()) {
            String field = entry.getKey();
            String range = entry.getValue();
            String[] rangeArr = StringUtils.split(range, "_");
            Integer rankingMix = Integer.valueOf(StringUtils.replace(rangeArr[0], "\"", ""));
            Integer rankingMax = Integer.valueOf(StringUtils.replace(rangeArr[1], "\"", ""));
            // 獲取排名範圍內的排行榜資料
            List<SnsShareRecord> ssrList = snsShareRecordDAO.getListOfAccomplishedAndFilteredByPromotionIdAndRange(promotionId, rankingMix-1, rankingMax - rankingMix + 1);
            if (CollectionUtils.isEmpty(ssrList)) {
                Object[] args1 = {field};
                logger.error("handleBatchInsertErrorList batchInsertErrorMap中field:{}沒有獲取到該範圍排行榜資料,下面將該field從快取刪除", args1);
                CacheUtil.hdel(batchInsertErrorKey, field);
                continue;
            }

            int size = ssrList.size();
            Object[] args2 = {field, size};
            logger.error("handleBatchInsertErrorList batchInsertErrorMap中field:{}對應的資料量:{},下面將進行批量插入", args2);

            List<SnsShareRecordRanking> ssrrListTemp = new ArrayList<>(size);
            SnsShareRecordRanking ssrrTemp = null;
            int ranking = rankingMix.intValue();//最小排名
            for (int i = 0; i < size; i++) {
                SnsShareRecord ssr = ssrList.get(i);
                ssrrTemp = new SnsShareRecordRanking();
                ssrrTemp.setShareRecordId(ssr.getId());
                ssrrTemp.setPromotionId(ssr.getPromotionId());
                ssrrTemp.setUserId(ssr.getUserId());
                ssrrTemp.setPrixResultState(ssr.getPrixResultState());
                ssrrTemp.setTotalRegisterDuration(ssr.getTotalRegisterDuration());
                ssrrTemp.setRanking(ranking + i);
                ssrrListTemp.add(ssrrTemp);
            }
            // 重新執行批量插入
            try {
                snsShareRecordRankingDAO.batchInsert(ssrrListTemp);
                // 插入成功之後刪除該批次失敗標記
                CacheUtil.hdel(batchInsertErrorKey, field);
                Object[] args3 = {size};
                logger.error("handleBatchInsertErrorList.batchInsert 寫入成功,本次寫入資料量：{}", args3);
            } catch (Exception e) {
                Object[] args4 = {size};
                logger.error("handleBatchInsertErrorList.batchInsert 寫入失敗,本次寫入資料量：{}", args4);
                logger.error("handleBatchInsertErrorList.batchInsert has error", e.getMessage());
            } finally {
                ssrrListTemp.clear();
            }
        }
        flag = true;
    }

    sw1.stop();
    logger.error("handleBatchInsertErrorList end==========batchInsertErrorKey：" + batchInsertErrorKey + ",flag is " + flag + ",耗時：" + sw1.getTotalTimeMillis() + "ms");
    return flag;
}