阿里雲大檔案分片上傳
阿新 • • 發佈：2020-07-29
1:新建阿里雲工具類
package com.bamboo.water_chivalry.project.file.controller; import com.aliyun.oss.OSSClient; import com.aliyun.oss.model.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.io.InputStream;import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; /** * @PROJECT_NAME: water_chivalry * @AUTHOR: Hanson-Hsc * @DATE: 2020-07-15 11:16 * @DESCRIPTION: * @VERSION: */ public class AliyunOSSUpload implements Runnable{ private MultipartFile localFile;private long startPos; private long partSize; private int partNumber; private String uploadId; private static String key; private static String bucketName; // 新建一個List儲存每個分塊上傳後的ETag和PartNumber protected static List<PartETag> partETags = Collections.synchronizedList(newArrayList<PartETag>()); private static Logger logger = LoggerFactory.getLogger(FileUploader.class); private static OSSClient client = null; /** * 建立構造方法 * * @param localFile * 要上傳的檔案 * @param startPos * 每個檔案塊的開始 * @param partSize * @param partNumber * @param uploadId * 作為塊的標識 * @param key * 上傳到OSS後的檔名 */ public AliyunOSSUpload(MultipartFile localFile, long startPos, long partSize, int partNumber, String uploadId, String key , String bucketName) { this.localFile = localFile; this.startPos = startPos; this.partSize = partSize; this.partNumber = partNumber; this.uploadId = uploadId; AliyunOSSUpload.key = key; AliyunOSSUpload.bucketName = bucketName; } /** * 分塊上傳核心方法(將檔案分成按照每個5M分成N個塊，並加入到一個list集合中) */ @Override public void run() { InputStream instream = null; try { // 獲取檔案流 instream = localFile.getInputStream(); // 跳到每個分塊的開頭 instream.skip(this.startPos); // 建立UploadPartRequest，上傳分塊 UploadPartRequest uploadPartRequest = new UploadPartRequest(); uploadPartRequest.setBucketName(bucketName); uploadPartRequest.setKey(key); uploadPartRequest.setUploadId(this.uploadId); uploadPartRequest.setInputStream(instream); uploadPartRequest.setPartSize(this.partSize); uploadPartRequest.setPartNumber(this.partNumber); UploadPartResult uploadPartResult = FileUploader.client.uploadPart(uploadPartRequest); logger.info("Part#" + this.partNumber + " done\n"); synchronized (partETags) { // 將返回的PartETag儲存到List中。 partETags.add(uploadPartResult.getPartETag()); } } catch (Exception e) { e.printStackTrace(); } finally { if (instream != null) { try { // 關閉檔案流 instream.close(); } catch (IOException e) { e.printStackTrace(); } } } } /** * 初始化分塊上傳事件並生成uploadID，用來作為區分分塊上傳事件的唯一標識 * * @return */ protected static String claimUploadId(String bucketName, String key) { InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key); InitiateMultipartUploadResult result = FileUploader.client.initiateMultipartUpload(request); logger.info(result.getUploadId()); return result.getUploadId(); } /** * 將檔案分塊進行升序排序並執行檔案上傳。 * * @param uploadId */ protected static void completeMultipartUpload(String uploadId) { // 將檔案分塊按照升序排序 Collections.sort(partETags, new Comparator<PartETag>() { @Override public int compare(PartETag p1, PartETag p2) { return p1.getPartNumber() - p2.getPartNumber(); } }); CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags); // 完成分塊上傳 FileUploader.client.completeMultipartUpload(completeMultipartUploadRequest); } /** * 列出檔案所有分塊的清單 * * @param uploadId */ protected static void listAllParts(String uploadId) { ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, key, uploadId); // 獲取上傳的所有分塊資訊 PartListing partListing = FileUploader.client.listParts(listPartsRequest); // 獲取分塊的大小 int partCount = partListing.getParts().size(); // 遍歷所有分塊 for (int i = 0; i < partCount; i++) { PartSummary partSummary = partListing.getParts().get(i); logger.info("分塊編號 " + partSummary.getPartNumber() + ", ETag=" + partSummary.getETag()); } } }
2:新建檔案上傳介面,此處只需要controller
package com.bamboo.water_chivalry.project.file.controller; import com.alibaba.fastjson.JSONObject; import com.aliyun.oss.OSSClient; import com.bamboo.water_chivalry.common.enums.ResultEnum; import com.bamboo.water_chivalry.common.exception.GlobalException; import com.bamboo.water_chivalry.common.utils.JudgeVIFormat; import com.bamboo.water_chivalry.common.utils.OSSUtil; import com.bamboo.water_chivalry.common.utils.ResultVoUtil; import com.bamboo.water_chivalry.common.vo.other.ResultVo; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.apache.shiro.authz.annotation.Logical; import org.apache.shiro.authz.annotation.RequiresRoles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static com.bamboo.water_chivalry.common.config.FileConfig.*; import static com.bamboo.water_chivalry.common.constant.FileConstant.FILE_UPLOAD_SIZE; import static com.bamboo.water_chivalry.common.enums.ResultEnum.IMAGE_FILES_CANNOT_BE_UPLOADED; import static com.bamboo.water_chivalry.common.enums.ResultEnum.VIDEO_UPLOAD_ERROR; /** * @PROJECT_NAME: water_chivalry * @AUTHOR: Hanson-Hsc * @DATE: 2020-07-15 11:18 * @DESCRIPTION: * @VERSION: */ @RestController @RequestMapping("/file") @Api(tags = "大檔案上傳介面(多執行緒分片上傳)") public class FileUploader { protected static OSSClient client = null; private static Logger logger = LoggerFactory.getLogger(FileUploader.class); @PostMapping("/thread") @ApiOperation(value = "視訊分片上傳")public static ResultVo fileUpload(@RequestParam("file") MultipartFile file) { // 建立一個可重用固定執行緒數的執行緒池。若同一時間執行緒數大於10，則多餘執行緒會放入佇列中依次執行 ExecutorService executorService = Executors.newFixedThreadPool(20); // 獲取上傳檔案的名稱，作為在OSS上的檔名 String key = file.getOriginalFilename(); String newFileName = UUID.randomUUID() + key.substring(key.lastIndexOf(".")); // 建立OSSClient例項 client = new OSSClient("oss-cn-shenzhen.aliyuncs.com", "你的accessKeyId", "你的access金鑰");
try {
String uploadId = AliyunOSSUpload.claimUploadId(VIDEO_BUCKET_NAME, newFileName);
// 設定每塊為 5M(除最後一個分塊以外，其他的分塊大小都要大於5MB)
final long partSize = 5 * 1024 * 1024L;
//final long partSize = 1024 * 1024L;
// 計算分塊數目
long fileLength = file.getSize();
int partCount = (int) (fileLength / partSize);
if (fileLength % partSize != 0) {
partCount++;
}
// 分塊 號碼的範圍是1~10000。如果超出這個範圍，OSS將返回InvalidArgument的錯誤碼。
if (partCount > BLOCK_SCOPE) {
throw new RuntimeException("檔案過大(分塊大小不能超過10000)");
} else {
logger.info("一共分了 " + partCount + " 塊");
}
/**
* 將分好的檔案塊加入到list集合中
*/
for (int i = 0; i < partCount; i++) {
long startPos = i * partSize;
long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize;
// 執行緒執行。將分好的檔案塊加入到list集合中
executorService.execute(new AliyunOSSUpload(file, startPos, curPartSize, i + 1, uploadId, newFileName, VIDEO_BUCKET_NAME));
}
/**
* 等待所有分片完畢
*/
// 關閉執行緒池（執行緒池不馬上關閉），執行以前提交的任務，但不接受新任務。
executorService.shutdown();
// 如果關閉後所有任務都已完成，則返回 true。
while (!executorService.isTerminated()) {
try {
// 用於等待子執行緒結束，再繼續執行下面的程式碼
executorService.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
* partETags(上傳塊的ETag與塊編號（PartNumber）的組合) 如果校驗與之前計算的分塊大小不同，則丟擲異常
*/
System.out.println(AliyunOSSUpload.partETags.size() + " ----- " + partCount);
if (AliyunOSSUpload.partETags.size() != partCount) {
throw new IllegalStateException("OSS分塊大小與檔案所計算的分塊大小不一致");
} else {
logger.info("將要上傳的檔名 " + key + "\n");
}
/*
* 列出檔案所有的分塊清單並列印到日誌中，該方法僅僅作為輸出使用
*/
AliyunOSSUpload.listAllParts(uploadId);
/*
* 完成分塊上傳
*/
AliyunOSSUpload.completeMultipartUpload(uploadId);
JSONObject result = new JSONObject();
result.put("url", HTTPS + VIDEO_BUCKET_NAME + "." + END_POINT + "/" + client.getObject(VIDEO_BUCKET_NAME, newFileName).getKey());
// 返回上傳檔案的URL地址
return ResultVoUtil.success(result);
} catch (Exception e) {
logger.error(VIDEO_UPLOAD_ERROR.getMsg(), e);
return ResultVoUtil.error(ResultEnum.VIDEO_UPLOAD_ERROR);
} finally {
AliyunOSSUpload.partETags.clear();
if (client != null) {
client.shutdown();
}
}
}
}