java http大檔案斷點續傳上傳
阿新 • • 發佈：2019-01-10
1，專案調研
因為需要研究下斷點上傳的問題。找了很久終於找到一個比較好的專案。
在GoogleCode上面，程式碼弄下來超級不方便，還是配置hosts才好，把程式碼重新上傳到了github上面。
效果：
上傳中，顯示進度，時間，百分比。
點選【Pause】暫停，點選【Resume】繼續。
2，程式碼分析
原始專案：
這個專案最後更新的時間是 2012 年，專案進行了封裝使用最簡單的方法實現了http的斷點上傳。
因為html5 裡面有讀取檔案分割檔案的類庫，所以才可以支援斷點上傳，所以這個只能在html5 支援的瀏覽器上面展示。
同時，在js 和 java 同時使用 cr32 進行檔案塊的校驗，保證資料上傳正確。
程式碼在使用了最新的servlet 3.0 的api，使用了非同步執行，監聽等方法。
上傳類UploadServlet
@Component("javaLargeFileUploaderServlet") @WebServlet(name = "javaLargeFileUploaderServlet", urlPatterns = { "/javaLargeFileUploaderServlet" }) public class UploadServlet extends HttpRequestHandlerServlet implements HttpRequestHandler { private static final Logger log = LoggerFactory.getLogger(UploadServlet.class); @Autowired UploadProcessor uploadProcessor; @Autowired FileUploaderHelper fileUploaderHelper; @Autowired ExceptionCodeMappingHelper exceptionCodeMappingHelper; @Autowired Authorizer authorizer; @Autowired StaticStateIdentifierManager staticStateIdentifierManager; @Override public void handleRequest(HttpServletRequest request, HttpServletResponse response) throws IOException { log.trace("Handling request"); Serializable jsonObject = null; try { // extract the action from the request UploadServletAction actionByParameterName = UploadServletAction.valueOf(fileUploaderHelper.getParameterValue(request, UploadServletParameter.action)); // check authorization checkAuthorization(request, actionByParameterName); // then process the asked action jsonObject = processAction(actionByParameterName, request); // if something has to be written to the response if (jsonObject != null) { fileUploaderHelper.writeToResponse(jsonObject, response); } } // If exception, write it catch (Exception e) { exceptionCodeMappingHelper.processException(e, response); } } private void checkAuthorization(HttpServletRequest request, UploadServletAction actionByParameterName) throws MissingParameterException, AuthorizationException { // check authorization // if its not get progress (because we do not really care about authorization for get // progress and it uses an array of file ids) if (!actionByParameterName.equals(UploadServletAction.getProgress)) { // extract uuid final String fileIdFieldValue = fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId, false); // if this is init, the identifier is the one in parameter UUID clientOrJobId; String parameter = fileUploaderHelper.getParameterValue(request, UploadServletParameter.clientId, false); if (actionByParameterName.equals(UploadServletAction.getConfig) && parameter != null) { clientOrJobId = UUID.fromString(parameter); } // if not, get it from manager else { clientOrJobId = staticStateIdentifierManager.getIdentifier(); } // call authorizer authorizer.getAuthorization( request, actionByParameterName, clientOrJobId, fileIdFieldValue != null ? getFileIdsFromString(fileIdFieldValue).toArray(new UUID[] {}) : null); } } private Serializable processAction(UploadServletAction actionByParameterName, HttpServletRequest request) throws Exception { log.debug("Processing action " + actionByParameterName.name()); Serializable returnObject = null; switch (actionByParameterName) { case getConfig: String parameterValue = fileUploaderHelper.getParameterValue(request, UploadServletParameter.clientId, false); returnObject = uploadProcessor.getConfig( parameterValue != null ? UUID.fromString(parameterValue) : null); break; case verifyCrcOfUncheckedPart: returnObject = verifyCrcOfUncheckedPart(request); break; case prepareUpload: returnObject = prepareUpload(request); break; case clearFile: uploadProcessor.clearFile(UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId))); break; case clearAll: uploadProcessor.clearAll(); break; case pauseFile: List<UUID> uuids = getFileIdsFromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId)); uploadProcessor.pauseFile(uuids); break; case resumeFile: returnObject = uploadProcessor.resumeFile(UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId))); break; case setRate: uploadProcessor.setUploadRate(UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId)), Long.valueOf(fileUploaderHelper.getParameterValue(request, UploadServletParameter.rate))); break; case getProgress: returnObject = getProgress(request); break; } return returnObject; } List<UUID> getFileIdsFromString(String fileIds) { String[] splittedFileIds = fileIds.split(","); List<UUID> uuids = Lists.newArrayList(); for (int i = 0; i < splittedFileIds.length; i++) { uuids.add(UUID.fromString(splittedFileIds[i])); } return uuids; } private Serializable getProgress(HttpServletRequest request) throws MissingParameterException { Serializable returnObject; String[] ids = new Gson() .fromJson(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId), String[].class); Collection<UUID> uuids = Collections2.transform(Arrays.asList(ids), new Function<String, UUID>() { @Override public UUID apply(String input) { return UUID.fromString(input); } }); returnObject = Maps.newHashMap(); for (UUID fileId : uuids) { try { ProgressJson progress = uploadProcessor.getProgress(fileId); ((HashMap<String, ProgressJson>) returnObject).put(fileId.toString(), progress); } catch (FileNotFoundException e) { log.debug("No progress will be retrieved for " + fileId + " because " + e.getMessage()); } } return returnObject; } private Serializable prepareUpload(HttpServletRequest request) throws MissingParameterException, IOException { // extract file information PrepareUploadJson[] fromJson = new Gson() .fromJson(fileUploaderHelper.getParameterValue(request, UploadServletParameter.newFiles), PrepareUploadJson[].class); // prepare them final HashMap<String, UUID> prepareUpload = uploadProcessor.prepareUpload(fromJson); // return them return Maps.newHashMap(Maps.transformValues(prepareUpload, new Function<UUID, String>() { public String apply(UUID input) { return input.toString(); }; })); } private Boolean verifyCrcOfUncheckedPart(HttpServletRequest request) throws IOException, MissingParameterException, FileCorruptedException, FileStillProcessingException { UUID fileId = UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId)); try { uploadProcessor.verifyCrcOfUncheckedPart(fileId, fileUploaderHelper.getParameterValue(request, UploadServletParameter.crc)); } catch (InvalidCrcException e) { // no need to log this exception, a fallback behaviour is defined in the // throwing method. // but we need to return something! return Boolean.FALSE; } return Boolean.TRUE; } }
非同步上傳UploadServletAsync
@Component("javaLargeFileUploaderAsyncServlet") @WebServlet(name = "javaLargeFileUploaderAsyncServlet", urlPatterns = { "/javaLargeFileUploaderAsyncServlet" }, asyncSupported = true) public class UploadServletAsync extends HttpRequestHandlerServlet implements HttpRequestHandler { private static final Logger log = LoggerFactory.getLogger(UploadServletAsync.class); @Autowired ExceptionCodeMappingHelper exceptionCodeMappingHelper; @Autowired UploadServletAsyncProcessor uploadServletAsyncProcessor; @Autowired StaticStateIdentifierManager staticStateIdentifierManager; @Autowired StaticStateManager<StaticStatePersistedOnFileSystemEntity> staticStateManager; @Autowired FileUploaderHelper fileUploaderHelper; @Autowired Authorizer authorizer; /** * Maximum time that a streaming request can take.<br> */ private long taskTimeOut = DateUtils.MILLIS_PER_HOUR; @Override public void handleRequest(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException { // process the request try { //check if uploads are allowed if (!uploadServletAsyncProcessor.isEnabled()) { throw new UploadIsCurrentlyDisabled(); } // extract stuff from request final FileUploadConfiguration process = fileUploaderHelper.extractFileUploadConfiguration(request); log.debug("received upload request with config: "+process); // verify authorization final UUID clientId = staticStateIdentifierManager.getIdentifier(); authorizer.getAuthorization(request, UploadServletAction.upload, clientId, process.getFileId()); //check if that file is not paused if (uploadServletAsyncProcessor.isFilePaused(process.getFileId())) { log.debug("file "+process.getFileId()+" is paused, ignoring async request."); return; } // get the model StaticFileState fileState = staticStateManager.getEntityIfPresent().getFileStates().get(process.getFileId()); if (fileState == null) { throw new FileNotFoundException("File with id " + process.getFileId() + " not found"); } // process the request asynchronously final AsyncContext asyncContext = request.startAsync(); asyncContext.setTimeout(taskTimeOut); // add a listener to clear bucket and close inputstream when process is complete or // with // error asyncContext.addListener(new UploadServletAsyncListenerAdapter(process.getFileId()) { @Override void clean() { log.debug("request " + request + " completed."); // we do not need to clear the inputstream here. // and tell processor to clean its shit! uploadServletAsyncProcessor.clean(clientId, process.getFileId()); } }); // then process uploadServletAsyncProcessor.process(fileState, process.getFileId(), process.getCrc(), process.getInputStream(), new WriteChunkCompletionListener() { @Override public void success() { asyncContext.complete(); } @Override public void error(Exception exception) { // handles a stream ended unexpectedly , it just means the user has // stopped the // stream if (exception.getMessage() != null) { if (exception.getMessage().equals("Stream ended unexpectedly")) { log.warn("User has stopped streaming for file " + process.getFileId()); } else if (exception.getMessage().equals("User cancellation")) { log.warn("User has cancelled streaming for file id " + process.getFileId()); // do nothing } else { exceptionCodeMappingHelper.processException(exception, response); } } else { exceptionCodeMappingHelper.processException(exception, response); } asyncContext.complete(); } }); } catch (Exception e) { exceptionCodeMappingHelper.processException(e, response); } } }
3，請求流程圖：
主要思路就是將檔案切分，然後分塊上傳。
