hadoop MapReduce 原始碼 FileInputFormat 核心方法分析
阿新 • • 發佈:2018-12-25
public List<InputSplit> getSplits(JobContext job) throws IOException { StopWatch sw = new StopWatch().start(); //獲取檔案的最小值,如果沒有配置,就按預設值1 long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job)); //獲取最大值,如果沒有配置就按預設值 long的最大值 long maxSize = getMaxSplitSize(job); // generate splits //建立一個儲存切割資訊的陣列 List<InputSplit> splits = new ArrayList<InputSplit>(); //將job中的所有檔案資訊。存在一個list中返回 List<FileStatus> files = listStatus(job); //迴圈遍歷所有檔案 for (FileStatus file: files) { //獲取檔案路徑 Path path = file.getPath(); //獲取檔案長度 long length = file.getLen(); //當檔案長度不等於0 if (length != 0) { BlockLocation[] blkLocations; if (file instanceof LocatedFileStatus) { blkLocations = ((LocatedFileStatus) file).getBlockLocations(); } else { FileSystem fs = path.getFileSystem(job.getConfiguration()); blkLocations = fs.getFileBlockLocations(file, 0, length); } //判斷是否能切割。如果能切割那就切割,有些檔案是不可以切的 if (isSplitable(job, path)) { //獲取塊大小 long blockSize = file.getBlockSize(); //求切割長度。根據maxsize和minsize 和 塊大小來求 // max(minsize,min(maxsize,blocksize)) //如果是按照預設配置的話。就是塊大小。 long splitSize = computeSplitSize(blockSize, minSize, maxSize); //剩餘長度,一開始是lenth long bytesRemaining = length; //如果當前剩餘長度>要切的長度的1.1倍。那麼才切。否則不切了 while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) { //新增切片資訊 int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining); splits.add(makeSplit(path, length-bytesRemaining, splitSize, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); bytesRemaining -= splitSize; } //最後多餘了一個沒有切乾淨,就再新增一個切片資訊 if (bytesRemaining != 0) { int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining); splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); } } //如果不能切片 else { // not splitable //就不切。直接拿整個 splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(), blkLocations[0].getCachedHosts())); } } //當檔案長度等於零 else { //Create empty hosts array for zero length files //建立一個空的切割塊新增進去 splits.add(makeSplit(path, 0, length, new String[0])); } } // Save the number of input files for metrics/loadgen job.getConfiguration().setLong(NUM_INPUT_FILES, files.size()); sw.stop(); if (LOG.isDebugEnabled()) { LOG.debug("Total # of splits generated by getSplits: " + splits.size() + ", TimeTaken: " + sw.now(TimeUnit.MILLISECONDS)); } //返回切片資訊 return splits; }