1. 程式人生 > >hadoop MapReduce 原始碼 FileInputFormat 核心方法分析

hadoop MapReduce 原始碼 FileInputFormat 核心方法分析

public List<InputSplit> getSplits(JobContext job) throws IOException {
    StopWatch sw = new StopWatch().start();
     //獲取檔案的最小值,如果沒有配置,就按預設值1
    long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
     //獲取最大值,如果沒有配置就按預設值 long的最大值
    long maxSize = getMaxSplitSize(job);

    // generate splits
     //建立一個儲存切割資訊的陣列
    List<InputSplit> splits = new ArrayList<InputSplit>();
    //將job中的所有檔案資訊。存在一個list中返回
    List<FileStatus> files = listStatus(job);
     //迴圈遍歷所有檔案
    for (FileStatus file: files) {
      //獲取檔案路徑
      Path path = file.getPath();
      //獲取檔案長度
      long length = file.getLen();
      //當檔案長度不等於0
      if (length != 0) {
        BlockLocation[] blkLocations;
        if (file instanceof LocatedFileStatus) {
          blkLocations = ((LocatedFileStatus) file).getBlockLocations();
        } else {
          FileSystem fs = path.getFileSystem(job.getConfiguration());
          blkLocations = fs.getFileBlockLocations(file, 0, length);
        }
         //判斷是否能切割。如果能切割那就切割,有些檔案是不可以切的
        if (isSplitable(job, path)) {
          //獲取塊大小
          long blockSize = file.getBlockSize();
          //求切割長度。根據maxsize和minsize 和 塊大小來求
          // max(minsize,min(maxsize,blocksize))
          //如果是按照預設配置的話。就是塊大小。
          long splitSize = computeSplitSize(blockSize, minSize, maxSize);
		 //剩餘長度,一開始是lenth
          long bytesRemaining = length;
          //如果當前剩餘長度>要切的長度的1.1倍。那麼才切。否則不切了
          while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
             //新增切片資訊
            int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
            splits.add(makeSplit(path, length-bytesRemaining, splitSize,
                        blkLocations[blkIndex].getHosts(),
                        blkLocations[blkIndex].getCachedHosts()));
            bytesRemaining -= splitSize;
          }
		//最後多餘了一個沒有切乾淨,就再新增一個切片資訊
          if (bytesRemaining != 0) {
            int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
            splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining,
                       blkLocations[blkIndex].getHosts(),
                       blkLocations[blkIndex].getCachedHosts()));
          }
        } //如果不能切片
          else { // not splitable
           //就不切。直接拿整個
          splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(),
                      blkLocations[0].getCachedHosts()));
        }
      } //當檔案長度等於零
        else { 
        //Create empty hosts array for zero length files
         //建立一個空的切割塊新增進去
        splits.add(makeSplit(path, 0, length, new String[0]));
      }
    }
    // Save the number of input files for metrics/loadgen
    job.getConfiguration().setLong(NUM_INPUT_FILES, files.size());
    sw.stop();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Total # of splits generated by getSplits: " + splits.size()
          + ", TimeTaken: " + sw.now(TimeUnit.MILLISECONDS));
    }
    //返回切片資訊
    return splits;
  }