(5-1) InputFormat 소스 분석

//InputFormat 추상 클래스, 추상 방법 get Splits () 2개와createRecordReader ()

package org.apache.hadoop.mapreduce;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;


public abstract class InputFormat {


  public abstract  List getSplits(JobContext context) throws IOException, InterruptedException;
  
  public abstract  RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException,  InterruptedException;
}

//추상 클래스 File InputFormat는 InputFormat 클래스를 계승하고 get Splits () 방법을 실현했으며 get Splits () 방법은 파일을 Input Split로 나누고 Input Split의 개수는 맵 () 방법에 대응하는 개수로 나눈다.

package org.apache.hadoop.mapreduce.lib.input;
public abstract class FileInputFormat extends InputFormat {
  public List getSplits(JobContext job) throws IOException {
    Stopwatch sw = new Stopwatch().start();
    long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
    long maxSize = getMaxSplitSize(job);


    // generate splits
    List splits = new ArrayList();
    List files = listStatus(job);
    for (FileStatus file: files) {
      Path path = file.getPath();
      long length = file.getLen();
      if (length != 0) {
        BlockLocation[] blkLocations;
        if (file instanceof LocatedFileStatus) {
          blkLocations = ((LocatedFileStatus) file).getBlockLocations();
        } else {
          FileSystem fs = path.getFileSystem(job.getConfiguration());
          blkLocations = fs.getFileBlockLocations(file, 0, length);
        }
        if (isSplitable(job, path)) {
          long blockSize = file.getBlockSize();
          long splitSize = computeSplitSize(blockSize, minSize, maxSize);


          long bytesRemaining = length;
          while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
            int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
            splits.add(makeSplit(path, length-bytesRemaining, splitSize,
                        blkLocations[blkIndex].getHosts(),
                        blkLocations[blkIndex].getCachedHosts()));
            bytesRemaining -= splitSize;
          }


          if (bytesRemaining != 0) {
            int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
            splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining,
                       blkLocations[blkIndex].getHosts(),
                       blkLocations[blkIndex].getCachedHosts()));
          }
        } else { // not splitable
          splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(),
                      blkLocations[0].getCachedHosts()));
        }
      } else { 
        //Create empty hosts array for zero length files
        splits.add(makeSplit(path, 0, length, new String[0]));
      }
    }
    // Save the number of input files for metrics/loadgen
    job.getConfiguration().setLong(NUM_INPUT_FILES, files.size());
    sw.stop();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Total # of splits generated by getSplits: " + splits.size()
          + ", TimeTaken: " + sw.elapsedMillis());
    }
    return splits;
  }
}

//TextInputFormat 클래스는 File InputFormat 클래스를 계승하고createRecordReader () 방법을 실현했으며,createRecordReader ()는 InputSplit를 맵 () 처리 가능한 쌍으로 나누었습니다.

package org.apache.hadoop.mapreduce.lib.input;
public class TextInputFormat extends FileInputFormat {
  @Override
  public RecordReader  createRecordReader(InputSplit split,TaskAttemptContext context) {
    String delimiter = context.getConfiguration().get("textinputformat.record.delimiter");
    byte[] recordDelimiterBytes = null;
    if (null != delimiter)
      recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
    return new LineRecordReader(recordDelimiterBytes);
  }
}

//RecordReader 추상 클래스, TextInputFormat 클래스에서 클래스 객체를 사용합니다.

package org.apache.hadoop.mapreduce;
public abstract class RecordReader implements Closeable {
  public abstract void initialize(InputSplit split,TaskAttemptContext context) throws IOException, InterruptedException;


  public abstract  boolean nextKeyValue() throws IOException, InterruptedException;


  public abstract KEYIN getCurrentKey() throws IOException, InterruptedException;
  
  public abstract   VALUEIN getCurrentValue() throws IOException, InterruptedException;
}

//LineRecordReader 클래스는 RecordReader 클래스를 계승하고 nextKetVlue(), getCurrentKey(), getCurrentValue() 방법을 실현했다.

package org.apache.hadoop.mapreduce.lib.input;
@InterfaceAudience.LimitedPrivate({"MapReduce", "Pig"})
@InterfaceStability.Evolving
public class LineRecordReader extends RecordReader {
  public boolean nextKeyValue() throws IOException {
    if (key == null) {
      key = new LongWritable();
    }
    key.set(pos);
    if (value == null) {
      value = new Text();
    }
    int newSize = 0;
    // We always read one extra line, which lies outside the upper
    // split limit i.e. (end - 1)
    while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) {
      if (pos == 0) {
        newSize = skipUtfByteOrderMark();
      } else {
        newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos));
        pos += newSize;
      }


      if ((newSize == 0) || (newSize < maxLineLength)) {
        break;
      }


      // line too long. try again
      LOG.info("Skipped line of size " + newSize + " at pos " + 
               (pos - newSize));
    }
    if (newSize == 0) {
      key = null;
      value = null;
      return false;
    } else {
      return true;
    }
  }


  @Override
  public LongWritable getCurrentKey() {
    return key;
  }


  @Override
  public Text getCurrentValue() {
    return value;
  }
}