hadoop grep 예시 공유 실현

문서에서 일부 문자열을 포함하는 줄을 추출할 수 있는 간단한 grep 프로그램

/*
 *  grep ， 
 */
public class grep extends Configured  implements Tool{
 public static  class grepMap extends Mapper<LongWritable, Text, Text,NullWritable>{

  public void map(LongWritable line,Text value,Context context) throws IOException, InterruptedException{
   // Configuration 
   String str = context.getConfiguration().get("grep");
   if(value.toString().contains(str)){
    context.write(value, NullWritable.get());
   }
  }
 }
 @Override
 public int run(String[] args) throws Exception {

  if(args.length!=3){
   System.out.println("ERROR");
   System.exit(1);
  }

  Configuration configuration = getConf();
  // 
  configuration.set("grep", args[2]);
  Job job = new Job(configuration,"grep");

  job.setJarByClass(grep.class);
  job.setMapperClass(grepMap.class);
  job.setNumReduceTasks(0);

  job.setMapOutputKeyClass(Text.class);
  job.setOutputValueClass(NullWritable.class);

  Path in = new Path(args[0]);
  Path out = new Path(args[1]);
  FileSystem fileSystem = out.getFileSystem(configuration);
  if(fileSystem.exists(out))
   fileSystem.delete(out, true);

  FileInputFormat.addInputPath(job, in);
  FileOutputFormat.setOutputPath(job, out);

  System.exit(job.waitForCompletion(true)?0:1);
  return 0;
 }