Spark SQL saveMode 방식

1051 단어 Spark
package com.bjsxt.java.spark.sql.loadsave;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.SaveMode;

/**
 *   json  parquet      DataFrame
 * DataFrame         ,       parquet
 * SaveMode.Overwrite:  
 * Append:
 * @author root
 *
 */

public class SaveModeTest {

	@SuppressWarnings("deprecation")
	public static void main(String[] args) {
		SparkConf conf = new SparkConf()   
				.setAppName("SaveModeTest")
				.setMaster("local");
		JavaSparkContext sc = new JavaSparkContext(conf);
		SQLContext sqlContext = new SQLContext(sc);
	
		
		DataFrame peopleDF = sqlContext.read().json("hdfs://hadoop1:9000/input/people.json"); 
		
		
		peopleDF.write().mode(SaveMode.Append).save("hdfs://hadoop1:9000/output/namesAndFavColors_scala"); 

		sqlContext.read().parquet("hdfs://hadoop1:9000/output/namesAndFavColors_scala").show();
	}
}

좋은 웹페이지 즐겨찾기