spark에서 elasticsearch로 데이터 쓰기

7358 단어 elasticsearch
pom.xml 

  4.0.0
  com.ultrapower.scala
  spark_scala_demo
  1.0-SNAPSHOT
  2017
  
    2.10
    1.6.1
  
  
    
      org.apache.spark
      spark-core_${scala.version}
      ${spark.version}
    
    
      org.apache.spark
      spark-streaming-kafka_2.10
      1.6.1
    
    
      org.apache.spark
      spark-streaming_${scala.version}
      ${spark.version}
    
    
      org.apache.kafka
      kafka-clients
      1.0.1
    
    
      org.apache.spark
      spark-mllib_${scala.version}
      ${spark.version}
    
    
      redis.clients
      jedis
      2.8.1
    
    
      org.apache.commons
      commons-pool2
      2.4.2
    

    
      org.elasticsearch
      elasticsearch-hadoop
      5.4.0
    

    
      net.minidev
      json-smart
      2.3
    

  
  
    src/main/scala
    src/test/scala
    
    
      maven-compiler-plugin
      3.6.0
      
        1.8
        1.8
      
    
    
  

package com.ultrapower.scala.es;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.elasticsearch.spark.rdd.api.java.JavaEsSpark;

import java.util.Map;

/**
 * @author xiefg
 * @create 2018-04-26 10:44
 * @desc  spark  es
 **/
public class SparkToEs {

    public static void main(String[] args) {
        SparkConf conf=new SparkConf().setAppName("SparkToEs").setMaster("local");
        conf.set("es.index.auto.create", "true");
        conf.set("es.nodes", "192.168.186.229");
        conf.set("es.port", "9200");
        conf.set("es.nodes.wan.only", "true");
        JavaSparkContext jsc = new JavaSparkContext(conf);
        Map, ?> numbers = ImmutableMap.of("one", 1, "two", 2);
        Map, ?> airports = ImmutableMap.of("OTP", "Otopeni", "SFO", "San Fran");
        JavaRDD, ?>> javaRDD = jsc.parallelize(ImmutableList.of(numbers, airports));
        JavaEsSpark.saveToEs(javaRDD, "spark/xiefg");

    }
}
홈페이지 참조:https://www.elastic.co/guide/en/elasticsearch/hadoop/current/spark.html

좋은 웹페이지 즐겨찾기