SparkSql select 쿼리 실행

4928 단어 sparksql
package cn.itcast.spark.test

import java.sql.{Connection, Date, DriverManager, PreparedStatement}
import java.util.Properties

import org.apache.spark.sql.{DataFrame, SQLContext}
import org.apache.spark.{SparkConf, SparkContext}

//  sparksql     
object test3 {
  //     
  lazy val url = "jdbc:mysql://192.168.10.190:3306/bigdata"
  lazy val url2 = "jdbc:mysql://192.168.10.190:3306/bigdata?user=root&password=x5"
  lazy val username = "root"
  lazy val password = "x5"

  def main(args: Array[String]) {
    var conn: Connection = null
    var ps : PreparedStatement = null

    //val sparkConf = new SparkConf().setAppName("sparkSqlTest").setMaster("yarn-cluster").set("spark.app.id", "sqlTest")
    val sparkConf = new SparkConf().setAppName("SQLDemo").setMaster("local")
    //  context
    val sc = new SparkContext(sparkConf)
    //  sqlContext
    val sqlContext = new SQLContext(sc)
    //      ,    spark sql    
    import sqlContext.implicits._
    //  jdbc    
    val uri = url + "?user=" + username + "&password=" + password + "&useUnicode=true&characterEncoding=UTF-8"
    val prop = new Properties()
    //  :      ,        ,       mysql     
    prop.put("driver", "com.mysql.jdbc.Driver")
    //  mysql   
    val df_test1: DataFrame = sqlContext.read.jdbc(uri, "user_t", prop)
    val df_test2: DataFrame = sqlContext.read.jdbc(uri, "t_user2", prop)

    //    
    val predicates = Array[String]("id <= 2", "id >= 4 and id <= 5 ")
    val df_test3: DataFrame = sqlContext.read.jdbc(uri, "t_user2",predicates, prop)

    // dataframe       
    //     
    df_test2.select("id", "name", "age").collect()
      .foreach(row => {
        println("id  " + row(0) + " ,name  " + row(1) + ", age  " + row(2))
      })

    df_test3.select("id", "name", "age").collect()
      .foreach(row => {
        println("id  " + row(0) + " ,name  " + row(1) + ", age  " + row(2))
      })
  }
}

좋은 웹페이지 즐겨찾기