Maven 설정 Spark+hadop

프로젝트 기반 Spark  2.0.0+hadop 2.8.0 구축,데이터 바 지 는 MongoDB 를 사용 합 니 다.다음은 설정 에 대한 자세 한 내용 입 니 다.버 전 충돌 이 발생 하면 다음 명령 으로 maven 의존 관 계 를 보고 오류 의존 버 전 을 제거 하 며 부족 한 의존 버 전 을 설정 합 니 다.
하나.   보기 의존
화면 음악 n 생 성 의존 분석 트 리,버 전 충돌 보기,어떤 버 전 사용 화면 음악 n dependency:tree 화면 음악 n dependency:tree>dependency.log
둘째.  pom.xml 설정

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0modelVersion>
  <groupId>    groupId>
  <artifactId>   artifactId>
  <version>1.0-SNAPSHOTversion>
  <inceptionYear>2008inceptionYear>


  <properties>
    <scala.version>2.12.3scala.version>
    <spark.artifactID.suffix>2.11spark.artifactID.suffix>
    <spark.version>2.2.0spark.version>
    <hadoop.version>2.7.4hadoop.version>
 
  properties>

  <repositories>
    <repository>
      <id>scala-tools.orgid>
      <name>Scala-Tools Maven2 Repositoryname>
      <url>http://scala-tools.org/repo-releasesurl>
    repository>
  repositories>

  <pluginRepositories>
    <pluginRepository>
      <id>scala-tools.orgid>
      <name>Scala-Tools Maven2 Repositoryname>
      <url>http://scala-tools.org/repo-releasesurl>
    pluginRepository>
  pluginRepositories>

  <dependencies>


            

    


    
      -->

    
    <dependency>
      <groupId>junitgroupId>
      <artifactId>junitartifactId>
      <version>4.4version>
      <scope>testscope>
    dependency>
    <dependency>
      <groupId>org.specsgroupId>
      <artifactId>specsartifactId>
      <version>1.2.5version>
      <scope>testscope>
    dependency>





    
    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-core_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>
    dependency>

    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-streaming_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>

    dependency>


    
    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-sql_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>

    dependency>

    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-hive_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>

    dependency>
    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-mllib_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>

    dependency>


    

    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-commonartifactId>
      <version>${hadoop.version}version>
    dependency>
    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-clientartifactId>
      <version>${hadoop.version}version>
    dependency>


    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-hdfsartifactId>
      <version>${hadoop.version}version>
    dependency>




    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-mapreduce-client-coreartifactId>
      <version>${hadoop.version}version>
      <exclusions>
        <exclusion>
          <artifactId>org.apache.hadoopartifactId>
          <groupId>hadoop-coregroupId>
        exclusion>
      exclusions>
    dependency>

    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-annotationsartifactId>
      <version>${hadoop.version}version>
    dependency>

    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-miniclusterartifactId>
      <version>${hadoop.version}version>
      <scope>testscope>
    dependency>


    

 

    <dependency>
      <groupId>org.mongodbgroupId>
      <artifactId>mongo-java-driverartifactId>
      <version>3.4.2version>
    dependency>



    <dependency>
      <groupId>org.mongodb.sparkgroupId>
      <artifactId>mongo-spark-connector_2.11artifactId>
      <version>2.2.0version>
    dependency>
 


    <dependency>
      <groupId>com.alibabagroupId>
      <artifactId>fastjsonartifactId>
      <version>1.2.44version>
    dependency>


  dependencies>

  <build>
    <sourceDirectory>src/main/scalasourceDirectory>
    <testSourceDirectory>src/test/scalatestSourceDirectory>
    <plugins>
      <plugin>
        <groupId>org.scala-toolsgroupId>
        <artifactId>maven-scala-pluginartifactId>
        <executions>
          <execution>
            <goals>
              <goal>compilegoal>
              <goal>testCompilegoal>
            goals>
          execution>
        executions>
        <configuration>
          <scalaVersion>${scala.version}scalaVersion>
          <args>
            <arg>-target:jvm-1.5arg>
          args>
        configuration>
      plugin>
      <plugin>
        <groupId>org.apache.maven.pluginsgroupId>
        <artifactId>maven-eclipse-pluginartifactId>
        <configuration>
          <downloadSources>truedownloadSources>
          <buildcommands>
            <buildcommand>ch.epfl.lamp.sdt.core.scalabuilderbuildcommand>
          buildcommands>
          <additionalProjectnatures>
            <projectnature>ch.epfl.lamp.sdt.core.scalanatureprojectnature>
          additionalProjectnatures>
          <classpathContainers>
            <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINERclasspathContainer>
            <classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINERclasspathContainer>
          classpathContainers>
        configuration>
      plugin>



      
      

<plugin>
  <artifactId>maven-compiler-pluginartifactId>
  <configuration>
    <source>1.7source>
    <target>1.7target>
    <encoding>${project.build.sourceEncoding}encoding>
  configuration>
plugin>
<plugin>
  <artifactId>maven-assembly-pluginartifactId>
  <configuration>
    <descriptorRefs>
      <descriptorRef>jar-with-dependenciesdescriptorRef>
    descriptorRefs>
    
  configuration>
  <executions>
    <execution>
      <id>make-assemblyid> 
      <phase>packagephase> 
      <goals>
        <goal>singlegoal>
      goals>
    execution>
  executions>
plugin>


 
<plugin>
  <artifactId>maven-shade-pluginartifactId>
  <version>2.4.3version>
  <executions>
    <execution>
      <phase>packagephase>
      <goals>
        <goal>shadegoal>
      goals>
      <configuration>
        <filters>
          <filter>
            <artifact>*:*artifact>
            <excludes>
              <exclude>META-INF/*.SFexclude>
              <exclude>META-INF/*.DSAexclude>
              <exclude>META-INF/*.RSAexclude>
            excludes>
          filter>
        filters>
      configuration>
    execution>
  executions>
plugin>

plugins
>
build
>
<
reporting
>
<
plugins
>
<
plugin
>
<
groupId
>org.scala-tools
groupId
>
<
artifactId
>maven-scala-plugin
artifactId
>
<
configuration
>
<
scalaVersion
>${scala.version}
scalaVersion
>
configuration
>
plugin
>
plugins
>
reporting
>
project
>