자바 API Hadoop HDFS 작업 실현

의지 하 다

<dependency>
    <groupId>org.apache.hadoopgroupId>
    <artifactId>hadoop-commonartifactId>
    <version>2.7.3version>
dependency>
<dependency>
    <groupId>org.apache.hadoopgroupId>
    <artifactId>hadoop-hdfsartifactId>
    <version>2.7.3version>
dependency>
<dependency>
    <groupId>org.apache.hadoopgroupId>
    <artifactId>hadoop-clientartifactId>
    <version>2.7.3version>
dependency>

2, Java API

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * HDFS   
 */
public class HDFSUtil {

    /**
     *      
     * @param conf
     * @param path
     * @throws IOException
     */
    public static void mkdir(Configuration conf, String path) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        fs.mkdirs(new Path(path));
        fs.close();
    }

    /**
     *   
     * @param conf
     * @param path
     * @throws IOException
     */
    public static void delete(Configuration conf, String path) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        fs.delete(new Path(path), true); //  true          ，     
        fs.close();
    }

    /**
     *       
     * listFiles      ，      。    
     * @param conf
     * @param path
     * @return
     * @throws IOException
     */
    public static List<LocatedFileStatus> fileList(Configuration conf, String path) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> iterator =  fs.listFiles(new Path(path), true); // true   
        List<LocatedFileStatus> resultList = new ArrayList<>();
        while (iterator.hasNext()) {
            LocatedFileStatus locatedFileStatus  = iterator.next();
            resultList.add(locatedFileStatus);
        }
        fs.close();
        return resultList;
    }

    /**
     *       
     * listStatus          ，     。        。
     * @param conf
     * @param path
     * @return
     * @throws IOException
     */
    public static List<FileStatus> fileList2(Configuration conf, String path, List<FileStatus> resultList) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] fileStatus =  fs.listStatus(new Path(path));
        if (resultList == null) {
            resultList = new ArrayList<>();
        }
        for(FileStatus s : fileStatus) {
            //          
            if (s.isDirectory()) {
                //System.out.println("  ："  + s.getPath());
                fileList2(conf, s.getPath().toUri().getPath(), resultList);
            } else {
                //System.out.println("  ：" + s.getPath());
                resultList.add(s);
            }
        }
        fs.close();
        return resultList;
    }

    /**
     *   
     * @param conf
     * @param filePath
     * @param hdfsPath
     * @throws IOException
     */
    public static void upload(Configuration conf, String filePath, String hdfsPath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        /*InputStream input = new FileInputStream(filePath);
        OutputStream out = fs.create(new Path(hdfsPath));
        IOUtils.copyBytes(input, out, 1024);*/
        fs.copyFromLocalFile(new Path(filePath), new Path(hdfsPath));
    }

    /**
     *   
     * @param conf
     * @param hdfsPath
     * @param savePath
     * @throws IOException
     */
    public static void download(Configuration conf, String hdfsPath, String savePath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        /*InputStream input = fs.open(new Path(hdfsPath));
        OutputStream out = new FileOutputStream(savePath);
        IOUtils.copyBytes(input, out, 1024);*/
        fs.copyToLocalFile(new Path(hdfsPath), new Path(savePath));
    }

    public static void main(String[] args) throws IOException {
        //      
        System.setProperty("HADOOP_USER_NAME","root");
        //System.setProperty(" hadoop.home.dir","D:\\hadoop-2.7.3");

        //     ，  NameNode  
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS","hdfs://10.1.255.121:9000");
        //      
        conf.set("dfs.blocksize","256m");
        //      
        conf.set("dfs.replication","2");

        //mkdir(conf, "/abc12323");

        //delete(conf, "/test");

        //upload(conf, "E:/jar/fastjson-1.2.7.jar", "/abc123/fastjson-1.2.7.jar");

        //download(conf, "/2020/a.txt", "E:/test/a.txt");

        /*List list = fileList(conf, "/");
        for (LocatedFileStatus locatedFileStatus: list) {
            System.out.println(locatedFileStatus.getPath());
        }*/

       /* System.out.println("----------------");
        List list2 = fileList2(conf, "/", null);
        for(FileStatus s : list2) {
            System.out.println(s.getPath());
        }*/


     }
}