한 서버의 파일을 다른 서버의hdfs에 놓다

8440 단어 경험


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.springframework.stereotype.Service;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;

/**
 * Created by panghu on 2017/9/13.
 */
@Service
public class FileManageService {


    /**
     *  hdfs 
     * @param local
     * @param target
     * @throws IOException
     */
    public static void fileUp(String local ,String target) throws IOException {
        URL url = new URL(local);
        HttpURLConnection conn = (HttpURLConnection)url.openConnection();
        // 3 
        conn.setConnectTimeout(3*1000);
        // 403 
        conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
        // 
        InputStream in = conn.getInputStream();
        System.out.println(target);
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(target), conf);
        OutputStream out = fs.create(new Path(target));
        IOUtils.copyBytes(in, out, 4096, true);
        System.out.println(" 。。。。。。。");
    }


    /**
     *  hdfs
     * @param local
     * @param remote
     * @throws IOException
     *
     *
     */
    public static String copyFile(String uri , String local, String remote) throws IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        fs.copyFromLocalFile(new Path(local), new Path(remote));
        System.out.println("copy from: " + local + " to " + remote);
        fs.close();
        return "111";
    }

    public static void main(String[] args) {
        System.out.println(1234);
    }
}

좋은 웹페이지 즐겨찾기