한 서버의 파일을 다른 서버의hdfs에 놓다
8440 단어 경험
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.springframework.stereotype.Service;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
/**
* Created by panghu on 2017/9/13.
*/
@Service
public class FileManageService {
/**
* hdfs
* @param local
* @param target
* @throws IOException
*/
public static void fileUp(String local ,String target) throws IOException {
URL url = new URL(local);
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
// 3
conn.setConnectTimeout(3*1000);
// 403
conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
//
InputStream in = conn.getInputStream();
System.out.println(target);
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(target), conf);
OutputStream out = fs.create(new Path(target));
IOUtils.copyBytes(in, out, 4096, true);
System.out.println(" 。。。。。。。");
}
/**
* hdfs
* @param local
* @param remote
* @throws IOException
*
*
*/
public static String copyFile(String uri , String local, String remote) throws IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri), conf);
fs.copyFromLocalFile(new Path(local), new Path(remote));
System.out.println("copy from: " + local + " to " + remote);
fs.close();
return "111";
}
public static void main(String[] args) {
System.out.println(1234);
}
}