작은 메모리 빅 데이터 정렬

9856 단어 Java


import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
 
/**
 * 
 * run configuration -Xms5m -Xmx5m
 * 
 */
public class BigDataSort {
 
    public final static String SMALL_FILE_PATH = "C://BigData//";
    public final static int BIG_NUM_LINE = 200;
    public final static String ORING_FILE_PATH = "C://BigData//bigData.txt";
    public final static int SMALL_FILE_LINE = 10; //1M for 1 small file
    private File tempFiles[];
 
   
 
    private void createBigsortNums() throws IOException {
        BufferedWriter writer = new BufferedWriter(new FileWriter(ORING_FILE_PATH));
        Random random = new Random();
        for (int i = 0; i < BIG_NUM_LINE; i++) {
            writer.write(String.valueOf(random.nextInt(100000000)));
            if(i!=BIG_NUM_LINE-1)writer.newLine();//           
        }
        writer.close();
    }
  
    private void beSmallFileAndSort() throws IOException {
        BufferedReader bigDataFile = new BufferedReader(new FileReader(ORING_FILE_PATH));
        List smallLine = null;
        tempFiles = new File[BIG_NUM_LINE / SMALL_FILE_LINE];
        for (int i = 0; i < tempFiles.length; i++) {
            tempFiles[i] = new File(SMALL_FILE_PATH + "sortTempFile" + i + ".txt");
            BufferedWriter smallWtite = new BufferedWriter(new FileWriter(tempFiles[i]));
            smallLine = new ArrayList();
            for (int j = 0; j < SMALL_FILE_LINE; j++)
                smallLine.add(Integer.parseInt(bigDataFile.readLine()));
            Collections.sort(smallLine);
            for(int k=0;k	smallWtite.write(String.valueOf(smallLine.get(k)));
            	if(k!=smallLine.size()-1)smallWtite.newLine();
            }
           /* for (Object num : smallLine.toArray())
                smallWtite.write(num + "
");*/             smallWtite.close();         }     }       private void unitFileToSort() throws IOException {         File tempFile = null;         for(int  i=1;i Thread.sleep(1500); // } catch (InterruptedException e) { // // TODO Auto-generated catch block // e.printStackTrace(); // }         }         File newFile = new File(SMALL_FILE_PATH+"sortResult.txt");         if(newFile.exists())newFile.delete();         tempFile.renameTo(newFile);     }     /*   private void unitFileToSort() throws IOException {         File tempFile = null;         for(int  i=0;i   BufferedReader fromRd = new BufferedReader(new FileReader(SMALL_FILE_PATH+"1.txt"));        BufferedReader toTempRd = new BufferedReader(new FileReader(SMALL_FILE_PATH+"2.txt"));        File newSortFile = new File(SMALL_FILE_PATH+"testunit.txt");        BufferedWriter newSortFileWt = new BufferedWriter(new FileWriter(newSortFile));        int index = Integer.MIN_VALUE;        int toPoint = Integer.MIN_VALUE;        index = Integer.parseInt(fromRd.readLine());        toPoint = Integer.parseInt(toTempRd.readLine());        while (fromRd.ready()||toTempRd.ready()) {                       if (index <= toPoint) {                newSortFileWt.write(String.valueOf(index));                newSortFileWt.newLine();                if(fromRd.ready())index = Integer.parseInt(fromRd.readLine());                else break;            } else {                                    newSortFileWt.write(String.valueOf(toPoint));                    newSortFileWt.newLine();                    if(toTempRd.ready())toPoint = Integer.parseInt(toTempRd.readLine());                    else break;            }                   } System.out.println(index + " " + toPoint); if (fromRd.ready() && (!toTempRd.ready())) { newSortFileWt.write(String.valueOf(index)); newSortFileWt.newLine(); while (fromRd.ready()) { newSortFileWt.write(fromRd.readLine()); if (fromRd.ready())newSortFileWt.newLine(); } } else if ((!fromRd.ready()) && toTempRd.ready()) { newSortFileWt.write(String.valueOf(toPoint)); newSortFileWt.newLine(); while (toTempRd.ready()) { newSortFileWt.write(toTempRd.readLine()); if(toTempRd.ready())newSortFileWt.newLine(); } } else if((!fromRd.ready())&&(!toTempRd.ready())){         newSortFileWt.write(String.valueOf(index<=toPoint?index:toPoint));         newSortFileWt.newLine();             newSortFileWt.write(String.valueOf(index>toPoint?index:toPoint));            // newSortFileWt.newLine();         }                            newSortFileWt.close();        fromRd.close();        toTempRd.close();                BufferedReader newFile=new BufferedReader(new FileReader(SMALL_FILE_PATH+"testunit.txt"));        while(newFile.ready()){       System.out.println(newFile.readLine());        }        newFile.close();    }     public static File sortBySmallFile(File fromFile, File toFile, int i) throws IOException {         BufferedReader fromRd = new BufferedReader(new FileReader(fromFile));         BufferedReader toTempRd = new BufferedReader(new FileReader(toFile));         File newSortFile = new File(SMALL_FILE_PATH+i+"unit.txt");         System.out.println(fromFile.getPath());         BufferedWriter newSortFileWt = new BufferedWriter(new FileWriter(newSortFile));         int index =  Integer.parseInt(fromRd.readLine());         int toPoint =  Integer.parseInt(toTempRd.readLine());         while (fromRd.ready()||toTempRd.ready()) {                          if (index <= toPoint) {                 newSortFileWt.write(String.valueOf(index));                 newSortFileWt.newLine();                 if(fromRd.ready())index = Integer.parseInt(fromRd.readLine());                 else break;             } else {                                      newSortFileWt.write(String.valueOf(toPoint));                     newSortFileWt.newLine();                     if(toTempRd.ready())toPoint = Integer.parseInt(toTempRd.readLine());                     else break;             }                     }         //     if (fromRd.ready() && (!toTempRd.ready())) { newSortFileWt.write(String.valueOf(index)); newSortFileWt.newLine(); while (fromRd.ready()) { newSortFileWt.write(fromRd.readLine()); if (fromRd.ready())newSortFileWt.newLine();// } } else if ((!fromRd.ready()) && toTempRd.ready()) { newSortFileWt.write(String.valueOf(toPoint)); newSortFileWt.newLine(); while (toTempRd.ready()) { newSortFileWt.write(toTempRd.readLine()); if(toTempRd.ready())newSortFileWt.newLine(); } } else if((!fromRd.ready())&&(!toTempRd.ready())){         newSortFileWt.write(String.valueOf(index<=toPoint?index:toPoint));         newSortFileWt.newLine();             newSortFileWt.write(String.valueOf(index>toPoint?index:toPoint));            // newSortFileWt.newLine();         }         newSortFileWt.close();         fromRd.close();         toTempRd.close();         toFile.delete();         return newSortFile;     }       public static void main(String[] args) throws IOException {     BigDataSort bds=new BigDataSort();     bds.createBigsortNums();     bds.beSmallFileAndSort();     bds.unitFileToSort();       //testSort();     } }
sortBySmallFile 방법의 사고방식 은 매우 간단 하지만 코드 를 잘 쓰 지 못 해서 좋 은 것 을 생각해 내지 못 했다.첫 번 째 파일 과 두 번 째 파일 을 순서대로 합 친 다음 에 합 친 파일 을 세 번 째 파일 과 합 친 것 으로 유추 하 는 것 이다.

좋은 웹페이지 즐겨찾기