아 리 면접: 큰 파일 10G, 한 줄 의 숫자 (Integer, 빈 칸 구분) 중 가장 큰 10 개의 숫자 값 을 찾 아야 합 니 다.

     10G，        (Integer,     )，        10     
         ，          20、10、13、20、3，    3     20、13、10

    ：
1.            ，        ，   1000   ，  10   ，      1-100，       101-200，     
2.               ，                  10   ，     list   
3. list              ，   10   

import org.apache.commons.lang.StringUtils;

import java.io.FileReader;
import java.io.IOException;
import java.io.LineNumberReader;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.TreeSet;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;

/**
 *      10G，        (Integer,     )，        10     
 *          ，
 *           20、10、13、20、3，    3     20、13、10
 */
public class BigFileSortUtil {

    //             10      
    private static CopyOnWriteArrayList> treeSetList = new CopyOnWriteArrayList<>();

    private static ThreadPoolExecutor executor =
            new ThreadPoolExecutor(
                    10,
                    1000,
                    10L,
                    TimeUnit.SECONDS,
                    new ArrayBlockingQueue<>(500), new NameThreadFactory("BigFileSortUtil"));

    public static void main(String[] args) throws InterruptedException {
        String fileName = System.getProperty("user.dir") + "/test.txt";

        System.out.println(sortFile(fileName));
    }


    public static String sortFile(String fileName) throws InterruptedException {
        //       
        int lineNumbers = getFileLineNum(fileName);

        Map dataRange = new LinkedHashMap<>();

        // 10GB=10*1024*1024*1024 bytes,     1KB，  10*1024*1024   ，  10000000    
        //         10000   ，   1000   
        int maxPageSize = 50000;
        
        //               
        for (Integer i = 0; i < lineNumbers; i += maxPageSize) {
            dataRange.put(i, Math.min(i + maxPageSize, lineNumbers));
        }
        
        CountDownLatch countDownLatch = new CountDownLatch(dataRange.size());
        for (Map.Entry entry : dataRange.entrySet()) {
            int startLine = entry.getKey() + 1;
            int pageSize = entry.getValue() - entry.getKey();
            
            //         
            executor.submit(new SortRunnable(countDownLatch, fileName, startLine, pageSize));
        }
        countDownLatch.await();
        
        // Set    ，Set       
        TreeSet treeSet = new TreeSet<>(Comparator.reverseOrder());
        for (TreeSet datas : treeSetList) {
            treeSet.addAll(datas);
        }
        
        //   Set       
        try (Stream stream = treeSet.stream().limit(10)) {
            return StringUtils.join(stream.toArray(), ",");
        }

    }

    /**
     * @return    10   ，   ，    ：20、18、13、10、9、8、7、6、5、1
     */
    public static TreeSet sortFile(Stream lines) {
        TreeSet list = new TreeSet<>();
        //       
        lines.forEach(line -> {
            if (StringUtils.isNotBlank(line)) {
                //      ，             
                String[] lineDatas = line.split("\\s");

                for (String lineData : lineDatas) {
                    int data = Integer.parseInt(lineData);

                    list.add(data);

                    if (list.size() > 10) {
                        list.pollFirst();
                    }
                }
            }
        });
        return list;
    }

    public static int getFileLineNum(String fileName) {
        try (LineNumberReader lineNumberReader = new LineNumberReader(new FileReader(fileName))) {
            lineNumberReader.skip(Long.MAX_VALUE);
            int lineNumber = lineNumberReader.getLineNumber();
            //            ,     +1
            return lineNumber + 1;
        } catch (IOException e) {
            return -1;
        }
    }

    private static class SortRunnable implements Runnable {

        /**
         *      
         */
        private CountDownLatch countDownLatch;

        /**
         *     
         */
        private String fileName;

        /**
         *          
         */
        private int startLine;

        /**
         *        
         */
        private int pageSize;

        public SortRunnable(CountDownLatch countDownLatch,
                            String fileName,
                            int startLine, int pageSize) {
            this.countDownLatch = countDownLatch;
            this.fileName = fileName;
            this.startLine = startLine;
            this.pageSize = pageSize;
        }

        @Override
        public void run() {
            try (LineNumberReader lineNumberReader = new LineNumberReader(new FileReader(fileName))) {

                //          
                lineNumberReader.setLineNumber(startLine);

                //          
                Stream lines = lineNumberReader.lines().limit(pageSize);

                TreeSet treeSet = sortFile(lines);

                treeSetList.add(treeSet);

            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                countDownLatch.countDown();
            }
        }
    }

    private static class NameThreadFactory implements ThreadFactory {

        private final static AtomicInteger POOL_SEQ = new AtomicInteger();

        private String prefix;

        private ThreadGroup group;

        private boolean demon = false;

        public NameThreadFactory() {
            this("pool-" + POOL_SEQ.getAndIncrement(), false);
        }

        public NameThreadFactory(String prefix) {
            this(prefix, false);
        }


        public NameThreadFactory(String prefix, boolean demon) {
            this.prefix = prefix + "-thread-";

            SecurityManager securityManager = System.getSecurityManager();

            ThreadGroup threadGroup = securityManager != null ? securityManager.getThreadGroup() : null;
            this.group = threadGroup != null ? threadGroup : Thread.currentThread().getThreadGroup();

            this.demon = demon;
        }

        @Override
        public Thread newThread(Runnable runnable) {
            String name = prefix + POOL_SEQ.getAndIncrement();
            Thread thread = new Thread(group, runnable, name, 0);
            thread.setDaemon(demon);
            return thread;
        }
    }

}