JAVA WEB 파일 데이터 가 져 오기 (CSV, TXT, 엑셀 (큰 파일 읽 기)

18533 단어 JAVA
디자인 사고방식: 1. 먼저 파일 을 서버 에 업로드 하고 파일 이름과 파일 이 저 장 된 경 로 를 가 져 옵 니 다. 2. 현재 프로젝트 가 실행 중인 경 로 를 가 져 옵 니 다. 서버 에 있 는 파일 읽 기 3, 파일 데이터 분석 4, 데이터 처리, 데이터베이스 가 져 오기
파일 부분 업로드 코드:
	if (!file.isEmpty()) {
		FileTb upLoadFile = null;
		String originalName = file.getOriginalFilename();
		int dotPosition = originalName.indexOf(".");
		String fileType = null;
		String filename;
		if (dotPosition != -1) {
			fileType = originalName.substring(
					originalName.indexOf(".") + 1, originalName.length());
			filename = originalName;
		} else {
			filename = DateUtils.currentDate();
		}
		File savedFile = new File(path, filename);
		try {
			FileUtils.copyInputStreamToFile(file.getInputStream(),
					savedFile);
		} catch (IOException e) {
			e.printStackTrace();
  • 엑셀 파일 업로드 (특정한 처리 방식 POI 상용 모드 포함: 작은 데이터 양 에 적합):
     	 Workbook wookbook = getSheets(path);
     //  sheet   
     for (int i = 0; i < wookbook.getNumberOfSheets(); i++) {
     	Date startTime = new Date();
     	//       
     	Sheet sheet = wookbook.getSheetAt(i);
     	
     	//1.             :importList:List
     	
     	List importList = new ArrayList<>();
     	
     	
     	//2. excel       key-value   :excelFieldMap
     	Map excelFieldMap = new HashMap<>();
     	Row headRow = sheet.getRow(0);
     	//3.     
     	LinkedList propsTabList = new LinkedList<>();
     	propsTabList.add(MYJZConfig.TABLENAMEPROMARYKEY);
     	if (headRow != null) {
     		for (int j = 0; j < headRow.getPhysicalNumberOfCells(); j++) {
     			
     			//4.                id:propsList :List
     			List propsList = commenUtilMapper.findAllPropsName('"' + tableName + '"');
     			propsList.remove(MYJZConfig.TABLENAMEPROMARYKEY);
     			
     			//5.                  excel      ,  1. excel      :excelfield
     			
     			for (String prop : propsList) {
     				if (headRow.getCell(j).getStringCellValue().equalsIgnoreCase(excelTableFieldMapper.findExcelFieldByTabNameAndTabFiledName(tableName, prop))) {
     					
     					//6. excelFieldMap  value  prop  
     					excelFieldMap.put(prop, j);
     					
     					propsTabList.add(prop);
     				}
     			}
     		}
     		//TODO     -----------------------------------
     		//6.        
     		int totalRowNum = sheet.getLastRowNum();
     		int successCount = 0;
     		//7.      :    index  
     		
     		for (int rowNum = 1; rowNum < totalRowNum; rowNum++) {
     			
     			Row row = sheet.getRow(rowNum);
     	
     		}
    
    /* * *
  • 시트 가 져 오기

  • @ param filePath: 전체 file 의 URL
  • @return */

  • 	private Workbook getSheets(String filePath) {
    		
    		FileInputStream fis = null;
    		Workbook wookbook = null;
    		try {
    			//          
    			fis = new FileInputStream(filePath);
    		} catch (Exception e) {
    			e.printStackTrace();
    		}
    		if (filePath.endsWith(".xls")) {
    			/*2003   excel, .xls  :     */
    			try {
    				wookbook = new HSSFWorkbook(fis);//
    			} catch (IOException e) {
    				e.printStackTrace();
    			}
    		} else if (filePath.endsWith(".xlsx")) {
    			try {
    				/*2007   excel, .xlsx   :   */
    				wookbook = new XSSFWorkbook(fis);
    			} catch (IOException e) {
    				//  Auto-generated catch block
    				e.printStackTrace();
    			}
    		} else {
    			throw new CustomException("    excel  ");
    		}
    		
    		return wookbook;
    	}
    
  • csv 파일 가 져 오기:
     /*       */
     try {
     	
     	//   CSV      :CsvReader(    ,   ,    );
     	CsvReader reader = new CsvReader(savePath, ',', Charset.forName("UTF-8"));
     	
     	//              ,      
     	reader.readHeaders();
     	
     	/**
     	 *           
     	 */
     	List tableFieldsList = commenUtilMapper.findAllPropsName('"' + tableName + '"');
     	
     	//           
     	while (reader.readRecord()) {
     		
     		String excelfieldValue = reader.get(excelfield);
    
     	}
    
     	reader.close();
    
  • txt 데이터 파일 가 져 오기:
     RandomAccessFile raf = new RandomAccessFile(path, “rw”);
       while ((line_record = raf.readLine()) != null) {
     line_record = new String(line_record.getBytes("ISO-8859-1"), "utf8");
    
  • excel 2007: 빠 른 데이터 읽 기 (POI 사용자 모드: 빅 데이터 에 적합)
    package com.swad.asura.common;
    
    import java.io.IOException;
    import java.io.InputStream;
    import java.io.PrintStream;
    import java.util.ArrayList;
    import java.util.Date;
    import java.util.HashMap;
    import java.util.List;
    import java.util.Map;
    
    import javax.xml.parsers.ParserConfigurationException;
    
    import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
    import org.apache.poi.openxml4j.opc.OPCPackage;
    import org.apache.poi.openxml4j.opc.PackageAccess;
    import org.apache.poi.ss.usermodel.DataFormatter;
    import org.apache.poi.util.SAXHelper;
    import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
    import org.apache.poi.xssf.eventusermodel.XSSFReader;
    import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler;
    import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler.SheetContentsHandler;
    import org.apache.poi.xssf.model.StylesTable;
    import org.apache.poi.xssf.usermodel.XSSFComment;
    import org.xml.sax.ContentHandler;
    import org.xml.sax.InputSource;
    import org.xml.sax.SAXException;
    import org.xml.sax.XMLReader;
    
    public class ExcelAnalyseUtils {
    	public static void main(String[] args) {
    		Date da = new Date();
    		List> datas = getFileData(
    				"D:\\apache-tomcat-8.5.20\\webapps\\files\\upload\\201805161306480986.xlsx", "  1");
    		System.out.println(new Date().getTime() - da.getTime());
    	}
    
    	public static List> getFileData(String filePath, String sheeName) {
    		List> results = new ArrayList<>();
    		ExcelParser parser;
    		OPCPackage pkg = null;
    		try {
    			pkg = OPCPackage.open(filePath, PackageAccess.READ);
    			parser = new ExcelParser(pkg, null, -1, results);
    			parser.process(sheeName);
    		} catch (OpenXML4JException | SAXException | IOException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    		} finally {
    			if (null != pkg) {
    				try {
    					pkg.close();
    				} catch (IOException e) {
    					// TODO Auto-generated catch block
    					e.printStackTrace();
    				}
    			}
    		}
    
    		return results;
    	}
    }
    
    class ExcelParser {
    	/**
    	 * Uses the XSSF Event SAX helpers to do most of the work of parsing the
    	 * Sheet XML, and outputs the contents as a (basic) CSV.
    	 */
    	private class SheetToCSV implements SheetContentsHandler {
    		private boolean firstCellOfRow;
    		private int currentRow = -1;
    		private int currentCol = -1;
    		private Map record = new HashMap();
    		private int index = 0;
    		private List title = new ArrayList<>();
    
    		private void outputMissingRows(int number) {
    			for (int i = 0; i < number; i++) {
    				for (int j = 0; j < minColumns; j++) {
    					output.append(',');
    				}
    				output.append('
    '); } } @Override public void startRow(int rowNum) { firstCellOfRow = true; currentRow = rowNum; currentCol = -1; index = 0; } @Override public void endRow(int rowNum) { if (rowNum > 0) { Map result = new HashMap(); result.putAll(record); record.clear(); results.add(result); } } @Override public void cell(String cellReference, String formattedValue, XSSFComment comment) { if (currentRow == 0) { title.add(formattedValue); } else { record.put(title.get(index), formattedValue); } index++; } @Override public void headerFooter(String text, boolean isHeader, String tagName) { } } private final OPCPackage xlsxPackage; /** * Number of columns to read starting with leftmost */ private final int minColumns; /** * Destination for data */ private final PrintStream output; private final List> results; /** * Creates a new XLSX -> CSV examples * * @param pkg * The XLSX package to process * @param output * The PrintStream to output the CSV to * @param minColumns * The minimum number of columns to output, or -1 for no minimum */ public ExcelParser(OPCPackage pkg, PrintStream output, int minColumns, List> results) { this.xlsxPackage = pkg; this.output = output; this.minColumns = minColumns; this.results = results; } /** * Parses and shows the content of one sheet using the specified styles and * shared-strings tables. * * @param styles * The table of styles that may be referenced by cells in the * sheet * @param strings * The table of strings that may be referenced by cells in the * sheet * @param sheetInputStream * The stream to read the sheet-data from. * * @exception java.io.IOException * An IO exception from the parser, possibly from a byte * stream or character stream supplied by the application. * @throws SAXException * if parsing the XML data fails. */ public void processSheet(StylesTable styles, ReadOnlySharedStringsTable strings, SheetContentsHandler sheetHandler, InputStream sheetInputStream) throws IOException, SAXException { DataFormatter formatter = new DataFormatter(); InputSource sheetSource = new InputSource(sheetInputStream); try { XMLReader sheetParser = SAXHelper.newXMLReader(); ContentHandler handler = new XSSFSheetXMLHandler(styles, null, strings, sheetHandler, formatter, false); sheetParser.setContentHandler(handler); sheetParser.parse(sheetSource); } catch (ParserConfigurationException e) { throw new RuntimeException("SAX parser appears to be broken - " + e.getMessage()); } } /** * Initiates the processing of the XLS workbook file to CSV. * * @throws IOException * If reading the data from the package fails. * @throws SAXException * if parsing the XML data fails. */ public void process(String sheet) throws IOException, OpenXML4JException, SAXException { ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(this.xlsxPackage); XSSFReader xssfReader = new XSSFReader(this.xlsxPackage); StylesTable styles = xssfReader.getStylesTable(); XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader.getSheetsData(); while (iter.hasNext()) { try (InputStream stream = iter.next()) { String sheetName = iter.getSheetName(); if (!sheetName.equals(sheet)) { continue; } processSheet(styles, strings, new SheetToCSV(), stream); } } } }

    StreamingReader 로 읽 기
      public static Map>> readExcelByStreamingReader(String filePath, int num) {
            Workbook wk = null;
            Map>> metaInfo = null;
            try {
                wk = StreamingReader.builder().rowCacheSize(100) //          ,   10
                        .bufferSize(4096) //      ,          ,   1024
                        .open(new FileInputStream(filePath)); //     ,  ,   InputStream   File,  :    XLSX     
    
    //            wk = StreamingReader.builder()
    //                    .sstCacheSize(100)
    //                    .open(new FileInputStream(filePath)); //     ,  ,   InputStream   File,  :    XLSX     
    
                metaInfo = new LinkedHashMap<>();
                for (int i = 0; i < wk.getNumberOfSheets(); i++) {
                    Sheet sheet = wk.getSheetAt(i);
                    List> sample = new LinkedList<>();
                    for (Row row : sheet) {
                        //       
                        List rowData = new LinkedList<>();
                        for (Cell cell : row) {
                            rowData.add(cell.getStringCellValue() + "");
                        }
                        if (num>=0 && sample.size() < num) {
                            sample.add(rowData);
                            break;
                        }
                    }
                    metaInfo.put(sheet.getSheetName(), sample);
                }
            } catch (FileNotFoundException e) {
                throw new AsuraRuntimeException(e);
            } finally {
                try {
                    if (null != wk) {
                        wk.close();
                    }
                } catch (IOException e) {
                    wk = null;
                }
            }
            return metaInfo;
    
    
    import java.io.ByteArrayInputStream;
    import java.io.InputStream;
    import java.sql.Connection;
    import java.sql.PreparedStatement;
    import java.sql.SQLException;
    import java.util.List;
    import java.util.Map;
    
    import javax.servlet.http.HttpServletRequest;
    
    import org.apache.commons.lang3.StringUtils;
    
    /**
     * @author liaowei
     * @create 2018-05-03 16:46
     * @desc    load mysql
     * sql.csv:   
     **/
    
    public class LoadDataToMysqlUtils {
    
    	/**
    	 *           
    	 *
    	 * @param datas
    	 * @return String testSql = "LOAD DATA LOCAL INFILE '12.csv' IGNORE INTO TABLE
    	 *         test.test (a,b,c,d,e,f)";
    	 */
    	public static LoadDataToMysqlFieldsVo getDataInputStream(HttpServletRequest request,
    			List> datas, Map MysqlToExcelProp, String userName, String caseId) {
    		LoadDataToMysqlFieldsVo loadDataToMysqlFieldsVo = new LoadDataToMysqlFieldsVo();
    		String sql = "LOAD DATA LOCAL INFILE 'sql.csv' IGNORE INTO TABLE  " +    + " (";
    		StringBuilder builder = new StringBuilder();
    		String midSql = "";
    		request.getSession().setAttribute("TotalNumberOfData", datas.size());
    		for (int i = 0; i < datas.size(); i++) {
    			for (Map.Entry mapData : MysqlToExcelProp.entrySet()) {
    				if (i == 0) {
    					if (StringUtils.isEmpty(midSql)) {
    						midSql = mapData.getValue();
    					} else {
    						midSql += "," + mapData.getValue();
    					}
    				}
    				if ("member_register_date".equalsIgnoreCase(mapData.getValue())) {
    					if (StringUtils.isNotEmpty(datas.get(i).get(mapData.getKey()))) {
    						List convetTime = DateUtils.ArrayDateForm(datas.get(i).get(mapData.getKey()));
    						String date = convetTime.get(0) + "-" + convetTime.get(1) + "-" + convetTime.get(2) + " "
    								+ convetTime.get(3) + ":" + convetTime.get(4) + ":" + convetTime.get(5);
    						builder.append(date);
    					} else {
    						builder.append("");
    					}
    				} else {
    					if (StringUtils.isNotEmpty(datas.get(i).get(mapData.getKey()))) {
    						builder.append(datas.get(i).get(mapData.getKey()));
    					} else {
    						builder.append("");
    					}
    				}
    				builder.append("\t");
    			}
    
    			builder.append(caseId);
    			builder.append("\t");
    			builder.append(AsuraTimeUtils.currentDate_Time());
    			builder.append("\t");
    			builder.append(userName);
    			builder.append("\t");
    			builder.append(AsuraTimeUtils.currentDate_Time());
    			builder.append("\t");
    			builder.append(userName);
    			builder.append("\t");
    			builder.append("
    "); request.getSession().setAttribute("LoadedData", (long) (i + 1)); } sql += midSql + ",case_id,create_date,create_account,last_modify_time,last_modify_account)"; byte[] bytes = builder.toString().getBytes(); InputStream is = new ByteArrayInputStream(bytes); loadDataToMysqlFieldsVo.setSql(sql); loadDataToMysqlFieldsVo.setDataStream(is); return loadDataToMysqlFieldsVo; } /** * * * @param loadDataToMysqlFieldsVo */ public static void loadDataIntoMysql(LoadDataToMysqlFieldsVo loadDataToMysqlFieldsVo) { Connection conn = null; try { System.out.println(loadDataToMysqlFieldsVo.getSql()); conn = JdbcConnUtils.getConn(); PreparedStatement statement = conn.prepareStatement(loadDataToMysqlFieldsVo.getSql()); if (statement.isWrapperFor(com.mysql.jdbc.Statement.class)) { com.mysql.jdbc.PreparedStatement mysqlStatement = statement .unwrap(com.mysql.jdbc.PreparedStatement.class); mysqlStatement.setLocalInfileInputStream(loadDataToMysqlFieldsVo.getDataStream()); mysqlStatement.executeUpdate(); } } catch (Exception e) { throw new RuntimeException(" !" + e); } finally { try { if (null != conn) { conn.close(); } else { } } catch (SQLException e) { conn = null; } } } }

    데이터베이스 연결 방식
    import java.io.FileNotFoundException;
    import java.io.IOException;
    import java.io.InputStream;
    import java.sql.Connection;
    import java.sql.DriverManager;
    import java.sql.SQLException;
    import java.util.Properties;
    
    import org.springframework.util.ResourceUtils;
    
    /**
     * @author liaowei
     * @create 2017-11-06 10:36
     * @desc          
     */
    public class JdbcConnUtils {
    
    	private static Connection conn = null;
    
    	private static Properties props = null;
    
    	static {
    		loadProps();
    	}
    
    	synchronized static private void loadProps() {
    		props = new Properties();
    		InputStream in = null;
    		try {
    			//               properties   
    			in = ResourceUtils.class.getClassLoader().getResourceAsStream("project_config.properties");
    	
    			props.load(in);
    		} catch (FileNotFoundException e) {
    			throw new RuntimeException("          !");
    		} catch (IOException e) {
    			throw new RuntimeException("IO  !");
    		} finally {
    			try {
    				if (null != in) {
    					in.close();
    				}
    			} catch (IOException e) {
    				throw new RuntimeException("       !");
    			}
    		}
    	}
    
    	public static String getProperty(String key) {
    		if (null == props || props.size() <= 0) {
    			loadProps();
    		}
    		return props.getProperty(key);
    	}
    
    	public static Connection getConn() {
    		try {
    			Class.forName(getProperty("asura.db.driver"));
    			conn = DriverManager.getConnection(props.getProperty("asura.db.url"),
    					props.getProperty("asura.db.username"), props.getProperty("asura.db.pwd"));
    		} catch (ClassNotFoundException cnfe) {
    			throw new RuntimeException("      ");
    		} catch (SQLException e) {
    			throw new RuntimeException("      ");
    		}
    
    		return conn;
    	}
    
    	public void closeConn() {
    		try {
    			if (conn != null)
    				conn.close();
    		} catch (SQLException e) {
    			e.printStackTrace();
    		}
    
    	}
    
    	public static void main(String[] args) {
    		System.out.println(JdbcConnUtils.getProperty("asura.db.pwd"));
    	}
    }
    

    좋은 웹페이지 즐겨찾기