当前位置:文档之家› POI 处理大数据量实现

POI 处理大数据量实现

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.openxml4j.opc.PackageAccess;
import https://www.doczj.com/doc/c87171115.html,ermodel.BuiltinFormats;
import https://www.doczj.com/doc/c87171115.html,ermodel.DataFormatter;
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.model.StylesTable;
import https://www.doczj.com/doc/c87171115.html,ermodel.XSSFCellStyle;
import https://www.doczj.com/doc/c87171115.html,ermodel.XSSFRichTextString;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;

/**
*
* @author Chris Lott
*/
public class CopyOfXLSX2CSV {

/**
* The type of the data value is indicated by an attribute on the cell. The
* value is usually in a "v" element within the cell.
*/
enum xssfDataType {
BOOL, ERROR, FORMULA, INLINESTR, SSTINDEX, NUMBER,
}

/**
* Derived from https://www.doczj.com/doc/c87171115.html,/spreadsheet/how-to.html#xssf_sax_api
*


* Also see Standard ECMA-376, 1st edition, part 4, pages 1928ff, at
* https://www.doczj.com/doc/c87171115.html,/publications/standards/Ecma-376.htm
*


* A web-friendly version is https://www.doczj.com/doc/c87171115.html,/Ecma/376/Part4
*/
class MyXSSFSheetHandler extends DefaultHandler {

/**
* Table with styles
*/
private StylesTable stylesTable;

/**
* Table with unique strings
*/
private ReadOnlySharedStringsTable sharedStringsTable;

/**
* Destination for data
*/
private final PrintStream output;

private List row;
private Queue rows;

/**
* Number of columns to read starting with leftmost
*/
private final int minColumnCount;

// Set when V start element is seen
private boolean vIsOpen;

// Set when cell start element is seen;
// used when cell close element is seen.
private xssfDataType nextDataType;

// Used to format numeric cell values.
private short formatIndex;
private String formatString;
private final DataFormatter formatter;

private int thisColumn = -1;
// The last column printed to the output stream
private int lastColumnNumber = -1;

// Gathers characters as they are seen.
private StringBuffer value;

/**
* Accepts objects needed while parsing.
*
* @param styles
* Table of styles
* @param strings
* Table of shared strings
* @param cols
* Min

imum number of columns to show
* @param target
* Sink for output
*/
public MyXSSFSheetHandler(StylesTable styles,
ReadOnlySharedStringsTable strings, int cols,
PrintStream target, List row, Queue rows) {
this.stylesTable = styles;
this.sharedStringsTable = strings;
this.minColumnCount = cols;
this.output = target;
this.value = new StringBuffer();
this.nextDataType = xssfDataType.NUMBER;
this.formatter = new DataFormatter();
this.rows = rows;
this.row = row;
}

/*
* (non-Javadoc)
*
* @see
* org.xml.sax.helpers.DefaultHandler#startElement(https://www.doczj.com/doc/c87171115.html,ng.String,
* https://www.doczj.com/doc/c87171115.html,ng.String, https://www.doczj.com/doc/c87171115.html,ng.String, org.xml.sax.Attributes)
*/
public void startElement(String uri, String localName, String name,
Attributes attributes) throws SAXException {

if ("inlineStr".equals(name) || "v".equals(name)) {
vIsOpen = true;
// Clear contents cache
value.setLength(0);
}
// c => cell
else if ("c".equals(name)) {
// Get the cell reference
String r = attributes.getValue("r");
int firstDigit = -1;
for (int c = 0; c < r.length(); ++c) {
if (Character.isDigit(r.charAt(c))) {
firstDigit = c;
break;
}
}
thisColumn = nameToColumn(r.substring(0, firstDigit));

// Set up defaults.
this.nextDataType = xssfDataType.NUMBER;
this.formatIndex = -1;
this.formatString = null;
String cellType = attributes.getValue("t");
String cellStyleStr = attributes.getValue("s");
if ("b".equals(cellType))
nextDataType = xssfDataType.BOOL;
else if ("e".equals(cellType))
nextDataType = xssfDataType.ERROR;
else if ("inlineStr".equals(cellType))
nextDataType = xssfDataType.INLINESTR;
else if ("s".equals(cellType))
nextDataType = xssfDataType.SSTINDEX;
else if ("str".equals(cellType))
nextDataType = xssfDataType.FORMULA;
else if (cellStyleStr != null) {
// It's a number, but almost certainly one
// with a special style or format
int styleIndex = Integer.parseInt(cellStyleStr);
XSSFCellStyle style = stylesTable.getStyleAt(styleIndex);
this.formatIndex = style.getDataFormat();
this.formatString = style.getDataFormatString();
if (this.formatString == null)
this.formatString = BuiltinFormats
.getBuiltinFormat(this.formatIndex);
}
}

}

/*
* (non-Javadoc)
*
* @see org.xml.sax.helpers.DefaultHandler#endElement(https://www.doczj.com/doc/c87171115.html,ng.String,
* https://www.doczj.com/doc/c87171115.html,ng.String, https://www.doczj.com/doc/c87171115.html,ng.String)
*/
public void endElement(String uri, String localName, String name)
throws SAXException {

String thisStr = null;
// v => contents of a cell
if ("v".equals(name)) {
// Process the value contents as required.
// Do now, as characters() may be called more than once
switch (nextDataType) {

case BOOL:
char first = val

ue.charAt(0);
thisStr = first == '0' ? "FALSE" : "TRUE";
break;

case ERROR:
thisStr = "\"ERROR:" + value.toString() + '"';
break;

case FORMULA:
// A formula could result in a string value,
// so always add double-quote characters.
thisStr = '"' + value.toString() + '"';
break;

case INLINESTR:
// TODO: have seen an example of this, so it's untested.
XSSFRichTextString rtsi = new XSSFRichTextString(
value.toString());
thisStr = '"' + rtsi.toString() + '"';
break;

case SSTINDEX:
String sstIndex = value.toString();
try {
int idx = Integer.parseInt(sstIndex);
XSSFRichTextString rtss = new XSSFRichTextString(
sharedStringsTable.getEntryAt(idx));
thisStr = '"' + rtss.toString() + '"';
} catch (NumberFormatException ex) {
output.println("Failed to parse SST index '" + sstIndex
+ "': " + ex.toString());
}
break;

case NUMBER:
String n = value.toString();
if (this.formatString != null)
thisStr = formatter.formatRawCellContents(
Double.parseDouble(n), this.formatIndex,
this.formatString);
else
thisStr = n;
break;

default:
thisStr = "(TODO: Unexpected type: " + nextDataType + ")";
break;
}

// Output after we've seen the string contents
// Emit commas for any fields that were missing on this row
if (lastColumnNumber == -1) {
lastColumnNumber = 0;
}

// Might be the empty string.
row.add(thisStr);

// Update column
if (thisColumn > -1)
lastColumnNumber = thisColumn;

} else if ("row".equals(name)) {

// Print out any missing commas if needed
if (minColumns > 0) {
// Columns are 0 based
if (lastColumnNumber == -1) {
lastColumnNumber = 0;
}
}

// We're onto a new row
lastColumnNumber = -1;
//Whether filter null row,if true
if (true && row != null && row.size() > 0) {
List dest = new ArrayList();
for(String one : row) {
dest.add(one);
}
rows.add(dest);
row.clear();
}
}

}

/**
* Captures characters only if a suitable element is open. Originally
* was just "v"; extended for inlineStr also.
*/
public void characters(char[] ch, int start, int length)
throws SAXException {
if (vIsOpen)
value.append(ch, start, length);
}

/**
* Converts an Excel column name like "C" to a zero-based index.
*
* @param name
* @return Index corresponding to the specified name
*/
private int nameToColumn(String name) {
int column = -1;
for (int i = 0; i < name.length(); ++i) {
int c = name.charAt(i);
column = (column + 1) * 26 + c - 'A';
}
return column;
}

}

// /////////////////////////////////////

private OPCPackage xlsxPackage;
pri

vate int minColumns;
private PrintStream output;
private List row = new ArrayList();
private Queue rows = new ConcurrentLinkedQueue();

class ThreadtFileInputExcel extends Thread {
Queue queue;
java.util.List> flows;

ThreadtFileInputExcel(java.util.Queue rows) {
this.queue = rows;
}

@Override
public void run() {
try {
leafThread();
} catch (Exception e) {
e.printStackTrace();
}
}

private void leafThread() {
// TODO Auto-generated method stub
do {
if (!queue.isEmpty()) {
List queueString = queue.poll();
// System.out.println("queueString:" + queueString);
// for(List list : queue){
for(String test: queueString){
System.out.println("test:"+test);
// }
}
}
} while (!queue.isEmpty());
}
}

ThreadtFileInputExcel tFileInputExcel = new ThreadtFileInputExcel(rows);

public void mainThread()
throws IOException, OpenXML4JException,
ParserConfigurationException, SAXException, InterruptedException {
File xlsxFile = new File("small-grid0.xlsx");
OPCPackage p = OPCPackage.open(xlsxFile.getPath(), PackageAccess.READ);
CopyOfXLSX2CSV xlsx2csv = new CopyOfXLSX2CSV(p, System.out, -1, row,
rows);
xlsx2csv.process();
tFileInputExcel.start();
}

/**
* Creates a new XLSX -> CSV converter
*
* @param pkg
* The XLSX package to process
* @param output
* The PrintStream to output the CSV to
* @param minColumns
* The minimum number of columns to output, or -1 for no minimum
*/
public CopyOfXLSX2CSV(OPCPackage pkg, PrintStream output, int minColumns,
List row, Queue rows) {
this.xlsxPackage = pkg;
this.output = output;
this.minColumns = minColumns;
this.row = row;
this.rows = rows;
}

public CopyOfXLSX2CSV() {
// TODO Auto-generated constructor stub
}

/**
* Parses and shows the content of one sheet using the specified styles and
* shared-strings tables.
*
* @param styles
* @param strings
* @param sheetInputStream
*/
public void processSheet(StylesTable styles,
ReadOnlySharedStringsTable strings, InputStream sheetInputStream)
throws IOException, ParserConfigurationException, SAXException {

InputSource sheetSource = new InputSource(sheetInputStream);
SAXParserFactory saxFactory = SAXParserFactory.newInstance();
SAXParser saxParser = saxFactory.newSAXParser();
XMLReader sheetParser = saxParser.getXMLReader();
ContentHandler handler = new MyXSSFSheetHandler(styles, strings,
this.minColumns, this.output, this.row, this.rows);
sheetParser.setContentHandler(handler);
sheetParser.parse(sheetSource);
}

/**
* Initiates the processing of the XLS workbook file to CSV.
*
* @throws IOException
* @throws OpenXML4JException
* @throws ParserConfigu

rationException
* @throws SAXException
*/
public void process() throws IOException, OpenXML4JException,
ParserConfigurationException, SAXException {

ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(
this.xlsxPackage);
XSSFReader xssfReader = new XSSFReader(this.xlsxPackage);
StylesTable styles = xssfReader.getStylesTable();
XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader
.getSheetsData();
int index = 0;
while (iter.hasNext()) {
InputStream stream = iter.next();
processSheet(styles, strings, stream);
stream.close();
++index;
}
}

public static void main(String[] args) throws Exception {

// The package open is instantaneous, as it should be.
CopyOfXLSX2CSV xlsx2csv = new CopyOfXLSX2CSV();
xlsx2csv.mainThread();
// List list = new ArrayList();
// list.add("1");
// list.add("1,23");

}

}

相关主题
文本预览
相关文档 最新文档