2.内存资源占用过大,基本上一个大数据量的导出会消耗可视化服务的所有资源,引起内存回收,其它接口无响应;
<!-- easyexcel --> <dependency> <groupId>com.alibaba</groupId> <artifactId>easyexcel</artifactId> <version>2.1.7</version> </dependency>三. 详细设计
ROW_SIZE:一次查询的数据量,此处设置为10000条 ROW_PAGE:一个页签多少次查询,此处设置为10次; private static Interger ROW_SIZE = 10000; private static interger ROW_PAGE = 10; /** * divide into sheets with 10W data per sheet * */ int sheetCount = (rowCount/ (ROW_SIZE*ROW_PAGE))+1; for(int i=0;i<sheetCount;i++){ threadExecutor.submit(()->{sheetWrite()}); }(2)sheet写入多线程,最后一个文件写入线程的最后一个sheet写入线程可能不足1W条数据;
// 单sheet页写入数 int sheetThreadCount = rowCount - (i+1)*(ROW_SIZE*ROW_PAGE) > 0 ? ROW_PAGE : (rowCount - i*(ROW_SIZE*ROW_PAGE))/ROW_SIZE+1; CountDownLatch threadSignal = new CountDownLatch(sheetThreadCount); for(int j=0;j<sheetThreadCount;j++) { threadExecutor.submit(()->{excelWriter()}); } threadSignal.await();(3)异步写入sheet文件,不同的文件写入线程写入不同的文件,所以只需要保证同一个文件写入线程下不同sheet写入线程的excelWriter异步即可;
// 获取数据 // todo // 数据格式处理 Synchronized(excelWriter){ WriteSheet writeSheet = EasyExcel.writerSheet(sheetNo, "第" + (sheetNo+1) + "页数据"); excelWriter.write(lists, writeSheet); }(4)压缩文件,将多个excel压缩成一个zip,最后上传至fast dfs,返回前端下载地址,使用hutool封装的ZipUtil方法;
// 堆代码 duidaima.com package cn.hutool.core.util; String[] paths = new String[10]; FileInputStream[] ins = new FileInputStream[10]; ZipUtil.zip(OutputStream out, String[] paths, InputStream[] ins); byte[] bytes = outputStream.toByteArray(); // 上传文件到FastDFS 返回上传路径 String path = fastWrapper.uploadFile(bytes, bytes.length, "zip"); return path + "?filename=" + fileName;四. 缓存
TableExport.java public String exportTable(ExportTable exportTable) throws Exception { StringBuffer path = new StringBuffer(); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); StringBuffer sign = new StringBuffer (); //redis key sign.append(exportTable.getId()); try { // 用来记录需要为 行 列设置样式 Map<String, Map<Integer, List<Map<Integer, ExportTable.ExportColumn.Font>>>> map = new HashMap<>(); sign.append("#").append(String.join(",", fields.stream().map(e-> e.isShow()?"true":"false").collect(Collectors.toList()))); setFontStyle(0, 0, exportTable.getFields(), map); // 获取表头长度 int headRow = head.stream().max(Comparator.comparingInt(List::size)).get().size(); // 数据量超过十万 则不带样式 // 只处理表头:表头合并 表头隐藏 表头冻结 if(rowCount*fields.size() > ROW_SIZE*6.4){ map.put("cellStyle", null); } sign.append("#").append(exportTable.getStyle()); // 数据量超过百万或者数据为空,只返回有表头得单元格 if(rowCount==0 || rowCount*fields.size() >= ROW_SIZE*1500){ EasyExcel.write(outputStream) // 这里放入动态头 .head(head).sheet("数据") // 传入表头样式 .registerWriteHandler(EasyExcelUtils.getStyleStrategy()) // 当然这里数据也可以用 List<List<String>> 去传入 .doWrite(new LinkedList<>()); byte[] bytes = outputStream.toByteArray(); // 上传文件到FastDFS 返回上传路径 return fastWrapper.uploadFile(bytes, bytes.length, "xlsx") + "?filename=" + fileName + ".xlsx"; } sign.append("#").append(rowCount); String fieldSign = fields.stream().sorted(Comparator.comparing(ExportTable.ExportColumn::getId)) .map(e->e.getId()).collect(Collectors.joining(",")); sign.append("#").append(fieldSign); /** * 相同的下载文件请求 直接返回 * the redis combines with datasetId - filter - size of data - fields */ if (redisClientImpl.hasKey(sign.toString())){ return redisClientImpl.get(sign.toString()).toString(); } /** * 分sheet页 * divide into sheets with 10M data per sheet */ int sheetCount = (rowCount/ (ROW_SIZE*ROW_PAGE))+1; String[] paths = new String[sheetCount]; ByteArrayInputStream[] ins = new ByteArrayInputStream[sheetCount]; CountDownLatch threadSignal = new CountDownLatch(sheetCount); for(int i=0;i<sheetCount;i++) { int finalI = i; String finalTable = table; Datasource finalDs = ds; String finalOrder = order; int finalRowCount = rowCount; threadExecutor.submit(()->{ // excel文件流 ByteArrayOutputStream singleOutputStream = new ByteArrayOutputStream(); ExcelWriter excelWriter = EasyExcel.write(singleOutputStream).build(); // 单sheet页写入数 int sheetThreadCount = finalI == (sheetCount-1) ? (finalRowCount - finalI *(ROW_SIZE*ROW_PAGE))/ROW_SIZE+1 : ROW_PAGE; CountDownLatch sheetThreadSignal = new CountDownLatch(sheetThreadCount); for(int j=0;j<sheetThreadCount;j++) { int page = finalI *ROW_PAGE + j + 1; // 最后一页数据 int pageSize = j==(sheetThreadCount-1)&& finalI ==(sheetCount-1) ? finalRowCount %ROW_SIZE : ROW_SIZE; threadExecutor.submit(()->{ try { writeExcel(dataSetTableRequest, datasetTable, finalTable, qp, datasetTableFields, exportTable, page, pageSize, finalDs, datasourceProvider, fieldArray, fields, head, map, headRow, excelWriter, mergeIndex, finalOrder); sheetThreadSignal.countDown(); } catch (Exception e) { e.printStackTrace(); } }); } try { sheetThreadSignal.await(); } catch (InterruptedException e) { e.printStackTrace(); } // 关闭写入流 excelWriter.finish(); paths[finalI] = (finalI +1) + "-" + fileName + ".xlsx"; // 单文件 if (sheetCount == 1){ // xlsx // 将sign存入redis并设置过期时间 } threadSignal.countDown(); }); } threadSignal.await(); if (sheetCount != 1){ ZipUtil.zip(outputStream, paths, ins); byte[] bytes = outputStream.toByteArray(); // 上传文件到FastDFS 返回上传路径 path.append(fastWrapper.uploadFile(bytes, bytes.length, "zip")) .append("?filename=").append(fileName).append(".zip"); // 将sign存入redis并设置过期时间 redisClientImpl.set(sign.toString(), path.toString(), SYS_REDIS_EXPIRE_TIME); } } catch (Exception e) { e.printStackTrace(); } return path.toString(); } private void writeExcel(ExcelWriter excelWriter){ //数据查询 // todo synchronized (excelWriter) { WriteSheet writeSheet = EasyExcel.writerSheet(0, "第" + 1 + "页数据") // 这里放入动态头 .head(head) //传入样式 .registerWriteHandler(EasyExcelUtils.getStyleStrategy()) .registerWriteHandler(new CellColorSheetWriteHandler(map, headRow)) .registerWriteHandler(new MergeStrategy(lists.size(), mergeIndex)) // 当然这里数据也可以用 List<List<String>> 去传入 .build(); excelWriter.write(lists, writeSheet); } }Excel导出的文件流样式处理类。
CellColorSheetWriteHandler.java import com.alibaba.excel.metadata.CellData; import com.alibaba.excel.metadata.Head; import com.alibaba.excel.util.StyleUtil; import com.alibaba.excel.write.handler.CellWriteHandler; import com.alibaba.excel.write.metadata.holder.WriteSheetHolder; import com.alibaba.excel.write.metadata.holder.WriteTableHolder; import com.alibaba.excel.write.metadata.style.WriteCellStyle; import org.apache.commons.lang3.StringUtils; import org.apache.poi.ss.usermodel.*; import org.apache.poi.xssf.usermodel.XSSFCellStyle; import org.apache.poi.xssf.usermodel.XSSFColor; import org.apache.poi.xssf.usermodel.XSSFFont; import java.awt.Color; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; /** * @Author 菜鸟 * @description 拦截处理单元格创建 */ public class CellColorSheetWriteHandler implements CellWriteHandler { /** * 多行表头行号 */ private int headRow; /** * 字体 */ private ExportTable.ExportColumn.Font columnFont = new ExportTable.ExportColumn.Font(); private static volatile XSSFCellStyle cellStyle = null; public static XSSFCellStyle getCellStyle(Workbook workbook, WriteCellStyle contentWriteCellStyle) { if(cellStyle == null) { synchronized (XSSFCellStyle.class) { if(cellStyle == null) { cellStyle =(XSSFCellStyle) StyleUtil.buildHeadCellStyle(workbook, contentWriteCellStyle); } } } return cellStyle; } /** * 字体 * Map<Integer, ExportTable.ExportColumn.Font> 当前列的字段样式 * Map<Integer, List<Map<...>>> 当前行包含那几列需要设置样式 * String head:表头; * String cell:内容; */ private Map<String, Map<Integer, List<Map<Integer, ExportTable.ExportColumn.Font>>>> map; /** * 有参构造 */ public CellColorSheetWriteHandler(Map<String, Map<Integer, List<Map<Integer, ExportTable.ExportColumn.Font>>>> map, int headRow) { this.map = map; this.headRow = headRow; } /** * 在单元上的所有操作完成后调用 */ @Override public void afterCellDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, List<CellData> cellDataList, Cell cell, Head head, Integer relativeRowIndex, Boolean isHead) { // 当前行的第column列 int column = cell.getColumnIndex(); // 当前第row行 int row = cell.getRowIndex(); AtomicInteger fixNum = new AtomicInteger(); // 处理行,表头 if (headRow > row && map.containsKey("head")){ Map<Integer, List<Map<Integer, ExportTable.ExportColumn.Font>>> fonts = map.get("head"); fonts.get(row).forEach(e->{ e.entrySet().forEach(ele -> { // 获取冻结字段 if (null != ele.getValue().getFixed() && !StringUtils.isEmpty(ele.getValue().getFixed())) { fixNum.getAndIncrement(); } // 字段隐藏 if(!ele.getValue().isShow()){ writeSheetHolder.getSheet().setColumnHidden(ele.getKey(), true); } }); }); if (fixNum.get() > 0 && row == 0) { writeSheetHolder.getSheet().createFreezePane(fixNum.get(), headRow, fixNum.get(), headRow); }else{ writeSheetHolder.getSheet().createFreezePane(0, headRow, 0, headRow); } setStyle(fonts, row, column, cell, writeSheetHolder, head); } // 处理内容 if (headRow <= row && map.containsKey("cell") && !map.containsKey("cellStyle")) { Map<Integer, List<Map<Integer, ExportTable.ExportColumn.Font>>> fonts = map.get("cell"); setStyle(fonts, -1, column, cell, writeSheetHolder, head); } } private void setStyle(Map<Integer, List<Map<Integer, ExportTable.ExportColumn.Font>>> fonts, int row, int column, Cell cell, WriteSheetHolder writeSheetHolder, Head head){ fonts.get(row).forEach(e->{ if (e.containsKey(column)){ // 根据单元格获取workbook Workbook workbook = cell.getSheet().getWorkbook(); //设置列宽 if(null != e.get(column).getWidth() && !e.get(column).getWidth().isEmpty()) { writeSheetHolder.getSheet().setColumnWidth(head.getColumnIndex(), Integer.parseInt(e.get(column).getWidth()) * 20); }else{ writeSheetHolder.getSheet().setColumnWidth(head.getColumnIndex(),2000); } // 单元格策略 WriteCellStyle contentWriteCellStyle = new WriteCellStyle(); // 设置垂直居中为居中对齐 contentWriteCellStyle.setVerticalAlignment(VerticalAlignment.CENTER); // 设置左右对齐方式 if(null != e.get(column).getAlign() && !e.get(column).getAlign().isEmpty()) { contentWriteCellStyle.setHorizontalAlignment(getHorizontalAlignment(e.get(column).getAlign())); }else{ contentWriteCellStyle.setHorizontalAlignment(HorizontalAlignment.LEFT); } if (!e.get(column).equal(columnFont) || column == 0){ /** * Prevent the creation of a large number of objects * Defects of the EasyExcel tool(巨坑,简直脱发神器) */ cellStyle = (XSSFCellStyle) StyleUtil.buildHeadCellStyle(workbook, contentWriteCellStyle); // 设置单元格背景颜色 if(null != e.get(column).getBackground() && !e.get(column).getBackground().isEmpty()) { cellStyle.setFillForegroundColor(new XSSFColor(hex2Color(e.get(column).getBackground()))); }else{ if(cell.getRowIndex() >= headRow) cellStyle.setFillForegroundColor(IndexedColors.WHITE.getIndex()); } // 创建字体实例 Font font = workbook.createFont(); // 设置字体是否加粗 if(null != e.get(column).getFontWeight() && !e.get(column).getFontWeight().isEmpty()) font.setBold(getBold(e.get(column).getFontWeight())); // 设置字体和大小 if(null != e.get(column).getFontFamily() && !e.get(column).getFontFamily().isEmpty()) font.setFontName(e.get(column).getFontFamily()); if(0 != e.get(column).getFontSize()) font.setFontHeightInPoints((short) e.get(column).getFontSize()); XSSFFont xssfFont = (XSSFFont)font; //设置字体颜色 if(null != e.get(column).getColor() && !e.get(column).getColor().isEmpty()) xssfFont.setColor(new XSSFColor(hex2Color(e.get(column).getColor()))); cellStyle.setFont(xssfFont); // 记录上一个样式 columnFont = e.get(column); } // // 设置当前行第column列的样式 cell.getRow().getCell(column).setCellStyle(cellStyle); // 设置行高 cell.getRow().setHeight((short) 400); } }); } }Excel导出的默认样式设置类。
EasyExcelUtils.java public static HorizontalCellStyleStrategy getStyleStrategy(){ // 头的策略 WriteCellStyle headWriteCellStyle = new WriteCellStyle(); // 背景设置为灰色 headWriteCellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); WriteFont headWriteFont = new WriteFont(); headWriteFont.setFontHeightInPoints((short)12); // 字体样式 headWriteFont.setFontName("Frozen"); // 字体颜色 headWriteFont.setColor(IndexedColors.BLACK1.getIndex()); headWriteCellStyle.setWriteFont(headWriteFont); // 自动换行 headWriteCellStyle.setWrapped(false); // 水平对齐方式(修改默认对齐方式——4.14 版本1.3.2) headWriteCellStyle.setHorizontalAlignment(HorizontalAlignment.LEFT); // 垂直对齐方式 headWriteCellStyle.setVerticalAlignment(VerticalAlignment.CENTER); // 内容的策略 WriteCellStyle contentWriteCellStyle = new WriteCellStyle(); // 这里需要指定 FillPatternType 为FillPatternType.SOLID_FOREGROUND 不然无法显示背景颜色.头默认了 FillPatternType所以可以不指定 // contentWriteCellStyle.setFillPatternType(FillPatternType.SQUARES); // 背景白色 contentWriteCellStyle.setFillForegroundColor(IndexedColors.WHITE.getIndex()); // 水平对齐方式(修改默认对齐方式——4.14 版本1.3.2) contentWriteCellStyle.setHorizontalAlignment(HorizontalAlignment.LEFT); WriteFont contentWriteFont = new WriteFont(); // 字体大小 contentWriteFont.setFontHeightInPoints((short)12); // 字体样式 contentWriteFont.setFontName("Calibri"); contentWriteCellStyle.setWriteFont(contentWriteFont); // 这个策略是 头是头的样式 内容是内容的样式 其他的策略可以自己实现 return new HorizontalCellStyleStrategy(headWriteCellStyle, contentWriteCellStyle); }Excel导出合并单元格处理类。
MergeStrategy.class import com.alibaba.excel.metadata.Head; import com.alibaba.excel.write.merge.AbstractMergeStrategy; import org.apache.commons.collections.map.HashedMap; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.util.CellRangeAddress; import java.util.*; /** * @Author 菜鸡 * @description 合并单元格策略 */ public class MergeStrategy extends AbstractMergeStrategy { /** * 合并的列编号,从0开始 * 指定的index或自己按字段顺序数 */ private Set<Integer> mergeCellIndex = new HashSet<>(); /** * 数据集大小,用于区别结束行位置 */ private Integer maxRow = 0; // 禁止无参声明 private MergeStrategy() { } public MergeStrategy(Integer maxRow, Set<Integer> mergeCellIndex) { this.mergeCellIndex = mergeCellIndex; this.maxRow = maxRow; } private Map<Integer, MergeRange> lastRow = new HashedMap(); @Override protected void merge(Sheet sheet, Cell cell, Head head, Integer relativeRowIndex) { int currentCellIndex = cell.getColumnIndex(); // 判断该行是否需要合并 if (mergeCellIndex.contains(currentCellIndex)) { String currentCellValue = cell.getStringCellValue(); int currentRowIndex = cell.getRowIndex(); if (!lastRow.containsKey(currentCellIndex)) { // 记录首行起始位置 lastRow.put(currentCellIndex, new MergeRange(currentCellValue, currentRowIndex, currentRowIndex, currentCellIndex, currentCellIndex)); return; } //有上行这列的值了,拿来对比. MergeRange mergeRange = lastRow.get(currentCellIndex); if (!(mergeRange.lastValue != null && mergeRange.lastValue.equals(currentCellValue))) { // 结束的位置触发下合并. // 同行同列不能合并,会抛异常 if (mergeRange.startRow != mergeRange.endRow || mergeRange.startCell != mergeRange.endCell) { sheet.addMergedRegionUnsafe(new CellRangeAddress(mergeRange.startRow, mergeRange.endRow, mergeRange.startCell, mergeRange.endCell)); } // 更新当前列起始位置 lastRow.put(currentCellIndex, new MergeRange(currentCellValue, currentRowIndex, currentRowIndex, currentCellIndex, currentCellIndex)); } // 合并行 + 1 mergeRange.endRow += 1; // 结束的位置触发下最后一次没完成的合并 if (relativeRowIndex.equals(maxRow - 1)) { MergeRange lastMergeRange = lastRow.get(currentCellIndex); // 同行同列不能合并,会抛异常 if (lastMergeRange.startRow != lastMergeRange.endRow || lastMergeRange.startCell != lastMergeRange.endCell) { sheet.addMergedRegionUnsafe(new CellRangeAddress(lastMergeRange.startRow, lastMergeRange.endRow, lastMergeRange.startCell, lastMergeRange.endCell)); } } } } } class MergeRange { public int startRow; public int endRow; public int startCell; public int endCell; public String lastValue; public MergeRange(String lastValue, int startRow, int endRow, int startCell, int endCell) { this.startRow = startRow; this.endRow = endRow; this.startCell = startCell; this.endCell = endCell; this.lastValue = lastValue; } }