EasyExcel多线程批量导出数据，动态表头
2023-12-27 16:26:35
1.导入依赖
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>3.1.1</version>
</dependency>
2.建立实体
@Data
public class ActResultLogVO implements Serializable {
private static final long serialVersionUID = 1L;
@ExcelProperty(value = "onlineseqid",index = 0)
private String onlineseqid;
@ExcelProperty(value = "businessid",index = 1)
private String businessid;
@ExcelProperty(value = "becifno",index = 2)
private String becifno;
@ExcelProperty(value = "ivisresult",index = 3)
private String ivisresult;
@ExcelProperty(value = "createdby",index = 4)
private String createdby;
@ExcelProperty(value = "createddate",index = 5)
private LocalDate createddate;
@ExcelProperty(value = "updateby",index = 6)
private String updateby;
@ExcelProperty(value = "updateddate",index = 7)
private LocalDate updateddate;
@ExcelProperty(value = "risklevel",index = 8)
private String risklevel;
/**
* 分页开始数量
*/
private Integer start;
/**
* 分页 size
*/
private Integer size;
}
3.多线程异步导出
@Async
@Transactional
public void exportExcel(String filePath) {
//filePath 文件路径
ExcelWriter writer = EasyExcel.write(filePath, ActResultLogVO.class).build();
// 根据数据读写速度来调整，一般来说读的逻辑复杂，比较慢，如果读比写快，这里设为1
int N = 2;
// 大小设置为2就可以，作为缓冲
BlockingQueue<List<ActResultLogVO>> queue = new ArrayBlockingQueue<>(2);
AtomicInteger start = new AtomicInteger(0);
AtomicInteger num = new AtomicInteger(0);
AtomicInteger sheet = new AtomicInteger(1);
// 分页大小可以适当调整
int pageSize = 100000;
//开启多个线程分页查数据
for (int i = 0; i < N; i++) {
executorService.submit(() -> {
while (true) {
//自增
int startNum= start.getAndAdd(pageSize);
try {
long l = System.currentTimeMillis();
logger.info("[多线程，分页查询] 线程：{}，开始执行查询：startNum :{},时间：{}",Thread.currentThread().getName(),startNum,l);
List<ActResultLogVO> list = selectActResultLogVOPage(startNum, pageSize);
logger.info("[多线程，分页查询] 线程：{}，执行查询完成 startNum :{} 用时：{}",Thread.currentThread().getName(),startNum,System.currentTimeMillis()-l);
if (CollectionUtils.isEmpty(list)) {
//读到没数据也要放入空集合
queue.put(Collections.EMPTY_LIST);
break;
}
queue.put(list);
} catch (Exception e) {
//异常情况也要放入空集合，防止写线程无法退出循环
try {
queue.put(Collections.EMPTY_LIST);
} catch (InterruptedException ex) {
throw new RuntimeException(ex);
}
throw new RuntimeException(e);
}
}
});
}
Future<?> submit = executorService.submit(() -> {
int count = 0;
while (true) {
List<ActResultLogVO> list = null;
try {
list = queue.take();
} catch (InterruptedException e) {
Thread.interrupted();
}
if (CollectionUtils.isEmpty(list)) {
count++;
// 当获取到两次空集合时，说明已经读完
if (count == N) {
break;
}
continue;
}
if(num.get()>=500000){
sheet.getAndAdd(1);
num.getAndSet(0);
}
num.getAndAdd(list.size());
long l = System.currentTimeMillis();
logger.info("[多线程，写入Excel] 线程：{}，开始执行写入：sheet :{},num:{}时间：{}",Thread.currentThread().getName(),sheet.get(),num.get(),l);
writer.write(list, EasyExcel.writerSheet("Sheet"+sheet).build());
logger.info("[多线程，写入Excel] 线程：{}，开始执行写入结束：sheet :{},num:{},时间：{}",Thread.currentThread().getName(),sheet.get(),num.get(),System.currentTimeMillis()-l);
}
writer.finish();
});
try {
// 阻塞等待完成，异步处理也可以去掉这段代码
submit.get();
} catch (Exception e) {
}
}
4.动态表头
@Autowired
@Qualifier("excelThreadPool")
private ExecutorService executorService;
@Async
@Transactional
public void exportExcel（String filePath,LinkedHashMap<String, DynamicExcelData> nameMap) {
// 指定写入的文件
ExcelWriter writer = EasyExcel.write(filePath).head(ExcelUtils.getHead(nameMap)).build();
// 根据数据读写速度来调整，一般来说读的逻辑复杂，比较慢，如果读比写快，这里设为1
int N = 2;
// 大小设置为2就可以，作为缓冲
BlockingQueue<List<Map<String,Object>>> queue = new ArrayBlockingQueue<>(2);
AtomicInteger num = new AtomicInteger(0);
AtomicInteger sheet = new AtomicInteger(1);
// 分页大小可以适当调整
int pageSize = ExcelConstants.SELECT_TO_DB_ROWS_MYBATIS;
//开启多个线程分页查数据
getData(lvnengAllCardBillVo,queue,pageSize);
Future<?> submit = executorService.submit(() -> {
int count = 0;
while (true) {
List<Map<String,Object>> list = null;
try {
list = queue.take();
} catch (InterruptedException e) {
Thread.interrupted();
}
if (CollectionUtils.isEmpty(list)) {
count++;
// 当获取到两次空集合时，说明已经读完
if (count == N) {
break;
}
continue;
}
if(num.get()>=600000){
sheet.getAndAdd(1);
num.getAndSet(0);
}
num.getAndAdd(list.size());
List<List<String>> dataList = ExcelUtils.getDataList(nameMap, list);
writer.write(dataList, EasyExcel.writerSheet("Sheet"+sheet).build());
}
writer.finish();
});
try {
// 阻塞等待完成，异步处理也可以去掉这段代码
submit.get();
} catch (Exception e) {
}
}
public void getData(ActResultLogVO actResultLogVO,BlockingQueue<List<Map<String,Object>>> queue,int pageSize) {
// 根据数据读写速度来调整，一般来说读的逻辑复杂，比较慢，如果读比写快，这里设为1
int N = 2;
AtomicInteger start = new AtomicInteger(0);
for (int i = 0; i < N; i++) {
executorService.submit(() -> {
while (true) {
//自增
int startNum = start.getAndAdd(pageSize);
List<Map<String, Object>> list = new ArrayList<>();
try {
ActResultLogVO actResultLogVO1 = DeepCopyUtil.deepCopy(actResultLogVO);
actResultLogVO1.setStart(startNum);
List<Map<String, Object>> list = selectactResultLogVOPage(actResultLogVO1);
if (CollectionUtils.isEmpty(list)) {
//读到没数据也要放入空集合
queue.put(Collections.EMPTY_LIST);
break;
}
queue.put(list);
} catch (Exception e) {
//异常情况也要放入空集合，防止写线程无法退出循环
try {
queue.put(Collections.EMPTY_LIST);
} catch (InterruptedException ex) {
throw new RuntimeException(ex);
}
throw new RuntimeException(e);
}
}
});
}
logger.info("查询数据完成");
}
@Configuration
public class ThreadPoolConfig {
@Bean("excelThreadPool")
public ExecutorService buildExcelThreadPool() {
int cpuNum = Runtime.getRuntime().availableProcessors();
BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(1000);
ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("excel-pool-%d").build();
return new ThreadPoolExecutor(10 * cpuNum, 30 * cpuNum,
1, TimeUnit.MINUTES, workQueue, threadFactory);
}
}
public class ExcelConstants {
public static final Integer PER_SHEET_ROW_COUNT = 100*10000;
public static final Integer PER_WRITE_ROW_COUNT = 20*10000;
public static final Integer SHEET_WRITE_ROW_COUNT = 50*10000;
public static final Integer SELECT_TO_DB_ROWS = 10*10000;
public static final Integer SELECT_TO_DB_ROWS_MYBATIS = 5*10000;
}
5,用到的util
5.1 EasyUtils
public class ExcelUtils {
public static void dynamicExportByURL(String filePath,
LinkedHashMap<String, DynamicExcelData> nameMap,
List<Map<String, Object>> list,
String sheetName) throws IOException {
//首先判断是否有数据，没有就返回
if(CollUtil.isEmpty(list)){
return;
}
//这里的map使用LinkedHashMap，实现字段的顺序功能
if(nameMap==null){
throw new RuntimeException("请填写好映射表数据");
}
File file = getFile(filePath);
dynamicExportByURL(file,nameMap,list,sheetName);
}
public static void dynamicExport(HttpServletResponse response,
LinkedHashMap<String, DynamicExcelData> nameMap,
List<Map<String, Object>> list,
String sheetName) throws IOException {
//首先判断是否有数据，没有就返回
if(CollUtil.isEmpty(list)){
return;
}
//这里的map使用LinkedHashMap，实现字段的顺序功能
if(nameMap==null){
throw new RuntimeException("请填写好映射表数据");
}
response.setHeader("Access-Control-Expose-Headers", "Content-Disposition");
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
response.setCharacterEncoding("utf-8");
//使用EasyExcel write进行写出
EasyExcel.write(response.getOutputStream()).head(getHead(nameMap)).sheet(sheetName).doWrite(getDataList(nameMap,list));
}
public static ArrayList<List<String>> getHead(LinkedHashMap<String, DynamicExcelData> nameMap) {
//获取表头
ArrayList<List<String>> head = new ArrayList<>();
for (Map.Entry<String, DynamicExcelData> titleMap : nameMap.entrySet()) {
DynamicExcelData data = titleMap.getValue();
head.add(Collections.singletonList(data.getName()));
}
return head;
}
public static List<List<String>> getDataList(LinkedHashMap<String, DynamicExcelData> nameMap, List<Map<String, Object>> list) {
//先初始化一下传入
int size = list.size();
List<List<String>> dataList = new ArrayList<>();
for (int i = 0; i < size; i++) {
dataList.add(new ArrayList<>());
}
//数据重组
for (int i = 0; i < list.size(); i++) {
Map<String, Object> map = list.get(i);
List<String> columns = dataList.get(i);
for (Map.Entry<String, DynamicExcelData> sortNameEntry : nameMap.entrySet()) {
String key = sortNameEntry.getKey();
Object value = map.get(key);
columns.add(value != null ? String.valueOf(value) : sortNameEntry.getValue().getDefaultValue());
}
}
return dataList;
}
public static File getFile(String filePath) throws IOException {
File file = new File(filePath);
if (!file.exists()) {
file.createNewFile();
}
return file;
}
public static void dynamicExportByURL(File file,LinkedHashMap<String, DynamicExcelData> nameMap, List date,String sheetName) {
long startTime = 0;
ExcelWriter excelWriter = null;
try {
startTime = System.currentTimeMillis();
// 获取 sheet 的个数
int sheetNum = getSheetNum(date.size());
// 获取每个sheet 写入的次数
int writeNumPerSheet = getWriteNumPerSheet();
// 最后一个 sheet 写入的数量
int writeNumLastSheet = date.size() - (sheetNum - 1) * ExcelConstants.PER_SHEET_ROW_COUNT;
// 最后一个 sheet 写入的次数
int writeNumPerLastSheet = writeNumLastSheet % ExcelConstants.PER_WRITE_ROW_COUNT == 0 ? writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT : writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT + 1;
// 指定写入的文件
excelWriter = EasyExcel.write(file).head(getHead(nameMap)).build();
for (int i = 0; i < sheetNum; i++) {
String sheet = sheetName + i;
WriteSheet writeSheet = EasyExcel.writerSheet(i, sheet).build();
int writeNum = i == sheetNum - 1 ? writeNumPerLastSheet : writeNumPerSheet; // 每个sheet 写入的次数
int endEndNum = i == sheetNum - 1 ? date.size() : (i + 1) * ExcelConstants.PER_SHEET_ROW_COUNT; // 每个sheet 最后一次写入的最后行数
for (int j = 0; j < writeNum; j++) {
long l = System.currentTimeMillis();
int startNum = i * ExcelConstants.PER_SHEET_ROW_COUNT + j * ExcelConstants.PER_WRITE_ROW_COUNT;
int endNum = j == writeNum - 1 ? endEndNum : i * ExcelConstants.PER_SHEET_ROW_COUNT + (j + 1) * ExcelConstants.PER_WRITE_ROW_COUNT;
excelWriter.write(getDataList(nameMap,date.subList(startNum, endNum)), writeSheet);
}
}
} catch (Exception e) {
} finally {
// 需要放入 finally 中
if (excelWriter != null) {
excelWriter.finish();
}
}
}
public static void writeExcel(String filePath, Class clazz, List date,String sheetName) throws Exception{
File file = getFile(filePath);
long startTime = System.currentTimeMillis();
// 获取 sheet 的个数
int sheetNum = getSheetNum(date.size());
// 获取每个sheet 写入的次数
int writeNumPerSheet = getWriteNumPerSheet();
// 最后一个 sheet 写入的数量
int writeNumLastSheet = date.size() - (sheetNum - 1) * ExcelConstants.PER_SHEET_ROW_COUNT;
// 最后一个 sheet 写入的次数
int writeNumPerLastSheet = writeNumLastSheet % ExcelConstants.PER_WRITE_ROW_COUNT == 0 ? writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT : writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT + 1;
// 指定写入的文件
ExcelWriter excelWriter = EasyExcel.write(file, clazz).build();
for (int i = 0; i < sheetNum; i++) {
String sheet = sheetName + i;
WriteSheet writeSheet = EasyExcel.writerSheet(i, sheet).build();
int writeNum = i == sheetNum - 1 ? writeNumPerLastSheet : writeNumPerSheet; // 每个sheet 写入的次数
int endEndNum = i == sheetNum - 1 ? date.size() : (i + 1) * ExcelConstants.PER_SHEET_ROW_COUNT; // 每个sheet 最后一次写入的最后行数
for (int j = 0; j < writeNum; j++) {
long l = System.currentTimeMillis();
int startNum = i * ExcelConstants.PER_SHEET_ROW_COUNT + j * ExcelConstants.PER_WRITE_ROW_COUNT;
int endNum = j == writeNum - 1 ? endEndNum : i * ExcelConstants.PER_SHEET_ROW_COUNT + (j + 1) * ExcelConstants.PER_WRITE_ROW_COUNT;
excelWriter.write(date.subList(startNum, endNum), writeSheet);
}
}
// 需要放入 finally 中
if (excelWriter != null) {
excelWriter.finish();
}
}
public static int getSheetNum(int dateSize){
return dateSize % ExcelConstants.PER_SHEET_ROW_COUNT == 0 ? (dateSize / ExcelConstants.PER_SHEET_ROW_COUNT) : (dateSize / ExcelConstants.PER_SHEET_ROW_COUNT + 1);
}
public static int getWriteNumPerSheet(){
return ExcelConstants.PER_SHEET_ROW_COUNT % ExcelConstants.PER_WRITE_ROW_COUNT == 0 ? (ExcelConstants.PER_SHEET_ROW_COUNT / ExcelConstants.PER_WRITE_ROW_COUNT) : (ExcelConstants.PER_SHEET_ROW_COUNT / ExcelConstants.PER_WRITE_ROW_COUNT + 1);
}
public static List<String> objectToList(Object obj){
ArrayList<String> list = new ArrayList<>();
//获取obj类中的所有字段
Field[] fields = obj.getClass().getDeclaredFields();
//遍历所有属性
for (Field field : fields) {
try {
field.setAccessible(true);
list.add(field.get(obj).toString());
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
return list;
}
public static Map<String,Object> objectToMap(Object obj){
HashMap<String, Object> map = new HashMap<>();
//获取obj类中的所有字段
Field[] fields = obj.getClass().getDeclaredFields();
//遍历所有属性
for (Field field : fields) {
try {
field.setAccessible(true);
map.put(field.getName(),field.get(obj));
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
return map;
}
/**
* 通过反射方式将头部作为公共部分进行设置
*/
public static List<List<String>> setTitles(Class clazz) {
List<List<String>> titles = new ArrayList<List<String>>();
for (Field declaredField : clazz.getDeclaredFields()) {
ExcelProperty annotation = declaredField.getAnnotation(ExcelProperty.class);
if (null != annotation) {
titles.add(Arrays.asList(annotation.value()));
}
}
return titles;
}
}
5.2 深拷贝
public class DeepCopyUtil {
public static <T extends Serializable> T deepCopy(T object) {
try {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ObjectOutputStream objectOutputStream = new ObjectOutputStream(outputStream);
objectOutputStream.writeObject(object);
ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
ObjectInputStream objectInputStream = new ObjectInputStream(inputStream);
return (T) objectInputStream.readObject();
} catch (IOException | ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
}
文章来源:https://blog.csdn.net/ai_haibin/article/details/135245524
本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。 如若内容造成侵权/违法违规/事实不符，请联系我的编程经验分享网邮箱：veading@qq.com进行投诉反馈，一经查实，立即删除！
本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。 如若内容造成侵权/违法违规/事实不符，请联系我的编程经验分享网邮箱：veading@qq.com进行投诉反馈，一经查实，立即删除！