package com.shkpr.service.alambizplugin.components; import com.global.base.log.LogLevelFlag; import com.global.base.log.LogPrintMgr; import com.shkpr.service.alambizplugin.apiparam.GisSurveyThirdImportParams; import com.shkpr.service.alambizplugin.commtools.ExcelUtils; import com.shkpr.service.alambizplugin.commtools.PointCodeUtil; import com.shkpr.service.alambizplugin.components.checker.DuplicatePointsFinder; import com.shkpr.service.alambizplugin.components.checker.InvalidLinesFinder; import com.shkpr.service.alambizplugin.components.checker.InvalidPropertiesFinder; import com.shkpr.service.alambizplugin.constants.ExcelEnum; import com.shkpr.service.alambizplugin.constants.GisMetadataDefine; import com.shkpr.service.alambizplugin.constants.GisSurveyCheckStatusEnum; import com.shkpr.service.alambizplugin.constants.GisSurveyImportDefine; import com.shkpr.service.alambizplugin.constants.GisSurveyImportStatusEnum; import com.shkpr.service.alambizplugin.constants.LogFlagBusiType; import com.shkpr.service.alambizplugin.dbdao.services.intef.GisMetadataLayerTemplateService; import com.shkpr.service.alambizplugin.dbdao.services.intef.GisSurveyLayerApplyThirdCopyService; import com.shkpr.service.alambizplugin.dto.GisMetadataLayerTemplate; import com.shkpr.service.alambizplugin.dto.GisMetadataPropertyTemplate; import com.shkpr.service.alambizplugin.dto.GisSurveyLayerApplyThirdCopy; import com.shkpr.service.alambizplugin.dto.GisSurveyPropertyValueThirdCopy; import com.shkpr.service.alambizplugin.dto.GisSurveyThirdImportElement; import com.shkpr.service.alambizplugin.dto.GisSurveyThirdImportResult; import com.shkpr.service.alambizplugin.dto.GisSurveyThirdImportResultDetail; import com.shkpr.service.alambizplugin.dto.GisSurveyThirdImportSubtask; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.AsyncResult; import org.springframework.stereotype.Component; import org.springframework.util.concurrent.ListenableFuture; import java.io.IOException; import java.io.InputStream; import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.stream.Collectors; /** * 第三方导入执行器 * * @author 欧阳劲驰 * @since 1.0.0 */ @Component public class GisSurveyThirdImporter { /** * log */ private final String mStrClassName; private final String mBizType; private final GisMetadataLayerTemplateService layerTemplateService; private final GisSurveyLayerApplyThirdCopyService gisSurveyLayerApplyThirdCopyService; private final DuplicatePointsFinder duplicatePointsFinder; private final InvalidLinesFinder invalidLinesFinder; private final InvalidPropertiesFinder invalidPropertiesFinder; public GisSurveyThirdImporter(GisMetadataLayerTemplateService layerTemplateService, GisSurveyLayerApplyThirdCopyService gisSurveyLayerApplyThirdCopyService , DuplicatePointsFinder duplicatePointsFinder, InvalidLinesFinder invalidLinesFinder, InvalidPropertiesFinder invalidPropertiesFinder) { mStrClassName = "GisSurveyThirdImporter"; mBizType = LogFlagBusiType.BUSI_GIS_SURVEY.toStrValue(); this.layerTemplateService = layerTemplateService; this.gisSurveyLayerApplyThirdCopyService = gisSurveyLayerApplyThirdCopyService; this.duplicatePointsFinder = duplicatePointsFinder; this.invalidLinesFinder = invalidLinesFinder; this.invalidPropertiesFinder = invalidPropertiesFinder; } /** * 第三方导入任务 * * @param params 系统检查参数 * @param inputStreams 文件输入流 * @param onStartSubtask 启动子任务 * @param onDeprecatedSubtask 弃用子任务(完成数据收集/取消) * @return 系统检查返回 */ @Async public ListenableFuture thirdImportTask(GisSurveyThirdImportParams params, List inputStreams , Consumer onStartSubtask, Consumer onDeprecatedSubtask) { //构建返回 GisSurveyThirdImportResult result = GisSurveyThirdImportResult.fail(params); //无效属性任务 ListenableFuture>>> invalidPropertiesFuture = null; //重复点号任务 ListenableFuture>> duplicatePointsFuture = null; //无效线任务 ListenableFuture>> invalidLinesFuture = null; //点集合 List> points = new ArrayList<>(); //线集合 List> lines = new ArrayList<>(); try { LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_INFO, mBizType, mStrClassName , String.format( "开始执行第三方导入;任务id: %s" , params.getJobId() ) ); //解析excel parseExcel(inputStreams, points, lines); //提取图层名 List pointLayerNames = getLayerNames(points, GisSurveyImportDefine.POINT.LAYER); List lineLayerNames = getLayerNames(lines, GisSurveyImportDefine.LINE.LAYER); //图层模版 List pointLayerTemplates = layerTemplateService.findByNatureAndNameIn(params.getNature(), pointLayerNames); List lineLayerTemplates = layerTemplateService.findByNatureAndNameIn(params.getNature(), lineLayerNames); //无效属性检查 invalidPropertiesFuture = invalidPropertiesFinder.findInvalidProperties(points, lines, pointLayerTemplates, lineLayerTemplates); //重复点检查 duplicatePointsFuture = duplicatePointsFinder.findDuplicatePoints(points); //无效线检查 invalidLinesFuture = invalidLinesFinder.finderInvalidLines(points, lines); //返回子任务 onStartSubtask.accept( new GisSurveyThirdImportSubtask(invalidPropertiesFuture, duplicatePointsFuture, invalidLinesFuture) ); //等待结果 GisSurveyThirdImportResultDetail>> invalidPropertiesResult = invalidPropertiesFuture.get(); //存入无效属性结果 result.setInvalidLayersResult(untarInvalidProperties(invalidPropertiesResult, GisSurveyImportDefine.RESULT.INVALID_LAYERS)); result.setMissingRequirementsResult(untarInvalidProperties(invalidPropertiesResult, GisSurveyImportDefine.RESULT.MISSING_REQUIREMENTS)); result.setInvalidTypesResult(untarInvalidProperties(invalidPropertiesResult, GisSurveyImportDefine.RESULT.INVALID_TYPES)); result.setOutRangesResult(untarInvalidProperties(invalidPropertiesResult, GisSurveyImportDefine.RESULT.OUT_RANGES)); //存入重复点结果 result.setDuplicatePointsResult(duplicatePointsFuture.get()); //存入无效线结果 result.setInvalidLinesResult(invalidLinesFuture.get()); //结果未通过 if (!checkResult(result)) { result.setImportStatus(GisSurveyImportStatusEnum.DATA_ERROR.getCode()); result.setCompleteTime(LocalDateTime.now()); LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_INFO, mBizType, mStrClassName , String.format( "第三方导入结果未通过;任务id: %s, 用时(毫秒):%d", params.getJobId(), Duration.between(result.getRequestTime(), result.getCompleteTime()).toMillis() ) ); //弃用子任务 onDeprecatedSubtask.accept(params.getJobId()); return new AsyncResult<>(result); } //解码点线数据 List layerApplyList = decodeDataToLayerApply(points, lines, pointLayerTemplates, lineLayerTemplates, params); //批量写入 Boolean saved = gisSurveyLayerApplyThirdCopyService.saveAll(params.getJobId(), layerApplyList); if (!saved) { result.setCompleteTime(LocalDateTime.now()); LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_INFO, mBizType, mStrClassName , String.format( "第三方导入入库失败;任务id: %s, 用时(毫秒):%d", params.getJobId(), Duration.between(result.getRequestTime(), LocalDateTime.now()).toMillis() ) ); //弃用子任务 onDeprecatedSubtask.accept(params.getJobId()); return new AsyncResult<>(result); } //完成任务 result.setImportStatus(GisSurveyImportStatusEnum.SUCCESS.getCode()); result.setCompleteTime(LocalDateTime.now()); result.setPreviewData(layerApplyList); LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_INFO, mBizType, mStrClassName , String.format( "结束执行第三方导入;任务id: %s, 用时(毫秒):%d", params.getJobId(), Duration.between(result.getRequestTime(), result.getCompleteTime()).toMillis() ) ); //弃用子任务 onDeprecatedSubtask.accept(params.getJobId()); return new AsyncResult<>(result); } catch (InterruptedException | ExecutionException | IOException | RuntimeException e) { LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_ERROR, mBizType, mStrClassName , String.format( "监测到中断或执行异常,开始清除子任务 任务id:%s error:%s", params.getJobId(), e ) ); //清除子任务 if (invalidPropertiesFuture != null) invalidPropertiesFuture.cancel(true); if (duplicatePointsFuture != null) duplicatePointsFuture.cancel(true); if (invalidLinesFuture != null) invalidLinesFuture.cancel(true); //失败信息 result.setImportStatus(GisSurveyCheckStatusEnum.FAIL.getCode()); result.setCompleteTime(LocalDateTime.now()); //弃用子任务 onDeprecatedSubtask.accept(params.getJobId()); return new AsyncResult<>(result); } } /** * 解析excel * * @param inputStreams 输入流 * @param points 点集合 * @param lines 线集合 * @throws IOException io异常 */ private void parseExcel(List inputStreams, List> points, List> lines) throws IOException, InterruptedException { LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_INFO, mBizType, mStrClassName, "开始执行解析excel ======>"); long begin = System.currentTimeMillis(); //解析文件 for (InputStream inputStream : inputStreams) { //检查线程中断,并响应 if (Thread.interrupted()) throw new InterruptedException(); //解析excel Map>> excelData = ExcelUtils.parseExcelFile(inputStream, ExcelEnum.XLSX, 0, 2); //填入集合 if (excelData != null && excelData.containsKey(GisMetadataDefine.TYPE_KINE.POINT)) points.addAll(excelData.get(GisMetadataDefine.TYPE_KINE.POINT)); if (excelData != null && excelData.containsKey(GisMetadataDefine.TYPE_KINE.LINE)) lines.addAll(excelData.get(GisMetadataDefine.TYPE_KINE.LINE)); //释放流 inputStream.close(); } long end = System.currentTimeMillis(); LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_INFO, mBizType, mStrClassName , String.format( "结束执行执行解析excel,用时(毫秒):%d" , (end - begin) ) ); } /** * 获取图层名称 * * @param datas 数据 * @param layerKey 图层的key * @return 图层名称集合 */ private List getLayerNames(List> datas, String layerKey) { //去重图层 return datas.parallelStream() .filter(data -> data.get(layerKey) != null) .map(data -> data.get(layerKey)) .distinct() .collect(Collectors.toList()); } /** * 解包无效属性 * * @param invalidPropertiesResult 无效属性结果 * @param resultKey 结果key * @return 具体结果 */ private GisSurveyThirdImportResultDetail> untarInvalidProperties( GisSurveyThirdImportResultDetail>> invalidPropertiesResult , String resultKey) { //获取结果 List importElements = invalidPropertiesResult.getResults().get(resultKey); return new GisSurveyThirdImportResultDetail<>(true, importElements); } /** * 检查结果 * * @param result 结果 * @return 结果通过状态 */ private Boolean checkResult(GisSurveyThirdImportResult result) { List invalidLayers = result.getInvalidLayersResult().getResults(); List missingRequirements = result.getMissingRequirementsResult().getResults(); List invalidTypesResult = result.getInvalidTypesResult().getResults(); List outRanges = result.getOutRangesResult().getResults(); List duplicatePoints = result.getDuplicatePointsResult().getResults(); List invalidLines = result.getInvalidLinesResult().getResults(); //检查是否都为空 return CollectionUtils.isEmpty(invalidLayers) && CollectionUtils.isEmpty(missingRequirements) && CollectionUtils.isEmpty(invalidTypesResult) && CollectionUtils.isEmpty(outRanges) && CollectionUtils.isEmpty(duplicatePoints) && CollectionUtils.isEmpty(invalidLines); } /** * 解码数据到采集元素对象 *

从Map转为 {@code GisSurveyLayerApplyThirdCopy}

* * @param points 点数据 * @param lines 线数据 * @param pointLayerTemplates 点图层模版 * @param lineLayerTemplates 线图层模版 * @param params 导入参数 * @return 格式化后数据 */ private List decodeDataToLayerApply(List> points, List> lines , List pointLayerTemplates, List lineLayerTemplates, GisSurveyThirdImportParams params) { LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_INFO, mBizType, mStrClassName, "导入结果通过,开始解码点线数据 ======>"); long begin = System.currentTimeMillis(); //按模版名分组,便于快速找到模版 Map poineTemplateMap = pointLayerTemplates.stream() .collect(Collectors.toMap( GisMetadataLayerTemplate::getName, it -> it, (it1, it2) -> it1 )); Map lineTemplateMap = lineLayerTemplates.stream() .collect(Collectors.toMap( GisMetadataLayerTemplate::getName, it -> it, (it1, it2) -> it1 )); //采集元素象集合 List result = new ArrayList<>(); //点号映射,用于线写入上下游节点 Map pointNoMapping = new HashMap<>(); //格式化点 for (Map point : points) { //获取模版 GisMetadataLayerTemplate layerTemplate = poineTemplateMap.get(point.get(GisSurveyImportDefine.POINT.LAYER)); //解码点对象 GisSurveyLayerApplyThirdCopy layerApply = decodePointToLayerApply(point, layerTemplate, params); //存入点号 pointNoMapping.put(layerApply.getNo(), layerApply.getCode()); result.add(layerApply); } //格式化线 for (Map line : lines) { //获取模版 GisMetadataLayerTemplate layerTemplate = lineTemplateMap.get(line.get(GisSurveyImportDefine.LINE.LAYER)); //解码对象 GisSurveyLayerApplyThirdCopy layerApply = decodeLineToLayerApply(line, layerTemplate, params, pointNoMapping); result.add(layerApply); } long end = System.currentTimeMillis(); LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_INFO, mBizType, mStrClassName , String.format( "结束执行解码点线数据,用时(毫秒):%d" , (end - begin) ) ); return result; } /** * 解码点数据到采集元素对象 * * @param point 点数据 * @param layerTemplate 图层模版 * @param params 导入参数 * @return 采集元素拷贝对象 */ private GisSurveyLayerApplyThirdCopy decodePointToLayerApply(Map point, GisMetadataLayerTemplate layerTemplate, GisSurveyThirdImportParams params) { GisSurveyLayerApplyThirdCopy layerApply = new GisSurveyLayerApplyThirdCopy(); //必填项 layerApply.setJobId(params.getJobId()); layerApply.setCode(PointCodeUtil.generateCode()); layerApply.setLayer(layerTemplate.getKey()); layerApply.setKind(GisMetadataDefine.TYPE_KINE.POINT); //解析坐标 double[] coordinate = {Double.parseDouble(point.get(GisSurveyImportDefine.POINT.LNG)), Double.parseDouble(point.get(GisSurveyImportDefine.POINT.LAT))}; layerApply.setGis(Arrays.toString(coordinate)); //默认值 layerApply.setApply(GisSurveyImportDefine.DEFAULT_VALUE.APPLY); layerApply.setSource(GisSurveyImportDefine.DEFAULT_VALUE.SOURCE); //高程和埋深 String elevationStr = point.get(GisSurveyImportDefine.POINT.ELEVATION); if (NumberUtils.isParsable(elevationStr)) layerApply.setElevation(Double.parseDouble(elevationStr)); String depthStr = point.get(GisSurveyImportDefine.POINT.DEPTH); if (NumberUtils.isParsable(depthStr)) layerApply.setDepth(Double.parseDouble(depthStr)); //点号(长度64截断) layerApply.setNo(StringUtils.substring(point.get(GisSurveyImportDefine.POINT.NO), 0, 64)); //遍历属性模版 List propertyValueList = new ArrayList<>(); for (GisMetadataPropertyTemplate propertyTemplate : layerTemplate.getPropertyTemplates()) { //解析属性 GisSurveyPropertyValueThirdCopy propertyValue = decodeDataToPropertyValue(point, layerApply , propertyTemplate, params, GisSurveyImportDefine.LayerType.POINT); if (propertyValue != null) propertyValueList.add(propertyValue); } layerApply.setPropertyValueList(propertyValueList); return layerApply; } /** * 解码线数据到采集元素对象 * * @param line 线数据 * @param layerTemplate 图层模版 * @param params 导入参数 * @param pointNoMapping 点号映射 * @return 采集元素拷贝对象 */ private GisSurveyLayerApplyThirdCopy decodeLineToLayerApply(Map line, GisMetadataLayerTemplate layerTemplate, GisSurveyThirdImportParams params, Map pointNoMapping) { GisSurveyLayerApplyThirdCopy layerApply = new GisSurveyLayerApplyThirdCopy(); //必填项 layerApply.setJobId(params.getJobId()); layerApply.setCode(PointCodeUtil.generateCode()); layerApply.setLayer(layerTemplate.getKey()); layerApply.setKind(GisMetadataDefine.TYPE_KINE.LINE); //默认值 layerApply.setApply(GisSurveyImportDefine.DEFAULT_VALUE.APPLY); layerApply.setSource(GisSurveyImportDefine.DEFAULT_VALUE.SOURCE); //上下游节点 layerApply.setUpNode(pointNoMapping.get(line.get(GisSurveyImportDefine.LINE.UP_NO))); layerApply.setDownNode(pointNoMapping.get(line.get(GisSurveyImportDefine.LINE.DOWN_NO))); //遍历属性模版 List propertyValueList = new ArrayList<>(); for (GisMetadataPropertyTemplate propertyTemplate : layerTemplate.getPropertyTemplates()) { //解析属性 GisSurveyPropertyValueThirdCopy propertyValue = decodeDataToPropertyValue(line, layerApply, propertyTemplate, params, GisSurveyImportDefine.LayerType.LINE); if (propertyValue != null) propertyValueList.add(propertyValue); } layerApply.setPropertyValueList(propertyValueList); return layerApply; } /** * 解码数据到属性值对象 * * @param data 数据 * @param layerApply 采集元素 * @param propertyTemplate 属性模版 * @param params 导出参数 * @param layerType 图层类型 * @return 属性值对象 */ private GisSurveyPropertyValueThirdCopy decodeDataToPropertyValue(Map data, GisSurveyLayerApplyThirdCopy layerApply , GisMetadataPropertyTemplate propertyTemplate, GisSurveyThirdImportParams params, GisSurveyImportDefine.LayerType layerType) { //点号直接返回点号属性 if (Objects.equals(GisSurveyImportDefine.TEMPLATE.CODE, propertyTemplate.getKey())) return new GisSurveyPropertyValueThirdCopy(params.getJobId(), layerApply.getCode() , propertyTemplate.getKey(), layerApply.getCode()); //根据图层类型和模版映射,获取表头 String templateName = propertyTemplate.getName(); if (layerType == GisSurveyImportDefine.LayerType.POINT) switch (propertyTemplate.getKey()) { case GisSurveyImportDefine.TEMPLATE.LNG: templateName = GisSurveyImportDefine.POINT.LNG; break; case GisSurveyImportDefine.TEMPLATE.LAT: templateName = GisSurveyImportDefine.POINT.LAT; break; } if (layerType == GisSurveyImportDefine.LayerType.LINE) switch (propertyTemplate.getKey()) { case GisSurveyImportDefine.TEMPLATE.UP_NODE: templateName = GisSurveyImportDefine.LINE.UP_NO; break; case GisSurveyImportDefine.TEMPLATE.DOWN_NODE: templateName = GisSurveyImportDefine.LINE.DOWN_NO; break; } //获取值 String value = data.get(templateName); //构建dto,并将值长度64截断 if (StringUtils.isNotBlank(value)) return new GisSurveyPropertyValueThirdCopy(params.getJobId() , layerApply.getCode(), propertyTemplate.getKey(), StringUtils.substring(value, 0, 64)); return null; } }