数据迁移排除空数据和空传感器code，完善传感器code配置，删除重复设备

custom-gateway-zhscada/src/main/java/com/shkpr/service/customgateway/zhscada/components/DataMigrator.java

@@ -21,7 +21,6 @@ import org.springframework.core.io.support.ResourcePatternResolver;
 import org.springframework.stereotype.Component;
 
 import java.io.IOException;
-import java.time.LocalDateTime;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -78,14 +77,19 @@ public class DataMigrator {
                 //传感器code
                 String code = sensorMap.getOrDefault(tag.getTag(), null);
                 if (StringUtils.isBlank(code)) continue;
-                //起止时间
-                SensorCollectData timeRange = sensorCollectDataService.findTimeRange(migrateProperties.getSchema(),code);
-                LocalDateTime minTime = timeRange.getMinTime();
-                LocalDateTime maxTime = timeRange.getMaxTime();
+                //数据时间
+                SensorCollectData timeRange = sensorCollectDataService.findTimeRange(migrateProperties.getSchema(), code);
+                if (Objects.isNull(timeRange) || timeRange.getMinTime() == null || timeRange.getMaxTime() == null) {
+                    LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_ERROR, BIZ_TYPE, CLASS_NAME
+                            , String.format("读取Scada数据空数据, tag:%s, code: %s", tag.getTag(), code)
+                    );
+                    continue;
+                }
+
                 //查询数据
                 List<SensorCollectData> dates = sensorCollectDataService.findAlign(migrateProperties.getSchema(),
-                        minTime, maxTime, migrateProperties.getInterval(), migrateProperties.getIntervalUnit(),
-                        migrateProperties.getAlignUnit(), code);
+                        timeRange.getMinTime(), timeRange.getMaxTime(), migrateProperties.getInterval(),
+                        migrateProperties.getIntervalUnit(), migrateProperties.getAlignUnit(), code);
 
                 //构建influxdb
                 List<Point> points = dates.parallelStream().map(d -> d.toPoint(devices, sensorMap))
@@ -113,8 +117,12 @@ public class DataMigrator {
             Resource resources = resourceResolver.getResources(migrateProperties.getSensorMapPath())[0];
             //解析map
             ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
-            sensorMap = mapper.readValue(resources.getFile(), mapper.getTypeFactory()
+            Map<String, String> readValues = mapper.readValue(resources.getFile(), mapper.getTypeFactory()
                     .constructMapType(HashMap.class, String.class, String.class));
+            //过滤空value
+            sensorMap = readValues.entrySet().stream()
+                    .filter(entry -> StringUtils.isNotBlank(entry.getValue()))
+                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         } catch (IOException e) {
             LogPrintMgr.getInstance().printLogMsg(LogLevelFlag.LOG_ERROR, BIZ_TYPE, CLASS_NAME
                     , String.format("传感器映射配置文件读取失败 error:%s", e)

custom-gateway-zhscada/src/main/resources/sensor-id.yml

@@ -1,2 +1,422 @@
 sc_ywbz_ssll: 800
-sc_ywbz_ljll: 801
+sc_ywbz_ljll: 801
+sc_txbz_yjss: 1062
+sc_txbz_wlss: 1065
+gw_fkl_ssll: 1629
+gw_fkl_zljll: 1630
+gw_fkl_fljll: 1631
+gw_dcddyhjbl_ssll: 1682
+gw_dcddyhjbl_zljll: 1683
+gw_dcddyhjbl_fljll: 1684
+gw_fklybcdd_ssll: 1700
+gw_fklybcdd_zljll: 1701
+gw_fklybcdd_fljll: 1702
+gw_szddyjcl_ssll: 1666
+gw_sdddyjcl_zljll: 1667
+gw_szddyjcl_fljll: 1668
+gw_szdd_ssll: 1663
+gw_szdd_zljll: 1664
+gw_szdd_fljll: 1665
+gw_zrl_ssll: 1660
+gw_zrl_zljll: 1661
+gw_zrl_fljll: 1662
+gw_wlddgtys_ssll: null
+gw_wlddgtys_zljll: null
+gw_wlddgtys_fljll: null
+gw_xfl_ssll: 1735
+gw_xfl_zljll: 1736
+gw_xfl_fljll: 1737
+gw_xml_ssll: 1732
+gw_xml_zljl: 1733
+gw_xml_fljll: 1734
+gw_jclybcdd_ssll: 1729
+gw_jclybcdd_zljll: 1730
+gw_jclybcdd_fljll: 1731
+gw_zsdd_ssll: 1726
+gw_zsdd_zljll: 1727
+gw_zsdd_fljll: 1728
+gw_jfl_ssll: 1723
+gw_jfl_zljll: 1724
+gw_jfl_fljll: 1725
+gw_mclk_ssll: 1720
+gw_mclk_zljll: 1721
+gw_mclk_fljll: 1722
+gw_bcddlc_ssll: 1717
+gw_bcddlc_zljll: 1718
+gw_bcddlc_fljll: 1719
+gw_wlgyl_ssll: 1714
+gw_wlgyl_zljll: 1715
+gw_wlgyl_fljll: 1716
+gw_hjl_ssll: null
+gw_hjl_zljll: null
+gw_hjl_fljll: null
+gw_gcdsc_ssll: 1669
+gw_gcdsc_zljll: 1680
+gw_gcdsc_fljll: 1681
+gw_ccddlk_ssll: 1738
+gw_ccddlk_zljll: 1739
+gw_ccddlk_fljll: 1740
+gw_fklbz_ssll: 1632
+gw_fklbz_zljll: 1633
+gw_fklbz_fljll: 1634
+gw_fhc_ssll: 1616
+gw_fhc_zljll: 1617
+gw_fhc_fljll: 1618
+gw_xydddn800_ssll: 1685
+gw_xydddn800_zljll: 1686
+gw_xydddn800_fljll: 1688
+gw_wlnl_ssll: 1623
+gw_wlnl_zljll: 1624
+gw_wlnl_fljll: 1625
+gw_tsddaqd_ssll: 1613
+gw_tsddaqd_zljll: 1614
+gw_tsddaqd_fljll: 1615
+gw_dcddfl_ssll: null
+gw_dcddfl_zljll: null
+gw_dcddfl_fljll: null
+gw_hyl_ssll: null
+gw_hyl_zljll: null
+gw_hyl_fljll: null
+gw_lgl_ssll: 1626
+gw_lgl_zljll: 1627
+gw_lgl_fljll: 1628
+gw_ltql_ssll: 1610
+gw_ltql_zljll: 1611
+gw_ltql_fljll: 1612
+gw_dldd_ssll: 1670
+gw_dldd_zljll: 1621
+gw_dldd_fljll: 1622
+gw_azg_ssll: 1619
+gw_azg_zljll: 1620
+gw_azg_fljll: null
+gw_htsl_ssll: 1671
+gw_htsl_zljll: 1672
+gw_htsl_fljll: 1673
+szgyl_ssll_dn600: 1600
+szgyl_ljll_dn600: 1601
+szgyl_ssll_dn400: 1602
+szgyl_ljll_dn400: 1603
+gw_xzdddn400_ssll: 1635
+gw_xzdddn400_zljll: 1636
+gw_xzdddn400_fljll: 1637
+yw_jkssll: 1071
+yw_jkljll: 1072
+yw_ckssll: 1073
+yw_ckljll: 1074
+奔驰大道瞬时流量: 1583
+奔驰大道累计流量: 1584
+gw_zhlznsc_瞬时流量: 1081
+gw_zhlznsc_累计流量: 1082
+gw_zhwldd__瞬时流量: 1100
+gw_zhwldd__累计流量: 1101
+gw_ylzqt_瞬时流量: 1181
+gw_ylzqt_累计流量: 1182
+gw_yjh__瞬时流量: 1140
+gw_yjh__累计流量: 1141
+gw_yjxsb__瞬时流量: null
+gw_yjxsb__累计流量: null
+gw_yjbz_瞬时流量: 741
+gw_yjbz_累计流量: 740
+gw_yjxz_瞬时流量1: 1467
+gw_yjxz_累计流量1: 1468
+gw_wygc_瞬时流量: 810
+gw_wygc_累计流量: 210
+gw_wlxcd_瞬时流量: 1475
+gw_wlxcd_累计流量: 1476
+gw_tdsc1_瞬时流量: 723
+gw_tdsc1_正累计流量: 724
+gw_tdsc1_负累计流量: 725
+gw_txwllj__瞬时流量: 1065
+gw_txwllj__累计流量: 1066
+gw_txbz2__瞬时流量: 1062
+gw_txbz2__累计流量: 1063
+gw_qjg_瞬时流量: 1256
+gw_qjg_累计流量: 1258
+gw_qhyqxz_瞬时流量: 1202
+gw_qhyqxz_累计流量: 1203
+gw_tbh_瞬时流量: 804
+gw_tbh_累计流量: 802
+gw_hjw_瞬时流量: 1120
+gw_hjw_累计流量: 1121
+gw_djsct_瞬时流量: 1232
+gw_djsct_累计流量: 1233
+gw_djsct_负累计流量: 1234
+gw_ccnl__瞬时流量: 1204
+gw_ccnl__正累计: 1207
+gw_ccnl__负流量: 1206
+gw_ccnl__净累计: 1205
+gw_wlz_瞬时流量: 760
+gw_wlz_累计流量: 761
+sc_esc_sqssll1: 980
+sc_esc_sqljll1: 720
+sc_esc_sqssll2: 981
+sc_esc_sqljll2: 721
+sc_esc_wsssll: 534
+sc_esc_wsljll: 536
+sc_esc_wdssll: 535
+sc_esc_wdljll: 537
+sc_ssc_dql1400: 550
+sc_ssc_dqlljll1400: 552
+sc_ssc_chl1600: 551
+sc_ssc_chlljll1600: 553
+sc_sisc_xxssll: 564
+sc_sisc_xxljll: 566
+sc_sisc_xdssll: 563
+sc_sisc_xdlljll: 565
+sc_wsc_dql1400: 577
+sc_wsc_dqlljll1400: 578
+sc_wsc_chl1600: 576
+sc_wsc_ljll1600: 579
+gw_zhgsljll: 1470
+gw_eqgsljll: 1471
+sc_ywbz_yl: 996
+sc_ywbz_jkyl: 620
+sc_ywbz_ckyl: 621
+sc_txbz_jkyl: 1060
+sc_txbz_wlyl: 1064
+sc_txbz_yjyl: 1061
+gw_jclybcdd_yl: null
+gw_jyc: 1568
+gw_jyc_ckyl: 1591
+奔驰大道设定压力: null
+奔驰大道进口压力: 1581
+奔驰大道出口压力: 1582
+gw_zylyhyljck_压力: 531
+gw_zylyhyljck_电压: null
+gw_zyy_压力: 1359
+gw_zyy_电压: null
+gw_zxyydjsq_压力: 1132
+gw_zxyydjsq_电压: null
+gw_zhlznsc_压力: null
+gw_zhlznsc_电压: null
+gw_zhwldd__压力: null
+gw_zhwldd__电压: null
+gw_ylzqt_压力: 1180
+gw_ylzqt_电压: null
+gw_ylzhrl_压力: 1040
+gw_ylzhrl_电压: null
+gw_ylz_压力: 704
+gw_ylz_电压: null
+gw_yjh_压力: 1160
+gw_yjh_电压: null
+gw_yfnd_压力: 703
+gw_yfnd_电压: null
+gw_yjxsb__压力: null
+gw_yjxsb__电压: null
+gw_yjcd_压力: 1125
+gw_yjcd_电压: null
+gw_yjbz_压力: 116
+gw_yjbz_电压: null
+gw_yjxz_压力1: null
+gw_yjxz_电压1: null
+gw_xfj_压力: 118
+gw_xfj_电压: null
+gw_xzmf_压力: 1127
+gw_xzmf_电压: null
+gw_xydz_压力: 1020
+gw_xydz_电压: null
+gw_xxbz_压力: 680
+gw_xxbz_电压: null
+gw_xgwl_压力: 1129
+gw_xgwl_电压: null
+gw_xcqzf_压力: 706
+gw_xcqzf_电压: null
+gw_jjkf_压力: 705
+gw_jjkf_电压: null
+gw_xjcbz_压力: 780
+gw_xjcbz_电压: null
+gw_wzbce_压力: 1136
+gw_wzbce_电压: null
+gw_wzgj_压力: 805
+gw_wzgj_电压: null
+gw_wygc_压力: null
+gw_wygc_电压: null
+gw_wlxcd_压力: null
+gw_wlxcd_电压: null
+gw_tdk_压力: 940
+gw_tdk_电压: null
+gw_teb_压力: 1230
+gw_teb_电压: null
+gw_tc_压力: 900
+gw_tc_电压: null
+gw_tdsc1_压力: 722
+gw_tdsc1_电压: null
+gw_txwllj__压力: null
+gw_txwllj__电压: null
+gw_txbz2__压力: null
+gw_txbz2__电压: null
+gw_sas_压力: 521
+gw_sas_电压: null
+smzx_压力: 1235
+smzx_电压: null
+gw_ss_压力: 1687
+gw_ss_电压: null
+gw_szj_压力: 117
+gw_szj_电压: null
+gw_szgyy_压力: 119
+gw_szgyy_电压: null
+gw_rmljwlk_压力: 963
+gw_rmljwlk_电压: null
+gw_qjg_压力: 1200
+gw_qjg_电压: null
+gw_qhyqxz_压力: null
+gw_qhyqxz_电压: null
+gw_qhyq_压力: 105
+gw_qhyq_电压: null
+gw_qxj_压力: 901
+gw_qxj_电压: null
+压力: null
+电压: null
+gw_om_压力: 841
+gw_om_电压: null
+nhjf_压力: 1237
+nhjf_电压: null
+gw_ltxny_压力: 960
+gw_ltxny_电压: null
+gw_lmddxbj_压力: 1146
+gw_lmddxbj_电压: null
+gw_lmdd_压力1: 1572
+gw_lmdd_电压1: null
+gw_wzb_压力: 1126
+gw_wzb_电压: null
+gw_tbh_压力: 803
+gw_tbh_电压: null
+gw_lzjyzx_压力: lzjyzx
+gw_lzjyzx_电压: null
+gw_lsdf_压力: 1130
+gw_lsdf_电压: null
+gw_jygs_压力: null
+gw_jygs_电压: null
+gw_jyzg_压力: 961
+gw_jyzg_电压: null
+gw_jsxy压力: 642
+gw_jsxy电压: null
+gw_jcl_压力: 2000
+gw_jcl_电压: null
+gw_hp_压力: 1128
+gw_hp_电压: null
+gw_hcz_压力: 107
+gw_hcz_电压: null
+gw_hjw_压力: null
+gw_hjw_电压: null
+gw_hw_压力: 640
+gw_hw_电压: null
+gw_hbhyjg_压力: 5167
+gw_hbhyjg_电压: null
+gw_hwsq_压力: 1111
+gw_hwsq_电压: null
+gw_gwzx_压力: 111
+gw_gwzx_电压: null
+gw_gld_压力: 1920
+gw_gld_电压: null
+gw_fkdd_压力: 920
+gw_fkdd_电压: null
+gw_fzg_压力: 806
+gw_fzg_电压: null
+gw_fxxc_压力: 1133
+gw_fxxc_电压: null
+gw_ez_压力: 702
+gw_ez_电压: null
+gw_23z_压力: 115
+gw_23z_电压: null
+gw_djctc_压力: 1231
+gw_djctc_电压: null
+djswdl_压力: 1236
+djswdl_电压: null
+gw_dfjg_压力: 1131
+gw_dfjg_电压: null
+gw_dlg_压力: 700
+gw_dlg_电压: null
+cydl_压力: 114
+cydl_电压: null
+gw_ccnl__压力: 1080
+gw_ccnl__电压: null
+gw_3542_压力: 106
+gw_3542_电压: null
+gw_byd_压力: 1154
+gw_byd_电压: null
+by6_压力: null
+by6_电压: null
+gw_bhj_压力: 641
+gw_bhj_电压: null
+gw_bswl_压力: 2001
+gw_bswl_电压: null
+gw_bmgc_压力: 354
+gw_bmgc_电压: null
+gw_yjh_水位: 1161
+gw_yjbz_水位: 120
+gw_xxbz_水位: 681
+gw_sas_水位: 1645
+gw_szgyy_水位: 112
+gw_qjg_水位: 1201
+gw_wlz_压力: 707
+gw_wlz_电压: null
+sc_esc_wsyl: 532
+sc_esc_sqyl: 538
+sc_esc_wdyl: 10000
+sc_ssc_ckyl: 548
+sc_sisc_ckyl: 561
+sc_wsc_ckyl1400: 587
+sc_wsc_ckyl1600: 861
+gw_tdsc_yv: 1481
+gw_tdsc_zd: 1482
+ssc_qszd: null
+ssc_qszd1: 666
+gw_ssc_cszd: 1
+wzsy_ph: 1484
+wzsy_yv: 1483
+wzsy_zd: 1480
+gw_esc_yszd: 1474
+gw_wscqs_zd: 670
+gw_wsc_yv: 672
+gw_wsc_zd: 671
+gw_wsc_wd: 673
+gw_sisc_ph: null
+gw_sisc_yv: 668
+gw_sisc_zd: 667
+gw_sisc_wd: 669
+gw_ssc_ph: null
+gw_ssc_yv: 664
+gw_ssc_zd: 663
+gw_ssc_wd: 665
+gw_esc_ph: null
+gw_esc_yv: 661
+gw_esc_zd: 660
+tyxq_zd: 1605
+sc_ywbz_sw1: 880
+sc_ywbz_sw2: null
+gw_ylzqt_水位: null
+sc_esc_sw: 533
+sc_ssc_sw: 549
+sc_sisc_sw: 562
+sc_wsc_sw: 575
+奔驰大道1号泵开停状况: 1585
+奔驰大道1号泵工作频率: 1588
+奔驰大道2号泵开停状况: 1586
+奔驰大道2号泵工作频率: 1589
+奔驰大道3号泵开停状况: 1587
+奔驰大道3号泵工作频率: 1590
+sc_esc_bjzt1: 539
+sc_esc_bjzt2: 540
+sc_esc_bjzt3: 541
+sc_esc_bjzt4: 542
+sc_esc_bjzt5: 543
+sc_esc_bjzt6: 544
+sc_esc_bjzt7: 545
+sc_ssc_hbzt1: 554
+sc_ssc_hbzt2: 555
+sc_ssc_hbzt3: 556
+sc_ssc_hbzt4: 557
+sc_ssc_hbzt5: 558
+sc_sisc_hbzt6: 567
+sc_sisc_hbzt7: 568
+sc_sisc_hbzt8: 569
+sc_sisc_hbzt9: 570
+sc_sisc_hbzt10: 571
+sc_wsc_hbzt1: 580
+sc_wsc_hbzt2: 581
+sc_wsc_hbzt3: 582
+sc_wsc_hbzt4: 583
+sc_wsc_hbzt5: 584
+sc_wsc_hbzt6: 585
+sc_wsc_hbzt17: 586

dev_zhscada.yml

@@ -1169,7 +1169,7 @@
   mfrs: null
   createTime: "2025-11-03 17:45:44"
   tags:
-    - tag: "w_xzdddn400_ssll"
+    - tag: "gw_xzdddn400_ssll"
       protocol: "json"
       measurement: "WaterMeter"
       field: "flow_cur"
@@ -4655,23 +4655,6 @@
       maxValue: null
       syncField: null
       calcFormula: null
-- deviceId: "710102511130001"
-  deviceName: "云湾泵站"
-  deviceSn: null
-  deviceKind: "quality"
-  platform: "zhonghuan-scada"
-  mfrs: null
-  createTime: "2025-11-03 17:45:44"
-  tags:
-    - tag: "sc_ywbz_yl"
-      protocol: "json"
-      measurement: "WaterQuality"
-      field: "chlorine"
-      valueType: "double"
-      minValue: null
-      maxValue: null
-      syncField: null
-      calcFormula: null
 - deviceId: "710102511130002"
   deviceName: "汤店水厂"
   deviceSn: null