1. 修正实时走航数据平滑处理算法，PM2.5、PM10、VOC采用新的修正算法

feiyu02

2024-07-18 5b0d58c3f7f35f61c0a0437bac3ff708db57fe61

 src/main/kotlin/com/flightfeather/uav/model/epw/EPWDataPrep.kt

@@ -1,37 +1,68 @@
package com.flightfeather.uav.model.epw

import com.flightfeather.uav.common.utils.DateUtil
import com.flightfeather.uav.common.utils.GsonUtils
import com.flightfeather.uav.lightshare.bean.CompanySOP
import com.flightfeather.uav.lightshare.bean.DataVo
import com.flightfeather.uav.model.BaseDataPrep
import com.flightfeather.uav.model.BaseSOP
import com.flightfeather.uav.socket.bean.AirData
import com.flightfeather.uav.socket.bean.AirDataPackage
import com.flightfeather.uav.socket.eunm.FactorType
import com.flightfeather.uav.socket.eunm.UWDeviceType
import kotlin.math.max
import kotlin.math.min
import kotlin.math.round
import kotlin.math.sqrt

class EPWDataPrep : BaseDataPrep<DataVo, BaseSOP>() {
/**
 * 数据平滑预处理
 * 对于最新的一组数据，根据其之前连续的若干数据，进行数据平滑处理
 * @Date 2024.5.16
 */
class EPWDataPrep(val deviceType: UWDeviceType? = UWDeviceType.GRID) : BaseDataPrep<DataVo, BaseSOP>() {

    // 向前检索的数据记录数
    private val ncal = 15

    // 标准差倍数参数
    private val nstd = 3

    // 均值倍数参数
    private val xratio = 3
    // 需要处理的因子类型
    private val calTypes =
//        emptyList<String>()
        WeightType.prep

    // 需要平滑处理的因子类型
    private var calTypes = when (deviceType) {
        UWDeviceType.VEHICLE,
        UWDeviceType.UAV,
        UWDeviceType.BOAT,
        -> WeightType.prepUnderWay
        UWDeviceType.GRID -> WeightType.prepFixed
        else -> WeightType.prepFixed
    }

    // 只需要检查范围和变化幅度的因子类型
    private var rangeTypes = listOf(
        FactorType.PM25.des,
        FactorType.PM10.des,
        FactorType.VOC.des
    )

    // 无需修正的因子类型
    private var noCalTypes = listOf(
        FactorType.TEMPERATURE.des,
        FactorType.HUMIDITY.des,
        FactorType.LNG.des,
        FactorType.LAT.des,
        FactorType.VELOCITY.des,
        FactorType.TIME.des,
        FactorType.WIND_DIRECTION.des,
        FactorType.HEIGHT.des
    )

    private val lastData = mutableListOf<DataVo>()

    override fun mDataPrep(mDataList: List<DataVo>): List<DataVo> {
        mDataList.forEach {
            it.values?.forEach v@{a ->
            it.values?.forEach v@{ a ->
                if (!calTypes.contains(a.factorName)) return@v

                val range = FactorType.getRange(a.factorName) ?: return@v
@@ -41,11 +72,6 @@
                }
            }
        }

//        val newDataList = mutableListOf<DataVo>()
//        mDataList.forEach {
//            newDataList.add(it.copy())
//        }

        var i = ncal
        if (lastData.isNotEmpty()) {
@@ -61,7 +87,7 @@

                if (it.factorData!! > vMax) {
                    val lastDataIndex = i
                    val thisIndex = if (i-ncal<0) 0 else i - ncal
                    val thisIndex = if (i - ncal < 0) 0 else i - ncal
                    val list = mutableListOf<DataVo>()
                    if (lastDataIndex < lastData.size) {
                        list.addAll(lastData.subList(lastDataIndex, lastData.lastIndex + 1))
@@ -104,11 +130,18 @@
        return sopList
    }


    /**
     * 实时数据平滑处理
     */
    fun mDataPrep2(dataPackage: AirDataPackage): List<DataVo> {
        val vo = dataPackage.toDataVo()
        return mDataPrep2(listOf(vo))
    }

    /**
     * 实时数据平滑处理
     */
    fun mDataPrep2(mDataList: List<DataVo>): List<DataVo> {
        var i = ncal
        if (lastData.isNotEmpty()) {
@@ -117,14 +150,12 @@
        while (i < mDataList.size) {
            for (y in mDataList[i].values?.indices ?: 0..0) {
                val it = mDataList[i].values?.get(y) ?: continue

                if (!calTypes.contains(it.factorName)) continue
                val vMax = FactorType.getVMax(it.factorName) ?: continue
//                it.factorData ?: continue
                it.factorData ?: continue

                if (it.factorData!! > vMax) {
                    val lastDataIndex = i
                    val thisIndex = if (i-ncal<0) 0 else i - ncal
                    val thisIndex = if (i - ncal < 0) 0 else i - ncal
                    val list = mutableListOf<DataVo>()
                    if (lastDataIndex < lastData.size) {
                        list.addAll(lastData.subList(lastDataIndex, lastData.lastIndex + 1))
@@ -136,17 +167,26 @@
                        // 去除无效值的标准差
                        val std = standardDeviation(avg.first, list, it.factorName)
                        // 合理最大值
                        val max = max(avg.first + std * nstd, avg.first + avg.first * xratio)
                        var maxValue = max(avg.first + std * nstd, avg.first + avg.first * xratio)
                        maxValue = max(maxValue, FactorType.getRange(it.factorName)?.second ?: .0)
                        // 合理最小值
                        val min = min(avg.first - std * nstd, avg.first / (1 + xratio))
                        val minValue = min(avg.first - std * nstd, avg.first / (1 + xratio))

                        // 数据不处于合理范围并且有效个数达标时，采用计算所得均值代替原始值
                        if (avg.second > max(ncal / 5, 2)
                            && (it.factorData!! < min || it.factorData!! > max)
                        ) {
                            // 原始数据
//                        it.factorData = null
                            it.factorData = avg.first
                        // 判断监测因子是否需要进行平滑处理，
                        if (calTypes.contains(it.factorName)) {
                            // 数据不处于合理范围并且有效个数达标时，采用计算所得均值代替原始值
                            if (avg.second > max(ncal / 5, 2)
                                && (it.factorData!! < minValue || it.factorData!! > maxValue)
                            ) {
                                it.factorData = avg.first
                            }
                        }
                        // 判断量级是否在合理范围内以及变化倍率是否在合理范围内
                        else if (rangeTypes.contains(it.factorName)) {
                            if (isInRange(it) != true || excessiveChange(it) == true) {
                                // 采用计算所得均值代替原始值
                                it.factorData = avg.first
                            }
                        }
                    }
                }
@@ -183,7 +223,7 @@
     * @param list 监测数据
     * @return 均值和有效数据个数
     */
    private fun average(list: List<DataVo>, factorName:String?): Pair<Double, Int>? {
    private fun average(list: List<DataVo>, factorName: String?): Pair<Double, Int>? {
        var t = 0.0
        var c = 0
        list.forEach {
@@ -192,7 +232,7 @@
                if (f?.factorName == factorName) {
                    val range = FactorType.getRange(f?.factorName) ?: continue
                    //判断数据是否在合理范围内
                    if (f?.factorData ?: 0.0 in range.first..range.second) {
                    if ((f?.factorData ?: 0.0) in range.first..range.second) {
                        t += f?.factorData!!
                        c++
                    }
@@ -240,4 +280,32 @@
            sqrt(t / (c - 1))
        }
    }

    /**
     * 判断数据是否在正常量程内
     */
    private fun isInRange(airData: AirData): Boolean? {
        val range = FactorType.getRange(airData.factorName) ?: return null
        //判断数据是否在合理范围内
        return (airData.factorData ?: 0.0) in range.first..range.second
    }

    /**
     * 判断连续的数据量级上升幅度是否过大
     */
    private fun excessiveChange(airData: AirData): Boolean? {
        airData.factorData ?: return null
        if (lastData.isEmpty()) return false
        val latestData = lastData.last()
        // 结果倍率
        var m = 1.0
        for (i in latestData.values?.indices ?: 0..0) {
            val f = latestData.values?.get(i)
            if (f?.factorName == airData.factorName) {
                m = airData.factorData!!.div(f?.factorData ?: airData.factorData!!)
                break
            }
        }
        return m > FactorType.getMultiplier(airData.factorName)
    }
}