1. 新增导入静安监测数据功能模块

feiyu02

2024-02-02 83455446544f89b0663a3f520744331ad8259289

 src/main/kotlin/com/flightfeather/uav/lightshare/service/impl/RealTimeDataServiceImpl.kt

@@ -1,15 +1,14 @@
package com.flightfeather.uav.lightshare.service.impl

import com.flightfeather.uav.common.exception.ResponseErrorException
import com.flightfeather.uav.common.scaleMap
import com.flightfeather.uav.common.utils.DateUtil
import com.flightfeather.uav.common.utils.ExcelUtil
import com.flightfeather.uav.common.utils.FileExchange
import com.flightfeather.uav.common.utils.GsonUtils
import com.flightfeather.uav.dataprocess.AverageUtil
import com.flightfeather.uav.domain.entity.*
import com.flightfeather.uav.domain.mapper.RealTimeDataGridMapper
import com.flightfeather.uav.domain.mapper.RealTimeDataMapper
import com.flightfeather.uav.domain.mapper.RealTimeDataUavMapper
import com.flightfeather.uav.domain.mapper.RealTimeDataVehicleMapper
import com.flightfeather.uav.domain.mapper.*
import com.flightfeather.uav.lightshare.bean.*
import com.flightfeather.uav.lightshare.service.RealTimeDataService
import com.flightfeather.uav.model.epw.EPWDataPrep
@@ -18,16 +17,21 @@
import com.flightfeather.uav.socket.eunm.UWDeviceType
import com.github.pagehelper.PageHelper
import org.apache.poi.xssf.streaming.SXSSFWorkbook
import org.springframework.beans.BeanUtils
import org.springframework.beans.factory.annotation.Value
import org.springframework.stereotype.Service
import org.springframework.web.multipart.MultipartFile
import tk.mybatis.mapper.entity.Example
import java.io.ByteArrayInputStream
import java.io.File
import java.text.SimpleDateFormat
import java.time.LocalDateTime
import java.time.ZoneId
import java.time.format.DateTimeFormatter
import java.util.*
import javax.servlet.http.HttpServletResponse
import javax.swing.text.DateFormatter
import kotlin.math.sqrt

@Service
class RealTimeDataServiceImpl(
@@ -35,14 +39,27 @@
    private val airDataRepository: AirDataRepository,
    private val realTimeDataVehicleMapper: RealTimeDataVehicleMapper,
    private val realTimeDataUavMapper: RealTimeDataUavMapper,
    private val realTimeDataGridMapper: RealTimeDataGridMapper
    private val realTimeDataGridMapper: RealTimeDataGridMapper,
    private val realTimeDataGridOptMapper: RealTimeDataGridOptMapper,
    private val realTimeDataGridMinMapper: RealTimeDataGridMinMapper,
    private val missionMapper: MissionMapper,
) : RealTimeDataService {

    @Value("\${filePath}")
    lateinit var filePath: String

    private var dateFormatter = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
    private var dateFormatter2 = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")
    private val fileExchange = FileExchange()

    override fun getSecondData(deviceCode: String?, startTime: String?, endTime: String?, page: Int?, perPage: Int?): BaseResponse<List<DataVo>> {
    override fun getSecondData(
        deviceCode: String?,
        startTime: String?,
        endTime: String?,
        type: Int?,
        page: Int?,
        perPage: Int?,
    ): BaseResponse<List<DataVo>> {
        val _perPage = perPage ?: 60
        val _page = page ?: 1
        val sTime = startTime?.let { dateFormatter.parse(it) }
@@ -70,12 +87,24 @@
                pages = pageInfo.pages
            }
            UWDeviceType.GRID -> {
                val pageInfo = PageHelper.startPage<RealTimeDataGrid>(_page, _perPage)
                realTimeDataGridMapper.selectByExample(Example(RealTimeDataGrid::class.java).apply {
                    getSecondDataExample(this, deviceCode, sTime, eTime)
                }).forEach { result.add(it.toDataVo()) }
                pageNum = pageInfo.pageNum
                pages = pageInfo.pages
                // 网格化监测秒级值
                if (type == null || type == 0) {
                    val pageInfo = PageHelper.startPage<RealTimeDataGrid>(_page, _perPage)
                    realTimeDataGridMapper.selectByExample(Example(RealTimeDataGrid::class.java).apply {
                        getSecondDataExample(this, deviceCode, sTime, eTime)
                    }).forEach { result.add(it.toDataVo()) }
                    pageNum = pageInfo.pageNum
                    pages = pageInfo.pages
                }
                // 网格化监测分钟值
                else if (type == 1) {
                    val pageInfo = PageHelper.startPage<RealTimeDataGridMin>(_page, _perPage)
                    realTimeDataGridMinMapper.selectByExample(Example(RealTimeDataGridMin::class.java).apply {
                        getSecondDataExample(this, deviceCode, sTime, eTime)
                    }).forEach { result.add(it.toDataVo()) }
                    pageNum = pageInfo.pageNum
                    pages = pageInfo.pages
                }
            }
            else -> {
                // 从原始数据表中获取数据
@@ -113,7 +142,12 @@
        }
    }

    override fun getNextData(deviceCode: String, updateTime: String, page: Int?, perPage: Int?): BaseResponse<List<DataVo>> {
    override fun getNextData(
        deviceCode: String,
        updateTime: String,
        page: Int?,
        perPage: Int?,
    ): BaseResponse<List<DataVo>> {
        val _perPage = perPage ?: 60
        val _page = page ?: 1
        val pageInfo = PageHelper.startPage<RealTimeData>(_page, _perPage)
@@ -155,7 +189,52 @@
        fileExchange.exchangeBoatData("0c0000000001", f).forEach {
            realTimeDataMapper.insert(it)
        }
        return BaseResponse(true, data = DataImportResult(""))
        return BaseResponse(true, data = DataImportResult("成功"))
    }

    override fun importJinanData(code: String, file: MultipartFile): DataImportResult {
        val f = ByteArrayInputStream(file.bytes)
        val result = fileExchange.exchangeJinanData(code, f)
        if (result.isNotEmpty()) {
            val first = result.first()
            val t = DateUtil.instance.dateToString(first.dataTime, DateUtil.DateStyle.YYYY_MM_DD)
            val last = result.last()
            val mission = Mission().apply {
                val tag = code.substring(0, 2) + "-" + code.substring(code.length - 2, code.length)
                missionCode = "SH-JA-${tag}-${t}"
                deviceType = UWDeviceType.getType(code)?.value
                deviceCode = code
            }
            missionMapper.selectOne(mission)?.run { throw ResponseErrorException("该设备该时段任务已存在，无法重复导入") }
            mission.apply {
                startTime = first.dataTime
                endTime = last.dataTime
            }
            missionMapper.insert(mission)
            realTimeDataVehicleMapper.insertList(result)
        }
        return DataImportResult("成功")
    }

    override fun downloadTemplate(response: HttpServletResponse): Boolean {
        val fileName = "JinAn-Template.xlsx"
        val path = (Thread.currentThread().contextClassLoader?.getResource("/")?.path
            ?: "src/main/resources") + "/templates/" + fileName
        val file = File(path)
        if (file.exists()) {
            val fName = Base64.getEncoder().encodeToString(fileName.toByteArray())
            response.apply {
                setHeader("Content-Disposition", "attachment;filename=$fName")
                setHeader("fileName", fName)
                addHeader("Access-Control-Expose-Headers", "fileName")
                contentType = "application/vnd.ms-excel;charset=UTF-8"
                setHeader("Pragma", "no-cache")
                setHeader("Cache-Control", "no-cache")
                setDateHeader("Expires", 0)
            }
            response.outputStream.write(file.readBytes())
        }
        return true
    }

    override fun outToWorkbook(deviceCode: String, startTime: String, endTime: String): SXSSFWorkbook {
@@ -177,13 +256,19 @@
            })
            if (r.isNotEmpty()) {
                val heads = if (page == 1) {
                    println("[${DateUtil.instance.dateToString(Date(), DateUtil.DateStyle.YYYY_MM_DD_HH_MM_SS)}] totalPage: ${pageInfo.pages}")
                    println("[${
                        DateUtil.instance.dateToString(Date(),
                            DateUtil.DateStyle.YYYY_MM_DD_HH_MM_SS)
                    }] totalPage: ${pageInfo.pages}")
                    getTableTitle(r[0])
                } else {
                    emptyList()
                }
                val contents = getTableContents(r)
                print("[${DateUtil.instance.dateToString(Date(), DateUtil.DateStyle.YYYY_MM_DD_HH_MM_SS)}] currentPage: ${pageInfo.pageNum}......")
                print("[${
                    DateUtil.instance.dateToString(Date(),
                        DateUtil.DateStyle.YYYY_MM_DD_HH_MM_SS)
                }] currentPage: ${pageInfo.pageNum}......")
                rowIndex = ExcelUtil.write(heads, contents, workbook, row = rowIndex)
                println("output done")
            }
@@ -193,7 +278,12 @@
        return workbook
    }

    override fun outToExcel(deviceCode: String, startTime: String, endTime: String, response: HttpServletResponse): HttpServletResponse {
    override fun outToExcel(
        deviceCode: String,
        startTime: String,
        endTime: String,
        response: HttpServletResponse,
    ): HttpServletResponse {
        val workbook = outToWorkbook(deviceCode, startTime, endTime)

        val out = response.outputStream
@@ -217,7 +307,13 @@
        return contents
    }

    private fun getOriginData(deviceCode: String?, startTime: String?, endTime: String?, page: Int?, perPage: Int?): BaseResponse<List<DataVo>> {
    override fun getOriginData(
        deviceCode: String?,
        startTime: String?,
        endTime: String?,
        page: Int?,
        perPage: Int?,
    ): BaseResponse<List<DataVo>> {
        val _perPage = perPage ?: 60
        val _page = page ?: 1
        val sTime = startTime?.let { dateFormatter.parse(it) }
@@ -253,7 +349,7 @@
        var count = 0

        while (total == -1 || page <= total) {
            println("------start------")
            println("------数据预处理start------")
            val res = getOriginData("0d0000000001", "2021-07-05 19:47:01", "2021-11-05 00:00:00", page, 50000)
            res.head?.let {
                total = it.totalPage
@@ -272,23 +368,22 @@
    }

    override fun averageData(): BaseResponse<String> {
        val epwDataPrep = EPWDataPrep()
        var page = 1
        var total = -1
        var count = 0

        val minFormatter = SimpleDateFormat("yyyy-MM-dd HH:mm")
        val averageUtil = AverageUtil<RealTimeDataGrid>({d ->
        val averageUtil = AverageUtil<RealTimeDataGridOpt, RealTimeDataGridMin>({ d ->
            minFormatter.format(d.dataTime)
        },{list ->
        }, { list ->
            list.avg()
        })

        while (total == -1 || page <= total) {
            println("------start------")
            println("------均值计算start------")

            val p = PageHelper.startPage<RealTimeDataGrid>(page, 50000)
            val res = realTimeDataGridMapper.selectByExample(Example(RealTimeDataGrid::class.java).apply {
            val p = PageHelper.startPage<RealTimeDataGridOpt>(page, 50000)
            val res = realTimeDataGridOptMapper.selectByExample(Example(RealTimeDataGridOpt::class.java).apply {
                createCriteria().andBetween("dataTime", "2021-06-01 00:00:00", "2021-11-05 00:00:00")
            })

@@ -299,7 +394,8 @@
            }
            println("当前页数：$page")
            averageUtil.avg(res).forEach {

                realTimeDataGridMinMapper.insert(it)
                count++
            }

            page++
@@ -307,4 +403,60 @@

        return BaseResponse(count > 0, data = "插入数据: ${count}条")
    }

    override fun dataCalibration(): BaseResponse<String> {
        var page = 1
        var total = -1
        var count = 0
        val sT = "2021-06-19 00:00:00"
        val eT = "2021-10-21 00:00:00"

        println("------数据优化start------")
        while (total == -1 || page <= total) {
            // 预处理后的网格化数据
            val p = PageHelper.startPage<RealTimeDataGrid>(page, 50000)
            val dataList1 = realTimeDataGridMapper.selectByExample(Example(RealTimeDataGrid::class.java).apply {
                createCriteria().andBetween("dataTime", sT, eT)
            })
            // 原始数据（H2S）
            val res = getOriginData("0d0000000001", sT, eT, page, 50000)
            val dataList2 = res.data ?: emptyList()

            total = p.pages

            if (page == 1) {
                println("总页数：$total")
            }
            println("当前页数：$page")

            for (i in dataList1.indices) {
                if (i >= dataList2.size) break
                val d1 = dataList1[i]
                val d2 = dataList2[i]
                // 判断小时，获取对应时段的优化系数
                val h = LocalDateTime.ofInstant(d1.dataTime?.toInstant(), ZoneId.systemDefault()).hour
                val scaleList = scaleMap[h] ?: continue
                d1.apply {
                    // 各预处理后的因子（除H2S）乘系数
                    no2 = no2?.times(scaleList[0])
                    co = co?.times(scaleList[1])
                    so2 = so2?.times(scaleList[3])
                    o3 = o3?.times(scaleList[4])
                    voc = voc?.times(scaleList[7])

                    // 单独把原始的H2S乘以系数
                    h2s = d2.values?.get(2)?.factorData?.toFloat()?.times(scaleList[2])
                    h2s = h2s?.let { sqrt(it) * 2 }
                }

                // 更新秒级值
                val opt = RealTimeDataGridOpt()
                BeanUtils.copyProperties(d1, opt)
                count += realTimeDataGridOptMapper.insert(opt)
            }
            page++
        }

        return BaseResponse(count > 0, data = "插入数据: ${count}条")
    }
}