feat(analog): 新增ICC模拟量数据同步 - 重建electricity_data表(迁移20261024_icc_analog_data.sql:删除equip_id与electricity占位列、icc_device_code更名imei、新增analog_type/analog_name/analog_unit/analog_value、collect_time以同步时间填充、按月分区2026_10~2027_03、唯一键(line_code,analog_type,collect_time));同步服务IccAnalogSyncService复用OAuth2鉴权,按已同步线路line_code分批(≤50)拉取最新模拟量(POST /analog/subsystem/page,pageSize=500短页停止)幂等upsert并写icc_sync_log(sync_type=electricity_data);新增POST /api/electricity-data/sync手动同步(RBAC device:list);注册icc_analog_sync每5分钟定时任务与sys_job种子;同步db_init/db_seed与设计文档;无前端改动

This commit is contained in:
wk authored and trae-solo-agent committed 2026-10-24 14:52:22 +08:00
1 parent 160ad85051
commit c92fc7f298
15 files changed
+701 -27

No files matched your search

@@ -0,0 +1,14 @@
import { Controller, Post, Inject } from "@midwayjs/core";
import { Context } from "@midwayjs/koa";
import { IccAnalogSyncService } from "../service/icc_analog_sync.service";
@Controller("/api/electricity-data")
export class ElectricityDataController {
@Inject() ctx!: Context;
@Inject() iccAnalogSyncService!: IccAnalogSyncService;
@Post("/sync") async sync() {
const result = await this.iccAnalogSyncService.syncFromIcc(this.ctx);
return { code: 200, message: "模拟量数据同步完成", data: result };
}
}
@@ -0,0 +1,56 @@
import { Column, Model, Table, DataType, ForeignKey, BelongsTo } from "sequelize-typescript";
import ElectricLine from "./electric_line.entity";
import Customer from "./customer.entity";
@Table({
tableName: "electricity_data",
comment: "线路用电信息表(ICC 模拟量同步,按月分区时序数据)",
timestamps: false,
indexes: [
{ name: "idx_ed_customer", fields: ["customer_id"] },
{ name: "idx_ed_line", fields: ["line_id"] },
{ name: "idx_ed_time", fields: ["collect_time"] }
]
})
export default class ElectricityData extends Model {
@Column({ type: DataType.BIGINT, autoIncrement: true, primaryKey: true })
id!: number;
@ForeignKey(() => ElectricLine)
@Column({ type: DataType.BIGINT, allowNull: false, field: "line_id", comment: "线路 ID(→ electric_line)" })
lineId!: number;
@Column({ type: DataType.STRING(64), allowNull: false, field: "line_code", comment: "线路编码(ICC deviceCode,冗余 + 幂等键)" })
lineCode!: string;
@Column({ type: DataType.STRING(64), field: "imei", comment: "设备唯一标识码(ICC uniqueCode)" })
imei!: string;
@Column({ type: DataType.STRING(128), field: "line_name", comment: "线路名称(冗余,加速展示)" })
lineName!: string;
@ForeignKey(() => Customer)
@Column({ type: DataType.BIGINT, allowNull: false, field: "customer_id", comment: "客户 ID(逻辑隔离键)" })
customerId!: number;
@Column({ type: DataType.STRING(64), field: "analog_type", comment: "模拟量类型(ICC analogType)" })
analogType!: string;
@Column({ type: DataType.STRING(128), field: "analog_name", comment: "模拟量名称(ICC analogName,如:剩余电流)" })
analogName!: string;
@Column({ type: DataType.STRING(32), field: "analog_unit", comment: "模拟量单位(ICC analogUnit,如:mA)" })
analogUnit!: string;
@Column({ type: DataType.DECIMAL(14, 4), field: "analog_value", comment: "模拟量值(ICC analogValue)" })
analogValue!: number;
@Column({ type: DataType.DATE, allowNull: false, field: "collect_time", comment: "采集时间(同步时间,分区键)" })
collectTime!: Date;
@BelongsTo(() => ElectricLine, { foreignKey: "lineId", as: "line" })
line!: ElectricLine;
@BelongsTo(() => Customer, { foreignKey: "customerId", as: "customer" })
customer!: Customer;
}
+1
View File
@@ -30,6 +30,7 @@ const PERMISSION_MAP: Record<string, string> = {
"/api/smart-devices": "device:model",
"/api/electric-lines": "device:list",
"/api/electric-gateways": "device:list",
"/api/electricity-data": "device:list",
"/api/survey-orders": "project:survey",
"/api/project-plans": "project:plan",
"/api/contracts": "project:contract",
@@ -0,0 +1,223 @@
import { Provide } from "@midwayjs/core";
import { InjectRepository } from "@midwayjs/sequelize";
import { Repository } from "sequelize-typescript";
import { Context } from "@midwayjs/koa";
import ElectricityData from "../entity/electricity_data.entity";
import ElectricLine from "../entity/electric_line.entity";
import IccSyncLog from "../entity/icc_sync_log.entity";
import { getIccAccessToken, iccFetch } from "../task/icc_customer_auth";
import { ICC_ANALOG_ENDPOINT, mapIccAnalogRecord, parseAnalogPage, IccAnalogRawRecord } from "../task/icc_analog_mapper";
export interface IccAnalogSyncResult {
/** ICC 拉取到的原始模拟量条数 */
pulled: number;
/** 成功落库的模拟量条数(新增+更新) */
synced: number;
/** 新增的模拟量条数 */
inserted: number;
/** 更新的模拟量条数 */
updated: number;
/** 跳过的条数(线路未同步等) */
skipped: number;
}
const SYNC_TYPE = "electricity_data";
/** 模拟量接口每页上限(短页判定阈值,ICC 接口约定) */
const PAGE_SIZE = 500;
/** deviceCodes 每批最多 50 个(ICC 模拟量接口约定) */
const DEVICE_BATCH = 50;
/** 最大分页数保护,防止异常接口导致无限拉取 */
const MAX_PAGES = 100;
/** ICC_VERSION 缺省值(与 icc_customer_auth 保持一致) */
const DEFAULT_VERSION = "1.0.0";
/** HTTP 超时 60s(项目约定) */
const HTTP_TIMEOUT_MS = 60000;
/** 线路预取信息(用于模拟量落库时回填 lineId / lineName / customerId,避免 N+1) */
interface LineMatchInfo {
lineId: number;
lineName?: string;
customerId: number;
}
/**
* ICC 探测器模拟量数据同步(默认每 5 分钟执行一次):
* 1) 复用 icc_customer_auth 的真实 OAuth2 密码模式鉴权(getIccAccessToken / iccFetch),
* 从 .env 读取 ICC_BASE_URL / ICC_VERSION
* 2) 先读本系统已同步的 electric_line(line_code = ICC deviceCode),把 line_code 每批 ≤50
* 作为 deviceCodes 分批 POST 模拟量接口(/analog/subsystem/page,pageSize=500 短页停止)
* 3) 以同步时间作采集时间,按唯一键 (line_code, analog_type, collect_time) 幂等 upsert
* 写入 electricity_data(同一次同步内重复则覆盖更新)
* 4) 每次同步写一条 icc_sync_log(sync_type=electricity_data);拉取失败仅记录 error,不抛业务错误
*/
@Provide()
export class IccAnalogSyncService {
@InjectRepository(ElectricityData)
electricityModel!: Repository<ElectricityData>;
@InjectRepository(ElectricLine)
lineModel!: Repository<ElectricLine>;
@InjectRepository(IccSyncLog)
syncLogModel!: Repository<IccSyncLog>;
async syncFromIcc(ctx?: Context): Promise<IccAnalogSyncResult> {
const startTime = new Date();
// 采集时间 = 同步拉取时间(ICC 最新模拟量接口实际不返回 reportTime,经确认以同步时间作采集时间/分区键)
const collectTime = startTime;
const baseUrl = process.env.ICC_BASE_URL?.trim();
const result: IccAnalogSyncResult = { pulled: 0, synced: 0, inserted: 0, updated: 0, skipped: 0 };
// 基础配置缺失视为拉取失败:记录日志,不抛业务错误
if (!baseUrl) {
await this.writeLog(startTime, 0, SYNC_TYPE, 0, "未配置 ICC_BASE_URL,请检查 server/.env");
return result;
}
try {
// 预取本系统已同步线路 → line_code → {lineId,lineName,customerId} 映射,避免 N+1 查询
const lines = await this.lineModel.findAll({
attributes: ["id", "lineCode", "lineName", "customerId"]
});
const lineMap = new Map<string, LineMatchInfo>();
for (const line of lines) {
if (!line.lineCode) continue;
lineMap.set(line.lineCode, {
lineId: line.id,
lineName: line.lineName || undefined,
customerId: line.customerId
});
}
// 本系统尚未同步任何线路(无 deviceCode 可查),直接写成功日志返回
if (lineMap.size === 0) {
await this.writeLog(startTime, 1, SYNC_TYPE, 0, null);
return result;
}
const token = await getIccAccessToken();
const url = `${baseUrl.replace(/\/$/, "")}${ICC_ANALOG_ENDPOINT.replace("{version}", process.env.ICC_VERSION?.trim() || DEFAULT_VERSION)}`;
// lineCodes 按 ≤DEVICE_BATCH 分批,逐批拉取模拟量
const allRaw: IccAnalogRawRecord[] = [];
const lineCodes = [...lineMap.keys()];
for (let i = 0; i < lineCodes.length; i += DEVICE_BATCH) {
const batch = lineCodes.slice(i, i + DEVICE_BATCH);
let pageNum = 1;
while (pageNum <= MAX_PAGES) {
const res = await iccFetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json",
Authorization: `bearer ${token}`
},
body: JSON.stringify({ pageNum, pageSize: PAGE_SIZE, deviceCodes: batch }),
signal: AbortSignal.timeout(HTTP_TIMEOUT_MS)
});
if (!res.ok) {
throw new Error(`ICC 模拟量接口返回异常:HTTP ${res.status}`);
}
const payload: any = await res.json();
const page = parseAnalogPage(payload);
allRaw.push(...page.list);
result.pulled += page.list.length;
// 短页判定:本页不满 pageSize 视为无下一页;另有最大页数保护
if (page.list.length < PAGE_SIZE) break;
pageNum++;
}
}
const { inserted, updated, skipped } = await this.batchUpsert(allRaw, lineMap, collectTime);
result.inserted = inserted;
result.updated = updated;
result.synced = inserted + updated;
result.skipped = skipped;
await this.writeLog(startTime, 1, SYNC_TYPE, result.synced, null);
return result;
} catch (err: any) {
const message = err instanceof Error ? err.message : String(err);
await this.writeLog(startTime, 0, SYNC_TYPE, result.pulled, message);
return result;
}
}
/** 按 (line_code, analog_type, collect_time) 幂等 upsert(collectTime 为本次同步时间);线路未同步的行跳过。 */
private async batchUpsert(
records: IccAnalogRawRecord[],
lineMap: Map<string, LineMatchInfo>,
collectTime: Date
): Promise<{ inserted: number; updated: number; skipped: number }> {
let inserted = 0;
let updated = 0;
let skipped = 0;
if (records.length === 0) return { inserted, updated, skipped };
for (const raw of records) {
const mapped = mapIccAnalogRecord(raw);
// deviceCode 必须在本系统已同步的线路中(line_code = ICC deviceCode),否则跳过
const lineInfo = lineMap.get(mapped.deviceCode);
if (!lineInfo) {
skipped++;
continue;
}
const payload = {
lineId: lineInfo.lineId,
lineCode: mapped.deviceCode,
imei: mapped.uniqueCode || null,
lineName: lineInfo.lineName || null,
customerId: lineInfo.customerId,
analogType: mapped.analogType || null,
analogName: mapped.analogName || null,
analogUnit: mapped.analogUnit || null,
analogValue: mapped.analogValue,
collectTime
} as any;
// 幂等 upsert:唯一键 (line_code, analog_type, collect_time)(collectTime 为本次同步时间),
// 同一次同步内同一 device+type 重复出现(如跨页)则覆盖更新
const existing = await this.electricityModel.findOne({
where: { lineCode: mapped.deviceCode, analogType: mapped.analogType || null, collectTime }
});
if (existing) {
await existing.update({
analogValue: mapped.analogValue,
imei: mapped.uniqueCode || null,
lineName: lineInfo.lineName || null,
analogName: mapped.analogName || null,
analogUnit: mapped.analogUnit || null
} as any);
updated++;
} else {
await this.electricityModel.create(payload);
inserted++;
}
}
return { inserted, updated, skipped };
}
private async writeLog(startTime: Date, status: number, syncType: string, recordCount: number, errorMessage: string | null): Promise<void> {
try {
await this.syncLogModel.create({
syncType,
startTime,
endTime: new Date(),
recordCount,
status,
errorMessage
} as any);
} catch (err) {
console.error("[ICC同步] 同步日志写入失败:", err);
}
}
}
@@ -11,6 +11,7 @@ import { NoticeRuleService } from "./notice_rule.service";
import { IccLineSyncService } from "./icc_line_sync.service";
import { IccCustomerSyncService } from "./icc_customer_sync.service";
import { IccGatewaySyncService } from "./icc_gateway_sync.service";
import { IccAnalogSyncService } from "./icc_analog_sync.service";
import { BuiltinJob, JobHandlerContext, JobRuntime, getBuiltinJob } from "../task/job_registry";
/** 任务执行结果(一次尝试) */
@@ -466,6 +467,12 @@ export class JobSchedulerService {
const iccGatewayService = (await ctx.requestContext.getAsync(IccGatewaySyncService)) as IccGatewaySyncService;
const result = await iccGatewayService.syncFromIcc();
return `拉取 ${result.pulled} 条,新增 ${result.inserted},更新 ${result.updated},跳过 ${result.skipped}`;
},
syncIccAnalogs: async () => {
const ctx = this.app.createAnonymousContext();
const iccAnalogService = (await ctx.requestContext.getAsync(IccAnalogSyncService)) as IccAnalogSyncService;
const result = await iccAnalogService.syncFromIcc();
return `拉取 ${result.pulled} 条,新增 ${result.inserted},更新 ${result.updated},跳过 ${result.skipped}`;
}
};
}
+104
View File
@@ -0,0 +1,104 @@
/**
* ICC 探测器模拟量数据字段映射(集中配置,便于按 ICC 平台实际返回结构调整)。
*
* ICC 返回的原始字段名 → 本系统 electricity_data 字段。接入真实 ICC 平台时
* 仅需调整本文件与 fetch 路径即可,业务层无需改动。纯映射逻辑,无 SQL 无 DOM。
*/
export interface IccAnalogRawRecord {
[field: string]: any;
}
/** 模拟量分页列表接口路径({version} 由调用方按 ICC_VERSION 替换) */
export const ICC_ANALOG_ENDPOINT = "/evo-apigw/evo-fdbu/{version}/analog/subsystem/page";
/** 原始记录关键字段映射:本系统字段 → ICC 返回字段名 */
export const ICC_ANALOG_FIELDS = {
deviceCode: "deviceCode", // 探测器编码(→ electric_line.line_code 幂等键)
uniqueCode: "uniqueCode", // 设备唯一标识码(→ electricity_data.imei)
analogType: "analogType", // 模拟量类型
analogName: "analogName", // 模拟量名称(→ electricity_data.analog_name)
analogUnit: "analogUnit", // 模拟量单位(→ electricity_data.analog_unit)
analogValue: "analogValue", // 模拟量值
reportTime: "reportTime" // 上报/采集时间(yyyy-MM-dd HH:mm:ss,接口文档声明字段,保留映射)
} as const;
/** 字符串 trim;null / undefined / 空串统一归一为 undefined。 */
function toOptionalString(value: unknown): string | undefined {
if (value === undefined || value === null) return undefined;
const s = String(value).trim();
return s === "" ? undefined : s;
}
/**
* 将 ICC 原始模拟量记录映射为本系统 electricity_data 落库字段。
* analogValue 用 Number 解析,空值/非法值统一归一为 null(可空落库)。
*/
export function mapIccAnalogRecord(raw: IccAnalogRawRecord): {
deviceCode: string;
uniqueCode?: string;
analogType?: string;
analogName?: string;
analogUnit?: string;
analogValue: number | null;
reportTime?: string;
} {
const f = ICC_ANALOG_FIELDS;
// 匹配键保持空串语义,由调用方判跳
const deviceCode = String(raw[f.deviceCode] ?? "").trim();
// 数值解析:空 / null / 非数值统一返回 null
let analogValue: number | null = null;
const rawValue = raw[f.analogValue];
if (rawValue !== undefined && rawValue !== null && String(rawValue).trim() !== "") {
const num = Number(rawValue);
analogValue = Number.isFinite(num) ? num : null;
}
return {
deviceCode,
uniqueCode: toOptionalString(raw[f.uniqueCode]),
analogType: toOptionalString(raw[f.analogType]),
analogName: toOptionalString(raw[f.analogName]),
analogUnit: toOptionalString(raw[f.analogUnit]),
analogValue,
// reportTime 保留映射(接口文档仍声明该字段;当前采集时间以同步时间为准,见同步服务)
reportTime: toOptionalString(raw[f.reportTime])
};
}
/**
* 解析模拟量分页响应(兼容三种形态):
* - payload 为数组 → 直接作为列表
* - payload.data.pageData 为数组 → 取 pageData
* - payload.data.list 为数组 → 取 list
* - 结构不匹配或解析异常 → 返回空列表
*/
export function parseAnalogPage(payload: any): {
list: IccAnalogRawRecord[];
nextPage?: number;
totalCount?: number;
} {
try {
if (Array.isArray(payload)) {
return { list: payload };
}
const page = payload?.data;
if (page && Array.isArray(page.pageData)) {
return {
list: page.pageData,
nextPage: Number(page.nextPage) > 0 ? Number(page.nextPage) : undefined,
totalCount: Number(page.totalCount) > 0 ? Number(page.totalCount) : undefined
};
}
if (page && Array.isArray(page.list)) {
return {
list: page.list,
nextPage: Number(page.nextPage) > 0 ? Number(page.nextPage) : undefined,
totalCount: Number(page.totalCount) > 0 ? Number(page.totalCount) : undefined
};
}
return { list: [] };
} catch {
return { list: [] };
}
}
+18
View File
@@ -0,0 +1,18 @@
import type { BuiltinJob } from "./job_registry";
/**
* 模拟量数据同步任务:从 ICC 平台按 deviceCode 分批拉取探测器最新模拟量,
* 以同步时间作采集时间写入 electricity_data(按 line_code/analog_type/collect_time 幂等),
* 写 icc_sync_log。默认每 5 分钟执行一次。
*/
export const iccAnalogSyncJob: BuiltinJob = {
jobKey: "icc_analog_sync",
jobName: "模拟量数据同步",
description: "拉取 ICC 平台探测器最新模拟量,以同步时间作采集时间写入 electricity_data(写 icc_sync_log)",
defaultCron: "*/5 * * * *",
defaultParams: {},
paramFields: [],
handler: async (_params, ctx) => {
return ctx.runtime.syncIccAnalogs();
}
};
+4 -1
View File
@@ -4,6 +4,7 @@ import { noticeScanJob } from "./notice_scan.task";
import { iccLineSyncJob } from "./icc_line_sync.task";
import { iccCustomerSyncJob } from "./icc_customer_sync.task";
import { iccGatewaySyncJob } from "./icc_gateway_sync.task";
import { iccAnalogSyncJob } from "./icc_analog_sync.task";
export type JobParamType = "number" | "string";
@@ -38,6 +39,8 @@ export interface JobRuntime {
syncIccCustomers(): Promise<string>;
/** 从 ICC 平台拉取智电改造网关设备并按 imei/deviceCode 幂等 upsert,返回执行摘要 */
syncIccGateways(): Promise<string>;
/** 从 ICC 平台按 deviceCode 分批拉取探测器模拟量并按 (line_code,analog_type,collect_time) 幂等 upsert,返回执行摘要 */
syncIccAnalogs(): Promise<string>;
}
export interface JobHandlerContext {
@@ -64,7 +67,7 @@ export interface BuiltinJob {
handler: JobHandler;
}
export const BUILTIN_JOBS: BuiltinJob[] = [demoHeartbeatJob, logCleanJob, noticeScanJob, iccLineSyncJob, iccCustomerSyncJob, iccGatewaySyncJob];
export const BUILTIN_JOBS: BuiltinJob[] = [demoHeartbeatJob, logCleanJob, noticeScanJob, iccLineSyncJob, iccCustomerSyncJob, iccGatewaySyncJob, iccAnalogSyncJob];
export function getBuiltinJob(jobKey: string): BuiltinJob | undefined {
return BUILTIN_JOBS.find(job => job.jobKey === jobKey);