admin/JiShe.CollectBus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

完善IoTDB的数据实体约束

Browse Source
This commit is contained in:
ChenYi 2025-04-21 09:45:30 +08:00
parent fe51402b1f
commit cc59b32742
11 changed files with 192 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
modules/JiShe.CollectBus.IoTDB/Attribute/EntityTypeAttribute.cs Normal file
View File

@ -0,0 +1,19 @@
using JiShe.CollectBus.IoTDB.Enums;
namespace JiShe.CollectBus.IoTDB.Attribute
{
/// <summary>
/// IoTDB实体类型特性
/// </summary>
[AttributeUsage(AttributeTargets.Class)]
public class EntityTypeAttribute : System.Attribute
{
public EntityTypeEnum EntityType { get; }
public EntityTypeAttribute(EntityTypeEnum entityType)
{
EntityType = entityType;
}
}
}

2
modules/JiShe.CollectBus.IoTDB/Context/IoTDBRuntimeContext.cs
View File

@ -17,7 +17,7 @@ namespace JiShe.CollectBus.IoTDB.Context
}
/// <summary>
/// 是否使用表模型存储, 默认false，使用tree模型存储
/// 存储模型切换标识，是否使用table模型存储, 默认为false，标识tree模型存储
/// </summary>
public bool UseTableSessionPool { get; set; }

24
modules/JiShe.CollectBus.IoTDB/EnumInfo/EntityTypeEnum.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDB.Enums
{
/// <summary>
/// IoTDB实体类型枚举
/// </summary>
public enum EntityTypeEnum
{
/// <summary>
/// 树模型
/// </summary>
TreeModel = 1,
/// <summary>
/// 表模型
/// </summary>
TableModel = 2,
}
}

6
modules/JiShe.CollectBus.IoTDB/Provider/DeviceMetadata.cs
View File

@ -1,4 +1,5 @@
using Apache.IoTDB;
using JiShe.CollectBus.IoTDB.Enums;
namespace JiShe.CollectBus.IoTDB.Provider
{
@ -7,6 +8,11 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// </summary>
public class DeviceMetadata
{
/// <summary>
/// IoTDB实体类型枚举
/// </summary>
public EntityTypeEnum EntityType { get; set; }
/// <summary>
/// 是否有单测量值
/// </summary>

2
modules/JiShe.CollectBus.IoTDB/Provider/DevicePathBuilder.cs
View File

@ -13,7 +13,7 @@
/// <returns></returns>
public static string GetDevicePath<T>(T entity) where T : IoTEntity
{
return $"root.{entity.SystemName.ToLower()}.`{entity.ProjectCode}`.`{entity.DeviceId}`";
return $"root.{entity.SystemName.ToLower()}.`{entity.ProjectCode}`.`{entity.DeviceType}`.`{entity.DeviceId}`";
}

206
modules/JiShe.CollectBus.IoTDB/Provider/IoTDBProvider.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using System;
using System.Collections.Concurrent;
using System.Reflection;
using System.Text;
using Apache.IoTDB;
@ -9,6 +10,7 @@ using JiShe.CollectBus.IoTDB.Context;
using JiShe.CollectBus.IoTDB.Interface;
using JiShe.CollectBus.IoTDB.Options;
using Microsoft.Extensions.Logging;
using Volo.Abp.Domain.Entities;
namespace JiShe.CollectBus.IoTDB.Provider
{
@ -139,66 +141,98 @@ namespace JiShe.CollectBus.IoTDB.Provider
List<string> tempColumnNames = new List<string>();
tempColumnNames.AddRange(metadata.ColumnNames);
foreach (var entity in entities)
var entityTypeAttribute = typeof(T).GetCustomAttribute<EntityTypeAttribute>();
if (entityTypeAttribute == null)
{
timestamps.Add(entity.Timestamps);
var rowValues = new List<object>();
foreach (var measurement in tempColumnNames)
throw new ArgumentException($"{nameof(BuildTablet)} 构建存储结构{nameof(Tablet)}时 {nameof(T)}的EntityType 没有指定，属于异常情况，-101");
}
if (metadata.EntityType != entityTypeAttribute.EntityType)
{
throw new ArgumentException($"{nameof(BuildTablet)} 构建存储结构{nameof(Tablet)}时 {nameof(T)}的EntityType 和{nameof(DeviceMetadata)}的 EntityType 不一致，属于异常情况，-102");
}
if(metadata.EntityType == Enums.EntityTypeEnum.TreeModel && _runtimeContext.UseTableSessionPool == true)
{
throw new ArgumentException($"{nameof(BuildTablet)} 构建存储结构{nameof(Tablet)}时 tree模型不能使用table模型Session连接，属于异常情况，-103");
}
else if (metadata.EntityType == Enums.EntityTypeEnum.TableModel && _runtimeContext.UseTableSessionPool == false)
{
throw new ArgumentException($"{nameof(BuildTablet)} 构建存储结构{nameof(Tablet)}时 table模型不能使用tree模型Session连接，属于异常情况，-104");
}
foreach (var entity in entities)
{
timestamps.Add(entity.Timestamps);
var rowValues = new List<object>();
PropertyInfo propertyInfo = typeof(T).GetProperty(measurement);
if (propertyInfo == null)
foreach (var measurement in tempColumnNames)
{
throw new Exception($"{nameof(BuildTablet)} 构建表模型{typeof(T).Name}时，没有找到{measurement}属性，属于异常情况，-101。");
}
var value = propertyInfo.GetValue(entity);
if (propertyInfo.IsDefined(typeof(SingleMeasuringAttribute), false) && value != null)//表示当前对象是单测点模式
{
Type tupleType = value.GetType();
Type[] tupleArgs = tupleType.GetGenericArguments();
Type item2Type = tupleArgs[1]; // T 的实际类型
var item1 = tupleType.GetProperty("Item1")!.GetValue(value);
var item2 = tupleType.GetProperty("Item2")!.GetValue(value);
if (item1 == null || item2 == null)
PropertyInfo propertyInfo = typeof(T).GetProperty(measurement);
if (propertyInfo == null)
{
throw new Exception($"{nameof(BuildTablet)} 构建表模型{typeof(T).Name}时，单测点模式构建失败，没有获取测点名称或者测点值，-102。");
throw new Exception($"{nameof(BuildTablet)} 构建表模型{typeof(T).Name}时，没有找到{measurement}属性，属于异常情况，-101。");
}
var indexOf = metadata.ColumnNames.IndexOf(measurement);
metadata.ColumnNames[indexOf] = (string)item1!;
rowValues.Add(item2);
}
else
{
if (value != null)
var value = propertyInfo.GetValue(entity);
if (propertyInfo.IsDefined(typeof(SingleMeasuringAttribute), false) && metadata.IsSingleMeasuring == true)//表示当前对象是单测点模式
{
rowValues.Add(value);
if (value != null)
{
Type tupleType = value.GetType();
Type[] tupleArgs = tupleType.GetGenericArguments();
Type item2Type = tupleArgs[1]; // T 的实际类型
var item1 = tupleType.GetProperty("Item1")!.GetValue(value);
var item2 = tupleType.GetProperty("Item2")!.GetValue(value);
if (item1 == null || item2 == null)
{
throw new Exception($"{nameof(BuildTablet)} 构建表模型{typeof(T).Name}时，单测点模式构建失败，没有获取测点名称或者测点值，-102。");
}
var indexOf = metadata.ColumnNames.IndexOf(measurement);
metadata.ColumnNames[indexOf] = (string)item1!;
rowValues.Add(item2);
}
else
{
rowValues.Add(null);
}
}
else
{
//填充默认数据值
DataTypeDefaultValueMap.TryGetValue(propertyInfo.PropertyType.Name, out object defaultValue);
rowValues.Add(defaultValue);
rowValues.Add(value);
//if (value != null)
//{
// rowValues.Add(value);
//}
//else
//{
// ////填充默认数据值
// //DataTypeDefaultValueMap.TryGetValue(propertyInfo.PropertyType.Name, out object defaultValue);
// rowValues.Add(null);
//}
}
}
}
values.Add(rowValues);
values.Add(rowValues);
if (!_runtimeContext.UseTableSessionPool)//树模型
{
devicePaths.Add(DevicePathBuilder.GetDevicePath(entity));
if (!_runtimeContext.UseTableSessionPool)//树模型
{
devicePaths.Add(DevicePathBuilder.GetDevicePath(entity));
}
else
{
devicePaths.Add(DevicePathBuilder.GetTableName<T>());
}
}
else
{
devicePaths.Add(DevicePathBuilder.GetTableName<T>());
}
}
if (devicePaths.Count > 1)
{
@ -213,14 +247,16 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary>
/// 构建tree模型的Tablet
/// </summary>
/// <param name="metadata"></param>
/// <param name="devicePath"></param>
/// <param name="values"></param>
/// <param name="timestamps"></param>
/// <param name="metadata">已解析的设备数据元数据</param>
/// <param name="devicePath">设备路径</param>
/// <param name="values">数据集合</param>
/// <param name="timestamps">时间戳集合</param>
/// <returns></returns>
private Tablet BuildSessionTablet(DeviceMetadata metadata, string devicePath,
List<List<object>> values, List<long> timestamps)
{
//todo 树模型需要去掉TAG类型和ATTRIBUTE类型的字段，只需要保留FIELD类型字段即可
return new Tablet(
devicePath,
metadata.ColumnNames,
@ -233,16 +269,16 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary>
/// 构建表模型的Tablet
/// </summary>
/// <param name="metadata"></param>
/// <param name="devicePath"></param>
/// <param name="values"></param>
/// <param name="timestamps"></param>
/// <param name="metadata">已解析的设备数据元数据</param>
/// <param name="tableName">表名称</param>
/// <param name="values">数据集合</param>
/// <param name="timestamps">时间戳集合</param>
/// <returns></returns>
private Tablet BuildTableSessionTablet(DeviceMetadata metadata, string devicePath,
private Tablet BuildTableSessionTablet(DeviceMetadata metadata, string tableName,
List<List<object>> values, List<long> timestamps)
{
var tablet = new Tablet(
devicePath,
tableName,
metadata.ColumnNames,
metadata.ColumnCategories,
metadata.DataTypes,
@ -393,33 +429,35 @@ namespace JiShe.CollectBus.IoTDB.Provider
private DeviceMetadata GetMetadata<T>() where T : IoTEntity
{
var columns = CollectColumnMetadata(typeof(T));
var metadata = BuildDeviceMetadata(columns);
return _metadataCache.AddOrUpdate(
typeof(T),
addValueFactory: t => metadata, // 如果键不存在，用此值添加
updateValueFactory: (t, existingValue) =>
if (_runtimeContext.UseTableSessionPool)//表模型
{
return _metadataCache.GetOrAdd(typeof(T), type =>
{
var columns = CollectColumnMetadata(t);
var metadata = BuildDeviceMetadata(columns);
var columns = CollectColumnMetadata(type);
var metadata = BuildDeviceMetadata<T>(columns);
return metadata;
});
}
else
{
// 树模型
var columns = CollectColumnMetadata(typeof(T));
var metadata = BuildDeviceMetadata<T>(columns);
//对现有值 existingValue 进行修改，返回新值
existingValue.ColumnNames = metadata.ColumnNames;
return existingValue;
}
);
return _metadataCache.AddOrUpdate(
typeof(T),
addValueFactory: t => metadata, // 如果键不存在，用此值添加
updateValueFactory: (t, existingValue) =>
{
var columns = CollectColumnMetadata(t);
var metadata = BuildDeviceMetadata<T>(columns);
//return _metadataCache.GetOrAdd(typeof(T), type =>
//{
// var columns = CollectColumnMetadata(type);
// var metadata = BuildDeviceMetadata(columns);
// //if (metadata.IsSingleMeasuring)
// //{
// // _metadataCache.Remove(typeof(T));
// //}
// return metadata;
//});
//对现有值 existingValue 进行修改，返回新值
existingValue.ColumnNames = metadata.ColumnNames;
return existingValue;
}
);
}
}
/// <summary>
@ -483,9 +521,10 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary>
/// 构建设备元数据
/// </summary>
/// <param name="columns"></param>
/// <param name="typeInfo">待解析的类</param>
/// <param name="columns">已处理好的数据列</param>
/// <returns></returns>
private DeviceMetadata BuildDeviceMetadata(List<ColumnInfo> columns)
private DeviceMetadata BuildDeviceMetadata<T>(List<ColumnInfo> columns) where T : IoTEntity
{
var metadata = new DeviceMetadata();
@ -504,6 +543,15 @@ namespace JiShe.CollectBus.IoTDB.Provider
ProcessCategory(groupedColumns, ColumnCategory.ATTRIBUTE, metadata);
ProcessCategory(groupedColumns, ColumnCategory.FIELD, metadata);
var entityTypeAttribute = typeof(T).GetCustomAttribute<EntityTypeAttribute>();
if (entityTypeAttribute == null)
{
throw new ArgumentException($"{nameof(BuildDeviceMetadata)} 构建设备元数据时 {nameof(IoTEntity)}的EntityType 没有指定，属于异常情况，-101");
}
metadata.EntityType = entityTypeAttribute.EntityType;
return metadata;
}

10
modules/JiShe.CollectBus.IoTDB/Provider/IoTEntity.cs
View File

@ -11,29 +11,29 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// 系统名称
/// </summary>
[TAGColumn]
public string SystemName { get; set; }
public required string SystemName { get; set; }
/// <summary>
/// 项目编码
/// </summary>
[TAGColumn]
public string ProjectCode { get; set; }
public required string ProjectCode { get; set; }
/// <summary>
/// 设备类型集中器、电表、水表、流量计、传感器等
/// </summary>
[TAGColumn]
public string DeviceType { get; set; }
public required string DeviceType { get; set; }
/// <summary>
/// 设备ID
/// </summary>
[TAGColumn]
public string DeviceId { get; set; }
public required string DeviceId { get; set; }
/// <summary>
/// 当前时间戳，单位毫秒
/// </summary>
public long Timestamps { get; set; } = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
public required long Timestamps { get; set; } = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
}

2
services/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs
View File

@ -42,6 +42,8 @@ namespace JiShe.CollectBus.RedisDataCache
Instance = _freeRedisProvider.Instance;
}
//todo 单个数据查询
/// <summary>
/// 单个添加数据
/// </summary>

3
services/JiShe.CollectBus.Application/Samples/SampleAppService.cs
View File

@ -105,6 +105,7 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS
await _iotDBProvider.InsertAsync(meter2);
_dbContext.UseTableSessionPool = true;
var testTime = Convert.ToDateTime("2025-04-21 08:32:55");
ElectricityMeter meter = new ElectricityMeter()
{
@ -115,7 +116,7 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS
MeterModel = "DDZY-1980",
ProjectCode = "10059",
Voltage = 10,
Timestamps = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
Timestamps = new DateTimeOffset(testTime).ToUnixTimeMilliseconds(),
};
await _iotDBProvider.InsertAsync(meter);
}

2
services/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs
View File

@ -27,8 +27,6 @@ using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using static FreeSql.Internal.GlobalFilter;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace JiShe.CollectBus.ScheduledMeterReading
{

3
services/JiShe.CollectBus.Domain/Ammeters/ElectricityMeter.cs
View File

@ -33,5 +33,8 @@ namespace JiShe.CollectBus.Ammeters
[FIELDColumn]
public double Power => Voltage * Current;
[FIELDColumn]
public double? Currentd { get; set; }
}
}