dev #2

Merged
admin merged 176 commits from dev into master 2025-04-18 01:31:49 +00:00
12 changed files with 163 additions and 105 deletions
Showing only changes of commit a5f806c481 - Show all commits

View File

@ -57,7 +57,6 @@ namespace JiShe.CollectBus.Plugins
if (aFn.HasValue && fn.HasValue && aTuple != null && !string.IsNullOrWhiteSpace(aTuple.Item1)) if (aFn.HasValue && fn.HasValue && aTuple != null && !string.IsNullOrWhiteSpace(aTuple.Item1))
{ {
var tcpSessionClient = (ITcpSessionClient)client; var tcpSessionClient = (ITcpSessionClient)client;
if ((AFN)aFn == AFN.) if ((AFN)aFn == AFN.)
{ {

View File

@ -5,12 +5,12 @@ using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider namespace JiShe.CollectBus.IoTDBProvider
{ {
/// <summary> /// <summary>
/// Column分类标记特性TAG字段 /// Column分类标记特性ATTRIBUTE字段,也就是属性字段
/// </summary> /// </summary>
[AttributeUsage(AttributeTargets.Property)] [AttributeUsage(AttributeTargets.Property)]
public class TAGColumnAttribute : Attribute public class ATTRIBUTEColumnAttribute : Attribute
{ {
} }
} }

View File

@ -7,7 +7,7 @@ using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider namespace JiShe.CollectBus.IoTDBProvider
{ {
/// <summary> /// <summary>
/// Column分类标记特性FIELD字段 /// Column分类标记特性FIELD字段,数据列字段
/// </summary> /// </summary>
[AttributeUsage(AttributeTargets.Property)] [AttributeUsage(AttributeTargets.Property)]
public class FIELDColumnAttribute : Attribute public class FIELDColumnAttribute : Attribute

View File

@ -7,10 +7,10 @@ using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider namespace JiShe.CollectBus.IoTDBProvider
{ {
/// <summary> /// <summary>
/// Column分类标记特性ATTRIBUTE字段 /// Column分类标记特性TAG字段标签字段
/// </summary> /// </summary>
[AttributeUsage(AttributeTargets.Property)] [AttributeUsage(AttributeTargets.Property)]
public class ATTRIBUTEColumnAttribute : Attribute public class TAGColumnAttribute : Attribute
{ {
} }
} }

View File

@ -1,24 +0,0 @@
using Apache.IoTDB;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// 设备元数据
/// </summary>
public class DeviceMetadata
{
public List<string> Measurements { get; } = new();
public List<string> Tags { get; } = new();
public List<TSDataType> GetDataTypes()
{
// 根据实际类型映射TSDataType
return Measurements.Select(_ => TSDataType.TEXT).ToList();
}
}
}

View File

@ -1,19 +0,0 @@
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// IoT实体基类
/// </summary>
public abstract class IoTEntity
{
[TAGColumn]
public string SystemName { get; set; }
[TAGColumn]
public string ProjectCode { get; set; }
[TAGColumn]
public string DeviceId { get; set; }
public long Timestamp { get; set; }
}
}

View File

@ -6,7 +6,8 @@
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Apache.IoTDB" Version="1.3.3.1" /> <!--<PackageReference Include="Apache.IoTDB" Version="1.3.3.1" />-->
<PackageReference Include="Apache.IoTDB" Version="2.0.1" />
<PackageReference Include="Volo.Abp" Version="8.3.3" /> <PackageReference Include="Volo.Abp" Version="8.3.3" />
</ItemGroup> </ItemGroup>
</Project> </Project>

View File

@ -0,0 +1,30 @@
using Apache.IoTDB;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// 设备元数据
/// </summary>
public class DeviceMetadata
{
/// <summary>
/// 测量值集合用于构建Table的测量值也就是field参数
/// </summary>
public List<string> Measurements { get; } = new();
/// <summary>
/// 列类型集合用于构建Table的列类型也就是columnCategory参数
/// </summary>
public List<ColumnCategory> ColumnCategories { get; } = new();
/// <summary>
/// 值类型集合用于构建Table的值类型也就是dataType参数
/// </summary>
public List<TSDataType>DataTypes { get; } = new();
}
}

View File

@ -11,27 +11,29 @@ namespace JiShe.CollectBus.IoTDBProvider
/// </summary> /// </summary>
public static class DevicePathBuilder public static class DevicePathBuilder
{ {
/// <summary>
/// 构建存储组路径
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public static string BuildStorageGroupPath<T>() where T : IoTEntity
{
var type = typeof(T);
return $"root.{type.GetProperty("SystemName")?.Name}.{type.GetProperty("ProjectCode")?.Name}";
}
/// <summary> /// <summary>
/// 构建设备路径 /// 构建设备路径
/// </summary> /// </summary>
/// <typeparam name="T"></typeparam> /// <typeparam name="T"></typeparam>
/// <param name="entity"></param> /// <param name="entity"></param>
/// <returns></returns> /// <returns></returns>
public static string BuildDevicePath<T>(T entity) where T : IoTEntity public static string GetDeviceId<T>(T entity) where T : IoTEntity
{ {
return $"root.{entity.SystemName}.{entity.ProjectCode}.{entity.DeviceId}"; return $"root.{entity.SystemName}.{entity.ProjectCode}.{entity.DeviceId}";
} }
/// <summary>
/// 获取表名称
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entity"></param>
/// <returns></returns>
public static string GetTableName<T>() where T : IoTEntity
{
var type = typeof(T);
return $"{type.Name}";
}
} }
} }

View File

@ -17,17 +17,20 @@ namespace JiShe.CollectBus.IoTDBProvider
public class IoTDBProvider : IIoTDBProvider, IDisposable public class IoTDBProvider : IIoTDBProvider, IDisposable
{ {
private readonly IoTDBOptions _options; private readonly IoTDBOptions _options;
private readonly SessionPool _sessionPool; private readonly TableSessionPool _sessionPool;
private static readonly ConcurrentDictionary<Type, DeviceMetadata> _metadataCache = new(); private static readonly ConcurrentDictionary<Type, DeviceMetadata> _metadataCache = new();
public IoTDBProvider(IOptions<IoTDBOptions> options) public IoTDBProvider(IOptions<IoTDBOptions> options)
{ {
_options = options.Value; _options = options.Value;
_sessionPool = new SessionPool(
_options.ClusterList, _sessionPool = new TableSessionPool.Builder()
_options.UserName, .SetNodeUrls(_options.ClusterList)
_options.Password, .SetUsername(_options.UserName)
_options.PoolSize); .SetPassword(_options.Password)
.SetFetchSize(_options.PoolSize)
.Build();
_sessionPool.Open(false).Wait(); _sessionPool.Open(false).Wait();
} }
@ -43,14 +46,27 @@ namespace JiShe.CollectBus.IoTDBProvider
var metadata = new DeviceMetadata(); var metadata = new DeviceMetadata();
foreach (var prop in type.GetProperties()) foreach (var prop in type.GetProperties())
{ {
var attr = prop.GetCustomAttribute<ColumnCategoryAttribute>(); //标签列
if (attr != null) var attrTAG = prop.GetCustomAttribute<TAGColumnAttribute>();
if (attrTAG != null)
{ {
metadata.Tags.Add(prop.Name); metadata.ColumnCategories.Add(ColumnCategory.TAG);
} }
else if (prop.Name != nameof(IoTEntity.Timestamp))
//属性列
var attrATTRIBUTE = prop.GetCustomAttribute<ATTRIBUTEColumnAttribute>();
if (attrATTRIBUTE != null)
{ {
metadata.ColumnCategories.Add(ColumnCategory.ATTRIBUTE);
}
//数据列
var attrFIELD = prop.GetCustomAttribute<FIELDColumnAttribute>();
if (attrFIELD != null)
{
metadata.ColumnCategories.Add(ColumnCategory.FIELD);
metadata.Measurements.Add(prop.Name); metadata.Measurements.Add(prop.Name);
metadata.DataTypes.Add(GetDataTypeFromStr(prop.PropertyType.Name));
} }
} }
return metadata; return metadata;
@ -66,11 +82,9 @@ namespace JiShe.CollectBus.IoTDBProvider
public async Task InsertAsync<T>(T entity) where T : IoTEntity public async Task InsertAsync<T>(T entity) where T : IoTEntity
{ {
var metadata = GetMetadata<T>(); var metadata = GetMetadata<T>();
var storageGroup = DevicePathBuilder.BuildStorageGroupPath<T>();
await EnsureStorageGroupCreated(storageGroup);
var tablet = BuildTablet(new[] { entity }, metadata); var tablet = BuildTablet(new[] { entity }, metadata);
await _sessionPool.InsertAlignedTabletAsync(tablet); await _sessionPool.InsertAsync(tablet);
} }
/// <summary> /// <summary>
@ -81,9 +95,7 @@ namespace JiShe.CollectBus.IoTDBProvider
/// <returns></returns> /// <returns></returns>
public async Task BatchInsertAsync<T>(IEnumerable<T> entities) where T : IoTEntity public async Task BatchInsertAsync<T>(IEnumerable<T> entities) where T : IoTEntity
{ {
var metadata = GetMetadata<T>(); var metadata = GetMetadata<T>();
var storageGroup = DevicePathBuilder.BuildStorageGroupPath<T>();
await EnsureStorageGroupCreated(storageGroup);
var batchSize = 1000; var batchSize = 1000;
var batches = entities.Chunk(batchSize); var batches = entities.Chunk(batchSize);
@ -91,7 +103,7 @@ namespace JiShe.CollectBus.IoTDBProvider
foreach (var batch in batches) foreach (var batch in batches)
{ {
var tablet = BuildTablet(batch, metadata); var tablet = BuildTablet(batch, metadata);
await _sessionPool.InsertAlignedTabletAsync(tablet); await _sessionPool.InsertAsync(tablet);
} }
} }
@ -104,34 +116,33 @@ namespace JiShe.CollectBus.IoTDBProvider
/// <returns></returns> /// <returns></returns>
private Tablet BuildTablet<T>(IEnumerable<T> entities, DeviceMetadata metadata) where T : IoTEntity private Tablet BuildTablet<T>(IEnumerable<T> entities, DeviceMetadata metadata) where T : IoTEntity
{ {
var devicePath = DevicePathBuilder.BuildDevicePath(entities.First()); var deviceId = DevicePathBuilder.GetDeviceId(entities.First());
var timestamps = new List<long>(); var timestamps = new List<long>();
var values = new List<List<object>>(); var values = new List<List<object>>();
foreach (var entity in entities) foreach (var entity in entities)
{ {
timestamps.Add(entity.Timestamp); timestamps.Add(entity.Timestamps);
var rowValues = new List<object>(); var rowValues = new List<object>();
foreach (var measurement in metadata.Measurements) foreach (var measurement in metadata.Measurements)
{ {
var value = typeof(T).GetProperty(measurement)?.GetValue(entity); var value = typeof(T).GetProperty(measurement)?.GetValue(entity);
rowValues.Add(value ?? DBNull.Value); if(value == null)
{
throw new Exception($"{nameof(BuildTablet)} 构建表模型{typeof(T).Name}时,属性{measurement}值为空不符合IoTDB设计标准请赋值以后重新处理。");
}
rowValues.Add(value);
} }
values.Add(rowValues); values.Add(rowValues);
} }
return new Tablet( return new Tablet(
devicePath, deviceId,
metadata.Measurements, metadata.Measurements,
metadata.GetDataTypes(), metadata.DataTypes,
values, values,
timestamps timestamps
) );
{
Tags = metadata.Tags.ToDictionary(
t => t,
t => typeof(T).GetProperty(t)?.GetValue(entities.First())?.ToString())
};
} }
/// <summary> /// <summary>
@ -165,7 +176,7 @@ namespace JiShe.CollectBus.IoTDBProvider
var metadata = GetMetadata<T>(); var metadata = GetMetadata<T>();
var sb = new StringBuilder("SELECT "); var sb = new StringBuilder("SELECT ");
sb.AppendJoin(", ", metadata.Measurements); sb.AppendJoin(", ", metadata.Measurements);
sb.Append($" FROM {DevicePathBuilder.BuildStorageGroupPath<T>()}"); sb.Append($" FROM {DevicePathBuilder.GetTableName<T>()}");
if (options.Conditions.Any()) if (options.Conditions.Any())
{ {
@ -202,7 +213,7 @@ namespace JiShe.CollectBus.IoTDBProvider
/// <returns></returns> /// <returns></returns>
private async Task<int> GetTotalCount<T>(QueryOptions options) where T : IoTEntity private async Task<int> GetTotalCount<T>(QueryOptions options) where T : IoTEntity
{ {
var countQuery = $"SELECT COUNT(*) FROM {DevicePathBuilder.BuildStorageGroupPath<T>()}"; var countQuery = $"SELECT COUNT(*) FROM {DevicePathBuilder.GetTableName<T>()}";
if (options.Conditions.Any()) if (options.Conditions.Any())
{ {
countQuery += " WHERE " + string.Join(" AND ", options.Conditions.Select(TranslateCondition)); countQuery += " WHERE " + string.Join(" AND ", options.Conditions.Select(TranslateCondition));
@ -224,33 +235,35 @@ namespace JiShe.CollectBus.IoTDBProvider
var results = new List<T>(); var results = new List<T>();
var metadata = GetMetadata<T>(); var metadata = GetMetadata<T>();
var properties = typeof(T).GetProperties();
while (dataSet.HasNext() && results.Count < pageSize) while (dataSet.HasNext() && results.Count < pageSize)
{ {
var record = dataSet.Next(); var record = dataSet.Next();
var entity = new T var entity = new T
{ {
Timestamp = record.Timestamps Timestamps = record.Timestamps
}; };
foreach (var measurement in metadata.Measurements) foreach (var measurement in metadata.Measurements)
{ {
var value = record.GetValue(measurement); var value = record.Values;
typeof(T).GetProperty(measurement)?.SetValue(entity, value);
var prop = properties.FirstOrDefault(p =>
p.Name.Equals(measurement, StringComparison.OrdinalIgnoreCase));
if (prop != null)
{
typeof(T).GetProperty(measurement)?.SetValue(entity, value);
}
} }
results.Add(entity); results.Add(entity);
} }
return results; return results;
} }
private async Task EnsureStorageGroupCreated(string storageGroup)
{
if (!await _sessionPool.CheckStorageGroupExists(storageGroup))
{
await _sessionPool.SetStorageGroupAsync(storageGroup);
}
}
/// <summary> /// <summary>
/// 释放资源 /// 释放资源
/// </summary> /// </summary>
@ -258,5 +271,24 @@ namespace JiShe.CollectBus.IoTDBProvider
{ {
_sessionPool?.Close().Wait(); _sessionPool?.Close().Wait();
} }
private TSDataType GetDataTypeFromStr(string str)
{
return str switch
{
"BOOLEAN" => TSDataType.BOOLEAN,
"INT32" => TSDataType.INT32,
"INT64" => TSDataType.INT64,
"FLOAT" => TSDataType.FLOAT,
"DOUBLE" => TSDataType.DOUBLE,
"TEXT" => TSDataType.TEXT,
"NULLTYPE" => TSDataType.NONE,
"TIMESTAMP" => TSDataType.TIMESTAMP,
"DATE" => TSDataType.DATE,
"BLOB" => TSDataType.BLOB,
"STRING" => TSDataType.STRING,
_ => TSDataType.STRING
};
}
} }
} }

View File

@ -0,0 +1,37 @@
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// IoT实体基类
/// </summary>
public abstract class IoTEntity
{
/// <summary>
/// 系统名称
/// </summary>
[TAGColumn]
public string SystemName { get; set; }
/// <summary>
/// 项目编码
/// </summary>
[TAGColumn]
public string ProjectCode { get; set; }
/// <summary>
/// 设备类型集中器、电表、水表、流量计、传感器等
/// </summary>
[TAGColumn]
public string DeviceType { get; set; }
/// <summary>
/// 设备ID
/// </summary>
[TAGColumn]
public string DeviceId { get; set; }
/// <summary>
/// 时间戳
/// </summary>
public long Timestamps { get; set; }
}
}