admin/JiShe.CollectBus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

dev #2

Merged
admin merged 176 commits from dev into master 2025-04-18 01:31:49 +00:00
Conversation 0 Commits 176 Files Changed 427 +163 -105
12 changed files with 163 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a5f806c481 - Show all commits

1
src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs
View File

@ -58,7 +58,6 @@ namespace JiShe.CollectBus.Plugins
{
var tcpSessionClient = (ITcpSessionClient)client;
if ((AFN)aFn == AFN.链路接口检测)
{
switch (fn)

4
src/JiShe.CollectBus.IoTDBProvider/Attribute/ATTRIBUTEColumnAttribute.cs
View File

@ -7,10 +7,10 @@ using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// Column分类标记特性（TAG字段）
/// Column分类标记特性（ATTRIBUTE字段）,也就是属性字段
/// </summary>
[AttributeUsage(AttributeTargets.Property)]
public class TAGColumnAttribute : Attribute
public class ATTRIBUTEColumnAttribute : Attribute
{
}
}

2
src/JiShe.CollectBus.IoTDBProvider/Attribute/FIELDColumnAttribute.cs
View File

@ -7,7 +7,7 @@ using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// Column分类标记特性（FIELD字段）
/// Column分类标记特性（FIELD字段），数据列字段
/// </summary>
[AttributeUsage(AttributeTargets.Property)]
public class FIELDColumnAttribute : Attribute

4
src/JiShe.CollectBus.IoTDBProvider/Attribute/TAGColumnAttribute.cs
View File

@ -7,10 +7,10 @@ using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// Column分类标记特性（ATTRIBUTE字段）
/// Column分类标记特性（TAG字段），标签字段
/// </summary>
[AttributeUsage(AttributeTargets.Property)]
public class ATTRIBUTEColumnAttribute : Attribute
public class TAGColumnAttribute : Attribute
{
}
}

24
src/JiShe.CollectBus.IoTDBProvider/DeviceMetadata.cs
View File

@ -1,24 +0,0 @@
using Apache.IoTDB;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// 设备元数据
/// </summary>
public class DeviceMetadata
{
public List<string> Measurements { get; } = new();
public List<string> Tags { get; } = new();
public List<TSDataType> GetDataTypes()
{
// 根据实际类型映射TSDataType
return Measurements.Select(_ => TSDataType.TEXT).ToList();
}
}
}

0
src/JiShe.CollectBus.IoTDBProvider/IIoTDBProvider.cs → src/JiShe.CollectBus.IoTDBProvider/Interface/IIoTDBProvider.cs
View File

19
src/JiShe.CollectBus.IoTDBProvider/IoTEntity.cs
View File

@ -1,19 +0,0 @@
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// IoT实体基类
/// </summary>
public abstract class IoTEntity
{
[TAGColumn]
public string SystemName { get; set; }
[TAGColumn]
public string ProjectCode { get; set; }
[TAGColumn]
public string DeviceId { get; set; }
public long Timestamp { get; set; }
}
}

3
src/JiShe.CollectBus.IoTDBProvider/JiShe.CollectBus.IoTDBProvider.csproj
View File

@ -6,7 +6,8 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Apache.IoTDB" Version="1.3.3.1" />
<!--<PackageReference Include="Apache.IoTDB" Version="1.3.3.1" />-->
<PackageReference Include="Apache.IoTDB" Version="2.0.1" />
<PackageReference Include="Volo.Abp" Version="8.3.3" />
</ItemGroup>
</Project>

30
src/JiShe.CollectBus.IoTDBProvider/Provider/DeviceMetadata.cs Normal file
View File

@ -0,0 +1,30 @@
using Apache.IoTDB;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// 设备元数据
/// </summary>
public class DeviceMetadata
{
/// <summary>
/// 测量值集合，用于构建Table的测量值，也就是field参数
/// </summary>
public List<string> Measurements { get; } = new();
/// <summary>
/// 列类型集合，用于构建Table的列类型，也就是columnCategory参数
/// </summary>
public List<ColumnCategory> ColumnCategories { get; } = new();
/// <summary>
/// 值类型集合，用于构建Table的值类型，也就是dataType参数
/// </summary>
public List<TSDataType>DataTypes { get; } = new();
}
}

26
src/JiShe.CollectBus.IoTDBProvider/DevicePathBuilder.cs → src/JiShe.CollectBus.IoTDBProvider/Provider/DevicePathBuilder.cs
View File

@ -11,27 +11,29 @@ namespace JiShe.CollectBus.IoTDBProvider
/// </summary>
public static class DevicePathBuilder
{
/// <summary>
/// 构建存储组路径
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public static string BuildStorageGroupPath<T>() where T : IoTEntity
{
var type = typeof(T);
return $"root.{type.GetProperty("SystemName")?.Name}.{type.GetProperty("ProjectCode")?.Name}";
}
/// <summary>
/// 构建设备路径
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entity"></param>
/// <returns></returns>
public static string BuildDevicePath<T>(T entity) where T : IoTEntity
public static string GetDeviceId<T>(T entity) where T : IoTEntity
{
return $"root.{entity.SystemName}.{entity.ProjectCode}.{entity.DeviceId}";
}
/// <summary>
/// 获取表名称
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entity"></param>
/// <returns></returns>
public static string GetTableName<T>() where T : IoTEntity
{
var type = typeof(T);
return $"{type.Name}";
}
}
}

110
src/JiShe.CollectBus.IoTDBProvider/IoTDBProvider.cs → src/JiShe.CollectBus.IoTDBProvider/Provider/IoTDBProvider.cs
View File

@ -17,17 +17,20 @@ namespace JiShe.CollectBus.IoTDBProvider
public class IoTDBProvider : IIoTDBProvider, IDisposable
{
private readonly IoTDBOptions _options;
private readonly SessionPool _sessionPool;
private readonly TableSessionPool _sessionPool;
private static readonly ConcurrentDictionary<Type, DeviceMetadata> _metadataCache = new();
public IoTDBProvider(IOptions<IoTDBOptions> options)
{
_options = options.Value;
_sessionPool = new SessionPool(
_options.ClusterList,
_options.UserName,
_options.Password,
_options.PoolSize);
_sessionPool = new TableSessionPool.Builder()
.SetNodeUrls(_options.ClusterList)
.SetUsername(_options.UserName)
.SetPassword(_options.Password)
.SetFetchSize(_options.PoolSize)
.Build();
_sessionPool.Open(false).Wait();
}
@ -43,14 +46,27 @@ namespace JiShe.CollectBus.IoTDBProvider
var metadata = new DeviceMetadata();
foreach (var prop in type.GetProperties())
{
var attr = prop.GetCustomAttribute<ColumnCategoryAttribute>();
if (attr != null)
//标签列
var attrTAG = prop.GetCustomAttribute<TAGColumnAttribute>();
if (attrTAG != null)
{
metadata.Tags.Add(prop.Name);
metadata.ColumnCategories.Add(ColumnCategory.TAG);
}
else if (prop.Name != nameof(IoTEntity.Timestamp))
//属性列
var attrATTRIBUTE = prop.GetCustomAttribute<ATTRIBUTEColumnAttribute>();
if (attrATTRIBUTE != null)
{
metadata.ColumnCategories.Add(ColumnCategory.ATTRIBUTE);
}
//数据列
var attrFIELD = prop.GetCustomAttribute<FIELDColumnAttribute>();
if (attrFIELD != null)
{
metadata.ColumnCategories.Add(ColumnCategory.FIELD);
metadata.Measurements.Add(prop.Name);
metadata.DataTypes.Add(GetDataTypeFromStr(prop.PropertyType.Name));
}
}
return metadata;
@ -66,11 +82,9 @@ namespace JiShe.CollectBus.IoTDBProvider
public async Task InsertAsync<T>(T entity) where T : IoTEntity
{
var metadata = GetMetadata<T>();
var storageGroup = DevicePathBuilder.BuildStorageGroupPath<T>();
await EnsureStorageGroupCreated(storageGroup);
var tablet = BuildTablet(new[] { entity }, metadata);
await _sessionPool.InsertAlignedTabletAsync(tablet);
await _sessionPool.InsertAsync(tablet);
}
/// <summary>
@ -82,8 +96,6 @@ namespace JiShe.CollectBus.IoTDBProvider
public async Task BatchInsertAsync<T>(IEnumerable<T> entities) where T : IoTEntity
{
var metadata = GetMetadata<T>();
var storageGroup = DevicePathBuilder.BuildStorageGroupPath<T>();
await EnsureStorageGroupCreated(storageGroup);
var batchSize = 1000;
var batches = entities.Chunk(batchSize);
@ -91,7 +103,7 @@ namespace JiShe.CollectBus.IoTDBProvider
foreach (var batch in batches)
{
var tablet = BuildTablet(batch, metadata);
await _sessionPool.InsertAlignedTabletAsync(tablet);
await _sessionPool.InsertAsync(tablet);
}
}
@ -104,34 +116,33 @@ namespace JiShe.CollectBus.IoTDBProvider
/// <returns></returns>
private Tablet BuildTablet<T>(IEnumerable<T> entities, DeviceMetadata metadata) where T : IoTEntity
{
var devicePath = DevicePathBuilder.BuildDevicePath(entities.First());
var deviceId = DevicePathBuilder.GetDeviceId(entities.First());
var timestamps = new List<long>();
var values = new List<List<object>>();
foreach (var entity in entities)
{
timestamps.Add(entity.Timestamp);
timestamps.Add(entity.Timestamps);
var rowValues = new List<object>();
foreach (var measurement in metadata.Measurements)
{
var value = typeof(T).GetProperty(measurement)?.GetValue(entity);
rowValues.Add(value ?? DBNull.Value);
if(value == null)
{
throw new Exception($"{nameof(BuildTablet)} 构建表模型{typeof(T).Name}时，属性{measurement}值为空，不符合IoTDB设计标准，请赋值以后重新处理。");
}
rowValues.Add(value);
}
values.Add(rowValues);
}
return new Tablet(
devicePath,
deviceId,
metadata.Measurements,
metadata.GetDataTypes(),
metadata.DataTypes,
values,
timestamps
)
{
Tags = metadata.Tags.ToDictionary(
t => t,
t => typeof(T).GetProperty(t)?.GetValue(entities.First())?.ToString())
};
);
}
/// <summary>
@ -165,7 +176,7 @@ namespace JiShe.CollectBus.IoTDBProvider
var metadata = GetMetadata<T>();
var sb = new StringBuilder("SELECT ");
sb.AppendJoin(", ", metadata.Measurements);
sb.Append($" FROM {DevicePathBuilder.BuildStorageGroupPath<T>()}");
sb.Append($" FROM {DevicePathBuilder.GetTableName<T>()}");
if (options.Conditions.Any())
{
@ -202,7 +213,7 @@ namespace JiShe.CollectBus.IoTDBProvider
/// <returns></returns>
private async Task<int> GetTotalCount<T>(QueryOptions options) where T : IoTEntity
{
var countQuery = $"SELECT COUNT(*) FROM {DevicePathBuilder.BuildStorageGroupPath<T>()}";
var countQuery = $"SELECT COUNT(*) FROM {DevicePathBuilder.GetTableName<T>()}";
if (options.Conditions.Any())
{
countQuery += " WHERE " + string.Join(" AND ", options.Conditions.Select(TranslateCondition));
@ -224,33 +235,35 @@ namespace JiShe.CollectBus.IoTDBProvider
var results = new List<T>();
var metadata = GetMetadata<T>();
var properties = typeof(T).GetProperties();
while (dataSet.HasNext() && results.Count < pageSize)
{
var record = dataSet.Next();
var entity = new T
{
Timestamp = record.Timestamps
Timestamps = record.Timestamps
};
foreach (var measurement in metadata.Measurements)
{
var value = record.GetValue(measurement);
var value = record.Values;
var prop = properties.FirstOrDefault(p =>
p.Name.Equals(measurement, StringComparison.OrdinalIgnoreCase));
if (prop != null)
{
typeof(T).GetProperty(measurement)?.SetValue(entity, value);
}
}
results.Add(entity);
}
return results;
}
private async Task EnsureStorageGroupCreated(string storageGroup)
{
if (!await _sessionPool.CheckStorageGroupExists(storageGroup))
{
await _sessionPool.SetStorageGroupAsync(storageGroup);
}
}
/// <summary>
/// 释放资源
/// </summary>
@ -258,5 +271,24 @@ namespace JiShe.CollectBus.IoTDBProvider
{
_sessionPool?.Close().Wait();
}
private TSDataType GetDataTypeFromStr(string str)
{
return str switch
{
"BOOLEAN" => TSDataType.BOOLEAN,
"INT32" => TSDataType.INT32,
"INT64" => TSDataType.INT64,
"FLOAT" => TSDataType.FLOAT,
"DOUBLE" => TSDataType.DOUBLE,
"TEXT" => TSDataType.TEXT,
"NULLTYPE" => TSDataType.NONE,
"TIMESTAMP" => TSDataType.TIMESTAMP,
"DATE" => TSDataType.DATE,
"BLOB" => TSDataType.BLOB,
"STRING" => TSDataType.STRING,
_ => TSDataType.STRING
};
}
}
}

37
src/JiShe.CollectBus.IoTDBProvider/Provider/IoTEntity.cs Normal file
View File

@ -0,0 +1,37 @@
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// IoT实体基类
/// </summary>
public abstract class IoTEntity
{
/// <summary>
/// 系统名称
/// </summary>
[TAGColumn]
public string SystemName { get; set; }
/// <summary>
/// 项目编码
/// </summary>
[TAGColumn]
public string ProjectCode { get; set; }
/// <summary>
/// 设备类型集中器、电表、水表、流量计、传感器等
/// </summary>
[TAGColumn]
public string DeviceType { get; set; }
/// <summary>
/// 设备ID
/// </summary>
[TAGColumn]
public string DeviceId { get; set; }
/// <summary>
/// 时间戳
/// </summary>
public long Timestamps { get; set; }
}
}