admin/JiShe.CollectBus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

封装IoTDBProvider

Browse Source
This commit is contained in:
ChenYi 2025-04-02 14:06:40 +08:00
parent 838ef197e2
commit c57bd15b92
15 changed files with 577 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
JiShe.CollectBus.sln
View File

@ -31,6 +31,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.FreeSql",
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.FreeRedisProvider", "src\JiShe.CollectBus.FreeRedisProvider\JiShe.CollectBus.FreeRedisProvider.csproj", "{C06C4082-638F-2996-5FED-7784475766C1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.IoTDBProvider", "src\JiShe.CollectBus.IoTDBProvider\JiShe.CollectBus.IoTDBProvider.csproj", "{A3F3C092-0A25-450B-BF6A-5983163CBEF5}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -89,6 +91,10 @@ Global
{C06C4082-638F-2996-5FED-7784475766C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Release|Any CPU.Build.0 = Release|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -107,6 +113,7 @@ Global
{8BA01C3D-297D-42DF-BD63-EF07202A0A67} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545}
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545}
{C06C4082-638F-2996-5FED-7784475766C1} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545}
{A3F3C092-0A25-450B-BF6A-5983163CBEF5} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {4324B3B4-B60B-4E3C-91D8-59576B4E26DD}

16
src/JiShe.CollectBus.IoTDBProvider/Attribute/ATTRIBUTEColumnAttribute.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// Column分类标记特性（TAG字段）
/// </summary>
[AttributeUsage(AttributeTargets.Property)]
public class TAGColumnAttribute : Attribute
{
}
}

16
src/JiShe.CollectBus.IoTDBProvider/Attribute/FIELDColumnAttribute.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// Column分类标记特性（FIELD字段）
/// </summary>
[AttributeUsage(AttributeTargets.Property)]
public class FIELDColumnAttribute : Attribute
{
}
}

16
src/JiShe.CollectBus.IoTDBProvider/Attribute/TAGColumnAttribute.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// Column分类标记特性（ATTRIBUTE字段）
/// </summary>
[AttributeUsage(AttributeTargets.Property)]
public class ATTRIBUTEColumnAttribute : Attribute
{
}
}

19
src/JiShe.CollectBus.IoTDBProvider/CollectBusIoTDBModule.cs Normal file
View File

@ -0,0 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Volo.Abp.Modularity;
namespace JiShe.CollectBus.IoTDBProvider
{
public class CollectBusIoTDBModule : AbpModule
{
public override void ConfigureServices(ServiceConfigurationContext context)
{
context.Services.Configure<IoTDBOptions>(context.Services.GetConfiguration().GetSection(nameof(IoTDBOptions)));
context.Services.AddSingleton<IIoTDBProvider, IoTDBProvider>();
}
}
}

24
src/JiShe.CollectBus.IoTDBProvider/DeviceMetadata.cs Normal file
View File

@ -0,0 +1,24 @@
using Apache.IoTDB;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// 设备元数据
/// </summary>
public class DeviceMetadata
{
public List<string> Measurements { get; } = new();
public List<string> Tags { get; } = new();
public List<TSDataType> GetDataTypes()
{
// 根据实际类型映射TSDataType
return Measurements.Select(_ => TSDataType.TEXT).ToList();
}
}
}

37
src/JiShe.CollectBus.IoTDBProvider/DevicePathBuilder.cs Normal file
View File

@ -0,0 +1,37 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// 设备路径构建器
/// </summary>
public static class DevicePathBuilder
{
/// <summary>
/// 构建存储组路径
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public static string BuildStorageGroupPath<T>() where T : IoTEntity
{
var type = typeof(T);
return $"root.{type.GetProperty("SystemName")?.Name}.{type.GetProperty("ProjectCode")?.Name}";
}
/// <summary>
/// 构建设备路径
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entity"></param>
/// <returns></returns>
public static string BuildDevicePath<T>(T entity) where T : IoTEntity
{
return $"root.{entity.SystemName}.{entity.ProjectCode}.{entity.DeviceId}";
}
}
}

38
src/JiShe.CollectBus.IoTDBProvider/IIoTDBProvider.cs Normal file
View File

@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// IoTDB数据源
/// </summary>
public interface IIoTDBProvider
{
/// <summary>
/// 插入数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entity"></param>
/// <returns></returns>
Task InsertAsync<T>(T entity) where T : IoTEntity;
/// <summary>
/// 批量插入数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entities"></param>
/// <returns></returns>
Task BatchInsertAsync<T>(IEnumerable<T> entities) where T : IoTEntity;
/// <summary>
/// 查询数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="options"></param>
/// <returns></returns>
Task<PagedResult<T>> QueryAsync<T>(QueryOptions options) where T : IoTEntity, new();
}
}

262
src/JiShe.CollectBus.IoTDBProvider/IoTDBProvider.cs Normal file
View File

@ -0,0 +1,262 @@
using Apache.IoTDB;
using Apache.IoTDB.DataStructure;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// IoTDB数据源
/// </summary>
public class IoTDBProvider : IIoTDBProvider, IDisposable
{
private readonly IoTDBOptions _options;
private readonly SessionPool _sessionPool;
private static readonly ConcurrentDictionary<Type, DeviceMetadata> _metadataCache = new();
public IoTDBProvider(IOptions<IoTDBOptions> options)
{
_options = options.Value;
_sessionPool = new SessionPool(
_options.ClusterList,
_options.UserName,
_options.Password,
_options.PoolSize);
_sessionPool.Open(false).Wait();
}
/// <summary>
/// 获取设备元数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
private DeviceMetadata GetMetadata<T>() where T : IoTEntity
{
return _metadataCache.GetOrAdd(typeof(T), type =>
{
var metadata = new DeviceMetadata();
foreach (var prop in type.GetProperties())
{
var attr = prop.GetCustomAttribute<ColumnCategoryAttribute>();
if (attr != null)
{
metadata.Tags.Add(prop.Name);
}
else if (prop.Name != nameof(IoTEntity.Timestamp))
{
metadata.Measurements.Add(prop.Name);
}
}
return metadata;
});
}
/// <summary>
/// 插入数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entity"></param>
/// <returns></returns>
public async Task InsertAsync<T>(T entity) where T : IoTEntity
{
var metadata = GetMetadata<T>();
var storageGroup = DevicePathBuilder.BuildStorageGroupPath<T>();
await EnsureStorageGroupCreated(storageGroup);
var tablet = BuildTablet(new[] { entity }, metadata);
await _sessionPool.InsertAlignedTabletAsync(tablet);
}
/// <summary>
/// 批量插入数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entities"></param>
/// <returns></returns>
public async Task BatchInsertAsync<T>(IEnumerable<T> entities) where T : IoTEntity
{
var metadata = GetMetadata<T>();
var storageGroup = DevicePathBuilder.BuildStorageGroupPath<T>();
await EnsureStorageGroupCreated(storageGroup);
var batchSize = 1000;
var batches = entities.Chunk(batchSize);
foreach (var batch in batches)
{
var tablet = BuildTablet(batch, metadata);
await _sessionPool.InsertAlignedTabletAsync(tablet);
}
}
/// <summary>
/// 构建表模型
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entities"></param>
/// <param name="metadata"></param>
/// <returns></returns>
private Tablet BuildTablet<T>(IEnumerable<T> entities, DeviceMetadata metadata) where T : IoTEntity
{
var devicePath = DevicePathBuilder.BuildDevicePath(entities.First());
var timestamps = new List<long>();
var values = new List<List<object>>();
foreach (var entity in entities)
{
timestamps.Add(entity.Timestamp);
var rowValues = new List<object>();
foreach (var measurement in metadata.Measurements)
{
var value = typeof(T).GetProperty(measurement)?.GetValue(entity);
rowValues.Add(value ?? DBNull.Value);
}
values.Add(rowValues);
}
return new Tablet(
devicePath,
metadata.Measurements,
metadata.GetDataTypes(),
values,
timestamps
)
{
Tags = metadata.Tags.ToDictionary(
t => t,
t => typeof(T).GetProperty(t)?.GetValue(entities.First())?.ToString())
};
}
/// <summary>
/// 查询数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="options"></param>
/// <returns></returns>
public async Task<PagedResult<T>> QueryAsync<T>(QueryOptions options) where T : IoTEntity, new()
{
var query = BuildQuery<T>(options);
var sessionDataSet = await _sessionPool.ExecuteQueryStatementAsync(query);
var result = new PagedResult<T>
{
TotalCount = await GetTotalCount<T>(options),
Items = ParseResults<T>(sessionDataSet, options.PageSize)
};
return result;
}
/// <summary>
/// 构建查询语句
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="options"></param>
/// <returns></returns>
private string BuildQuery<T>(QueryOptions options) where T : IoTEntity
{
var metadata = GetMetadata<T>();
var sb = new StringBuilder("SELECT ");
sb.AppendJoin(", ", metadata.Measurements);
sb.Append($" FROM {DevicePathBuilder.BuildStorageGroupPath<T>()}");
if (options.Conditions.Any())
{
sb.Append(" WHERE ");
sb.AppendJoin(" AND ", options.Conditions.Select(TranslateCondition));
}
sb.Append($" LIMIT {options.PageSize} OFFSET {options.Page * options.PageSize}");
return sb.ToString();
}
/// <summary>
/// 将查询条件转换为SQL语句
/// </summary>
/// <param name="condition"></param>
/// <returns></returns>
/// <exception cref="NotSupportedException"></exception>
private string TranslateCondition(QueryCondition condition)
{
return condition.Operator switch
{
">" => $"{condition.Field} > {condition.Value}",
"<" => $"{condition.Field} < {condition.Value}",
"=" => $"{condition.Field} = '{condition.Value}'",
_ => throw new NotSupportedException($"Operator {condition.Operator} not supported")
};
}
/// <summary>
/// 获取查询条件的总数量
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="options"></param>
/// <returns></returns>
private async Task<int> GetTotalCount<T>(QueryOptions options) where T : IoTEntity
{
var countQuery = $"SELECT COUNT(*) FROM {DevicePathBuilder.BuildStorageGroupPath<T>()}";
if (options.Conditions.Any())
{
countQuery += " WHERE " + string.Join(" AND ", options.Conditions.Select(TranslateCondition));
}
var result = await _sessionPool.ExecuteQueryStatementAsync(countQuery);
return result.HasNext() ? Convert.ToInt32(result.Next().Values[0]) : 0;
}
/// <summary>
/// 解析查询结果
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="dataSet"></param>
/// <param name="pageSize"></param>
/// <returns></returns>
private IEnumerable<T> ParseResults<T>(SessionDataSet dataSet, int pageSize) where T : IoTEntity, new()
{
var results = new List<T>();
var metadata = GetMetadata<T>();
while (dataSet.HasNext() && results.Count < pageSize)
{
var record = dataSet.Next();
var entity = new T
{
Timestamp = record.Timestamps
};
foreach (var measurement in metadata.Measurements)
{
var value = record.GetValue(measurement);
typeof(T).GetProperty(measurement)?.SetValue(entity, value);
}
results.Add(entity);
}
return results;
}
private async Task EnsureStorageGroupCreated(string storageGroup)
{
if (!await _sessionPool.CheckStorageGroupExists(storageGroup))
{
await _sessionPool.SetStorageGroupAsync(storageGroup);
}
}
/// <summary>
/// 释放资源
/// </summary>
public void Dispose()
{
_sessionPool?.Close().Wait();
}
}
}

19
src/JiShe.CollectBus.IoTDBProvider/IoTEntity.cs Normal file
View File

@ -0,0 +1,19 @@
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// IoT实体基类
/// </summary>
public abstract class IoTEntity
{
[TAGColumn]
public string SystemName { get; set; }
[TAGColumn]
public string ProjectCode { get; set; }
[TAGColumn]
public string DeviceId { get; set; }
public long Timestamp { get; set; }
}
}

12
src/JiShe.CollectBus.IoTDBProvider/JiShe.CollectBus.IoTDBProvider.csproj Normal file
View File

@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Apache.IoTDB" Version="1.3.3.1" />
<PackageReference Include="Volo.Abp" Version="8.3.3" />
</ItemGroup>
</Project>

32
src/JiShe.CollectBus.IoTDBProvider/Options/IoTDBOptions.cs Normal file
View File

@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// IOTDB配置
/// </summary>
public class IoTDBOptions
{
/// <summary>
/// 集群列表
/// </summary>
public List<string> ClusterList { get; set; }
/// <summary>
/// 用户名
/// </summary>
public string UserName { get; set; }
/// <summary>
/// 密码
/// </summary>
public string Password { get; set; }
/// <summary>
/// 连接池大小
/// </summary>
public int PoolSize { get; set; } = 3;
}
}

25
src/JiShe.CollectBus.IoTDBProvider/Options/PagedResult.cs Normal file
View File

@ -0,0 +1,25 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// 查询结果
/// </summary>
/// <typeparam name="T"></typeparam>
public class PagedResult<T>
{
/// <summary>
/// 总条数
/// </summary>
public int TotalCount { get; set; }
/// <summary>
/// 数据集合
/// </summary>
public IEnumerable<T> Items { get; set; }
}
}

27
src/JiShe.CollectBus.IoTDBProvider/Options/QueryCondition.cs Normal file
View File

@ -0,0 +1,27 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// 查询条件
/// </summary>
public class QueryCondition
{
/// <summary>
/// 字段
/// </summary>
public string Field { get; set; }
/// <summary>
/// 操作符
/// </summary>
public string Operator { get; set; }
/// <summary>
/// 值
/// </summary>
public object Value { get; set; }
}
}

27
src/JiShe.CollectBus.IoTDBProvider/Options/QueryOptions.cs Normal file
View File

@ -0,0 +1,27 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDBProvider
{
/// <summary>
/// 查询条件
/// </summary>
public class QueryOptions
{
/// <summary>
/// 分页
/// </summary>
public int Page { get; set; }
/// <summary>
/// 分页大小
/// </summary>
public int PageSize { get; set; }
/// <summary>
/// 查询条件
/// </summary>
public List<QueryCondition> Conditions { get; } = new();
}
}