封装IoTDBProvider

JiShe.CollectBus.sln

@@ -31,6 +31,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.FreeSql",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.FreeRedisProvider", "src\JiShe.CollectBus.FreeRedisProvider\JiShe.CollectBus.FreeRedisProvider.csproj", "{C06C4082-638F-2996-5FED-7784475766C1}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.IoTDBProvider", "src\JiShe.CollectBus.IoTDBProvider\JiShe.CollectBus.IoTDBProvider.csproj", "{A3F3C092-0A25-450B-BF6A-5983163CBEF5}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -89,6 +91,10 @@ Global
 		{C06C4082-638F-2996-5FED-7784475766C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C06C4082-638F-2996-5FED-7784475766C1}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C06C4082-638F-2996-5FED-7784475766C1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -107,6 +113,7 @@ Global
 		{8BA01C3D-297D-42DF-BD63-EF07202A0A67} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545}
 		{FE0457D9-4038-4A17-8808-DCAD06CFC0A0} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545}
 		{C06C4082-638F-2996-5FED-7784475766C1} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545}
+		{A3F3C092-0A25-450B-BF6A-5983163CBEF5} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {4324B3B4-B60B-4E3C-91D8-59576B4E26DD}

src/JiShe.CollectBus.IoTDBProvider/Attribute/ATTRIBUTEColumnAttribute.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// Column分类标记特性（TAG字段）
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Property)]
+    public class TAGColumnAttribute : Attribute
+    {
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/Attribute/FIELDColumnAttribute.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// Column分类标记特性（FIELD字段）
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Property)]
+    public class FIELDColumnAttribute : Attribute
+    {
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/Attribute/TAGColumnAttribute.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// Column分类标记特性（ATTRIBUTE字段）
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Property)]
+    public class ATTRIBUTEColumnAttribute : Attribute
+    {
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/CollectBusIoTDBModule.cs

@@ -0,0 +1,19 @@
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Volo.Abp.Modularity;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    public class CollectBusIoTDBModule : AbpModule
+    {
+        public override void ConfigureServices(ServiceConfigurationContext context)
+        {
+            context.Services.Configure<IoTDBOptions>(context.Services.GetConfiguration().GetSection(nameof(IoTDBOptions)));
+            context.Services.AddSingleton<IIoTDBProvider, IoTDBProvider>();
+        }
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/DeviceMetadata.cs

@@ -0,0 +1,24 @@
+using Apache.IoTDB;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// 设备元数据
+    /// </summary>
+    public class DeviceMetadata
+    {
+        public List<string> Measurements { get; } = new();
+        public List<string> Tags { get; } = new();
+
+        public List<TSDataType> GetDataTypes()
+        {
+            // 根据实际类型映射TSDataType
+            return Measurements.Select(_ => TSDataType.TEXT).ToList();
+        }
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/DevicePathBuilder.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// 设备路径构建器
+    /// </summary>
+    public static class DevicePathBuilder
+    {
+        /// <summary>
+        /// 构建存储组路径
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <returns></returns>
+        public static string BuildStorageGroupPath<T>() where T : IoTEntity
+        {
+            var type = typeof(T);
+            return $"root.{type.GetProperty("SystemName")?.Name}.{type.GetProperty("ProjectCode")?.Name}";
+        }
+
+        /// <summary>
+        /// 构建设备路径
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="entity"></param>
+        /// <returns></returns>
+        public static string BuildDevicePath<T>(T entity) where T : IoTEntity
+        {
+            return $"root.{entity.SystemName}.{entity.ProjectCode}.{entity.DeviceId}";
+        }
+    }
+
+}

src/JiShe.CollectBus.IoTDBProvider/IIoTDBProvider.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// IoTDB数据源
+    /// </summary>
+    public interface IIoTDBProvider
+    {
+        /// <summary>
+        /// 插入数据
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="entity"></param>
+        /// <returns></returns>
+        Task InsertAsync<T>(T entity) where T : IoTEntity;
+
+        /// <summary>
+        /// 批量插入数据
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="entities"></param>
+        /// <returns></returns>
+        Task BatchInsertAsync<T>(IEnumerable<T> entities) where T : IoTEntity;
+
+        /// <summary>
+        /// 查询数据
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="options"></param>
+        /// <returns></returns>
+        Task<PagedResult<T>> QueryAsync<T>(QueryOptions options) where T : IoTEntity, new();
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/IoTDBProvider.cs

@@ -0,0 +1,262 @@
+using Apache.IoTDB;
+using Apache.IoTDB.DataStructure;
+using Microsoft.Extensions.Options;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// IoTDB数据源
+    /// </summary>
+    public class IoTDBProvider : IIoTDBProvider, IDisposable
+    {
+        private readonly IoTDBOptions _options;
+        private readonly SessionPool _sessionPool;
+        private static readonly ConcurrentDictionary<Type, DeviceMetadata> _metadataCache = new();
+
+        public IoTDBProvider(IOptions<IoTDBOptions> options)
+        {
+            _options = options.Value;
+            _sessionPool = new SessionPool(
+                _options.ClusterList,
+                _options.UserName,
+                _options.Password,
+                _options.PoolSize);
+            _sessionPool.Open(false).Wait();
+        }
+
+        /// <summary>
+        /// 获取设备元数据
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <returns></returns>
+        private DeviceMetadata GetMetadata<T>() where T : IoTEntity
+        {
+            return _metadataCache.GetOrAdd(typeof(T), type =>
+            {
+                var metadata = new DeviceMetadata();
+                foreach (var prop in type.GetProperties())
+                {
+                    var attr = prop.GetCustomAttribute<ColumnCategoryAttribute>();
+                    if (attr != null)
+                    {
+                        metadata.Tags.Add(prop.Name);
+                    }
+                    else if (prop.Name != nameof(IoTEntity.Timestamp))
+                    {
+                        metadata.Measurements.Add(prop.Name);
+                    }
+                }
+                return metadata;
+            });
+        }
+        
+        /// <summary>
+        /// 插入数据
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="entity"></param>
+        /// <returns></returns>
+        public async Task InsertAsync<T>(T entity) where T : IoTEntity
+        {
+            var metadata = GetMetadata<T>();
+            var storageGroup = DevicePathBuilder.BuildStorageGroupPath<T>();
+            await EnsureStorageGroupCreated(storageGroup);
+
+            var tablet = BuildTablet(new[] { entity }, metadata);
+            await _sessionPool.InsertAlignedTabletAsync(tablet);
+        }
+
+        /// <summary>
+        /// 批量插入数据
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="entities"></param>
+        /// <returns></returns>
+        public async Task BatchInsertAsync<T>(IEnumerable<T> entities) where T : IoTEntity
+        {
+            var metadata = GetMetadata<T>();
+            var storageGroup = DevicePathBuilder.BuildStorageGroupPath<T>();
+            await EnsureStorageGroupCreated(storageGroup);
+
+            var batchSize = 1000;
+            var batches = entities.Chunk(batchSize);
+
+            foreach (var batch in batches)
+            {
+                var tablet = BuildTablet(batch, metadata);
+                await _sessionPool.InsertAlignedTabletAsync(tablet);
+            }
+        }
+
+        /// <summary>
+        /// 构建表模型
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="entities"></param>
+        /// <param name="metadata"></param>
+        /// <returns></returns>
+        private Tablet BuildTablet<T>(IEnumerable<T> entities, DeviceMetadata metadata) where T : IoTEntity
+        {
+            var devicePath = DevicePathBuilder.BuildDevicePath(entities.First());
+            var timestamps = new List<long>();
+            var values = new List<List<object>>();
+
+            foreach (var entity in entities)
+            {
+                timestamps.Add(entity.Timestamp);
+                var rowValues = new List<object>();
+                foreach (var measurement in metadata.Measurements)
+                {
+                    var value = typeof(T).GetProperty(measurement)?.GetValue(entity);
+                    rowValues.Add(value ?? DBNull.Value);
+                }
+                values.Add(rowValues);
+            }
+
+            return new Tablet(
+                devicePath,
+                metadata.Measurements,
+                metadata.GetDataTypes(),
+                values,
+                timestamps
+            )
+            {
+                Tags = metadata.Tags.ToDictionary(
+                    t => t,
+                    t => typeof(T).GetProperty(t)?.GetValue(entities.First())?.ToString())
+            };
+        }
+
+        /// <summary>
+        /// 查询数据
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="options"></param>
+        /// <returns></returns>
+        public async Task<PagedResult<T>> QueryAsync<T>(QueryOptions options) where T : IoTEntity, new()
+        {
+            var query = BuildQuery<T>(options);
+            var sessionDataSet = await _sessionPool.ExecuteQueryStatementAsync(query);
+
+            var result = new PagedResult<T>
+            {
+                TotalCount = await GetTotalCount<T>(options),
+                Items = ParseResults<T>(sessionDataSet, options.PageSize)
+            };
+
+            return result;
+        }
+
+        /// <summary>
+        /// 构建查询语句
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="options"></param>
+        /// <returns></returns>
+        private string BuildQuery<T>(QueryOptions options) where T : IoTEntity
+        {
+            var metadata = GetMetadata<T>();
+            var sb = new StringBuilder("SELECT ");
+            sb.AppendJoin(", ", metadata.Measurements);
+            sb.Append($" FROM {DevicePathBuilder.BuildStorageGroupPath<T>()}");
+
+            if (options.Conditions.Any())
+            {
+                sb.Append(" WHERE ");
+                sb.AppendJoin(" AND ", options.Conditions.Select(TranslateCondition));
+            }
+
+            sb.Append($" LIMIT {options.PageSize} OFFSET {options.Page * options.PageSize}");
+            return sb.ToString();
+        }
+
+        /// <summary>
+        /// 将查询条件转换为SQL语句
+        /// </summary>
+        /// <param name="condition"></param>
+        /// <returns></returns>
+        /// <exception cref="NotSupportedException"></exception>
+        private string TranslateCondition(QueryCondition condition)
+        {
+            return condition.Operator switch
+            {
+                ">" => $"{condition.Field} > {condition.Value}",
+                "<" => $"{condition.Field} < {condition.Value}",
+                "=" => $"{condition.Field} = '{condition.Value}'",
+                _ => throw new NotSupportedException($"Operator {condition.Operator} not supported")
+            };
+        }
+
+        /// <summary>
+        /// 获取查询条件的总数量
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="options"></param>
+        /// <returns></returns>
+        private async Task<int> GetTotalCount<T>(QueryOptions options) where T : IoTEntity
+        {
+            var countQuery = $"SELECT COUNT(*) FROM {DevicePathBuilder.BuildStorageGroupPath<T>()}";
+            if (options.Conditions.Any())
+            {
+                countQuery += " WHERE " + string.Join(" AND ", options.Conditions.Select(TranslateCondition));
+            }
+
+            var result = await _sessionPool.ExecuteQueryStatementAsync(countQuery);
+            return result.HasNext() ? Convert.ToInt32(result.Next().Values[0]) : 0;
+        }
+
+        /// <summary>
+        /// 解析查询结果
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="dataSet"></param>
+        /// <param name="pageSize"></param>
+        /// <returns></returns>
+        private IEnumerable<T> ParseResults<T>(SessionDataSet dataSet, int pageSize) where T : IoTEntity, new()
+        {
+            var results = new List<T>();
+            var metadata = GetMetadata<T>();
+
+            while (dataSet.HasNext() && results.Count < pageSize)
+            {
+                var record = dataSet.Next();
+                var entity = new T
+                {
+                    Timestamp = record.Timestamps
+                };
+
+                foreach (var measurement in metadata.Measurements)
+                {
+                    var value = record.GetValue(measurement);
+                    typeof(T).GetProperty(measurement)?.SetValue(entity, value);
+                }
+
+                results.Add(entity);
+            }
+            return results;
+        }
+
+        private async Task EnsureStorageGroupCreated(string storageGroup)
+        {
+            if (!await _sessionPool.CheckStorageGroupExists(storageGroup))
+            {
+                await _sessionPool.SetStorageGroupAsync(storageGroup);
+            }
+        }
+
+        /// <summary>
+        /// 释放资源
+        /// </summary>
+        public void Dispose()
+        {
+            _sessionPool?.Close().Wait();
+        }
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/IoTEntity.cs

@@ -0,0 +1,19 @@
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// IoT实体基类
+    /// </summary>
+    public abstract class IoTEntity
+    {
+        [TAGColumn]
+        public string SystemName { get; set; }
+
+        [TAGColumn]
+        public string ProjectCode { get; set; }
+
+        [TAGColumn]
+        public string DeviceId { get; set; }
+
+        public long Timestamp { get; set; }
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/JiShe.CollectBus.IoTDBProvider.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Apache.IoTDB" Version="1.3.3.1" />
+		<PackageReference Include="Volo.Abp" Version="8.3.3" />
+	</ItemGroup>
+</Project>

src/JiShe.CollectBus.IoTDBProvider/Options/IoTDBOptions.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// IOTDB配置
+    /// </summary>
+    public class IoTDBOptions
+    {
+        /// <summary>
+        /// 集群列表
+        /// </summary>
+        public List<string> ClusterList { get; set; }
+        /// <summary>
+        /// 用户名
+        /// </summary>
+        public string UserName { get; set; }
+        /// <summary>
+        /// 密码
+        /// </summary>
+        public string Password { get; set; }
+
+        /// <summary>
+        /// 连接池大小
+        /// </summary>
+        public int PoolSize { get; set; } = 3;
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/Options/PagedResult.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// 查询结果
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public class PagedResult<T>
+    {
+        /// <summary>
+        /// 总条数
+        /// </summary>
+        public int TotalCount { get; set; }
+
+        /// <summary>
+        /// 数据集合
+        /// </summary>
+        public IEnumerable<T> Items { get; set; }
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/Options/QueryCondition.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// 查询条件
+    /// </summary>
+    public class QueryCondition
+    {
+        /// <summary>
+        /// 字段
+        /// </summary>
+        public string Field { get; set; }
+        /// <summary>
+        /// 操作符
+        /// </summary>
+        public string Operator { get; set; }
+        /// <summary>
+        /// 值
+        /// </summary>
+        public object Value { get; set; }
+    }
+}

src/JiShe.CollectBus.IoTDBProvider/Options/QueryOptions.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.IoTDBProvider
+{
+    /// <summary>
+    /// 查询条件
+    /// </summary>
+    public class QueryOptions
+    {
+        /// <summary>
+        /// 分页
+        /// </summary>
+        public int Page { get; set; }
+        /// <summary>
+        /// 分页大小
+        /// </summary>
+        public int PageSize { get; set; }
+        /// <summary>
+        /// 查询条件
+        /// </summary>
+        public List<QueryCondition> Conditions { get; } = new();
+    }
+}