解决冲突

This commit is contained in:
zenghongyao 2025-04-22 09:37:49 +08:00
commit 2c202081f9
67 changed files with 1288 additions and 1111 deletions

View File

@ -1,156 +0,0 @@
using System.Collections.Concurrent;
using Cassandra;
using Cassandra.Mapping;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
namespace JiShe.CollectBus.Cassandra
{
public class CassandraQueryOptimizer
{
private readonly ISession _session;
private readonly ILogger<CassandraQueryOptimizer> _logger;
private readonly IMemoryCache _cache;
private readonly ConcurrentDictionary<string, PreparedStatement> _preparedStatements;
private readonly int _batchSize;
private readonly TimeSpan _cacheExpiration;
public CassandraQueryOptimizer(
ISession session,
ILogger<CassandraQueryOptimizer> logger,
IMemoryCache cache,
int batchSize = 100,
TimeSpan? cacheExpiration = null)
{
_session = session;
_logger = logger;
_cache = cache;
_preparedStatements = new ConcurrentDictionary<string, PreparedStatement>();
_batchSize = batchSize;
_cacheExpiration = cacheExpiration ?? TimeSpan.FromMinutes(5);
}
public async Task<PreparedStatement> GetOrPrepareStatementAsync(string cql)
{
return _preparedStatements.GetOrAdd(cql, key =>
{
try
{
var statement = _session.Prepare(key);
_logger.LogDebug($"Prepared statement for CQL: {key}");
return statement;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed to prepare statement for CQL: {key}");
throw;
}
});
}
public async Task ExecuteBatchAsync(IEnumerable<BoundStatement> statements)
{
var batch = new BatchStatement();
var count = 0;
foreach (var statement in statements)
{
batch.Add(statement);
count++;
if (count >= _batchSize)
{
await ExecuteBatchAsync(batch);
batch = new BatchStatement();
count = 0;
}
}
if (count > 0)
{
await ExecuteBatchAsync(batch);
}
}
private async Task ExecuteBatchAsync(BatchStatement batch)
{
try
{
await _session.ExecuteAsync(batch);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to execute batch statement");
throw;
}
}
public async Task<T> GetOrSetFromCacheAsync<T>(string cacheKey, Func<Task<T>> getData)
{
if (_cache.TryGetValue(cacheKey, out T cachedValue))
{
_logger.LogDebug($"Cache hit for key: {cacheKey}");
return cachedValue;
}
var data = await getData();
_cache.Set(cacheKey, data, _cacheExpiration);
_logger.LogDebug($"Cache miss for key: {cacheKey}, data cached");
return data;
}
public async Task<IEnumerable<T>> ExecutePagedQueryAsync<T>(
string cql,
object[] parameters,
int pageSize = 100,
string pagingState = null) where T : class
{
var statement = await GetOrPrepareStatementAsync(cql);
var boundStatement = statement.Bind(parameters);
if (!string.IsNullOrEmpty(pagingState))
{
boundStatement.SetPagingState(Convert.FromBase64String(pagingState));
}
boundStatement.SetPageSize(pageSize);
try
{
var result = await _session.ExecuteAsync(boundStatement);
//TODO: RETURN OBJECT
throw new NotImplementedException();
//result.GetRows()
//return result.Select(row => row);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed to execute paged query: {cql}");
throw;
}
}
public async Task BulkInsertAsync<T>(IEnumerable<T> items, string tableName)
{
var mapper = new Mapper(_session);
var batch = new List<BoundStatement>();
var cql = $"INSERT INTO {tableName} ({{0}}) VALUES ({{1}})";
foreach (var chunk in items.Chunk(_batchSize))
{
var statements = chunk.Select(item =>
{
var props = typeof(T).GetProperties();
var columns = string.Join(", ", props.Select(p => p.Name));
var values = string.Join(", ", props.Select(p => "?"));
var statement = _session.Prepare(string.Format(cql, columns, values));
return statement.Bind(props.Select(p => p.GetValue(item)).ToArray());
});
batch.AddRange(statements);
}
await ExecuteBatchAsync(batch);
}
}
}

View File

@ -1,19 +1,13 @@
using Cassandra; using System.Linq.Expressions;
using Cassandra.Data.Linq;
using Cassandra.Mapping; using Cassandra.Mapping;
using JiShe.CollectBus.Cassandra.Extensions; using JiShe.CollectBus.Cassandra.Extensions;
using JiShe.CollectBus.Common.Attributes;
using Microsoft.AspNetCore.Http;
using System.Reflection;
using Thrift.Protocol.Entities;
using Volo.Abp.Domain.Entities; using Volo.Abp.Domain.Entities;
using Volo.Abp.Domain.Repositories;
namespace JiShe.CollectBus.Cassandra namespace JiShe.CollectBus.Cassandra
{ {
public class CassandraRepository<TEntity, TKey> public class CassandraRepository<TEntity, TKey>
: ICassandraRepository<TEntity, TKey> : ICassandraRepository<TEntity, TKey>
where TEntity : class where TEntity : class, ICassandraEntity<TKey>
{ {
private readonly ICassandraProvider _cassandraProvider; private readonly ICassandraProvider _cassandraProvider;
public CassandraRepository(ICassandraProvider cassandraProvider, MappingConfiguration mappingConfig) public CassandraRepository(ICassandraProvider cassandraProvider, MappingConfiguration mappingConfig)
@ -27,12 +21,29 @@ namespace JiShe.CollectBus.Cassandra
public virtual async Task<TEntity> GetAsync(TKey id) public virtual async Task<TEntity> GetAsync(TKey id)
{ {
return await Mapper.SingleOrDefaultAsync<TEntity>("WHERE id = ?", id); return await GetAsync("WHERE id = ?", id);
} }
public virtual async Task<List<TEntity>> GetListAsync() public virtual async Task<TEntity?> GetAsync(string cql, params object[] args)
{ {
return (await Mapper.FetchAsync<TEntity>()).ToList(); return await Mapper.SingleAsync<TEntity?>(cql, args);
}
public virtual async Task<TEntity> FirstOrDefaultAsync(TKey id)
{
return await FirstOrDefaultAsync("WHERE id = ?", id);
}
public virtual async Task<TEntity?> FirstOrDefaultAsync(string cql, params object[] args)
{
return await Mapper.FirstOrDefaultAsync<TEntity>(cql, args);
}
public virtual async Task<List<TEntity>?> GetListAsync(string? cql = null, params object[] args)
{
return cql.IsNullOrWhiteSpace() ? (await Mapper.FetchAsync<TEntity>()).ToList() : (await Mapper.FetchAsync<TEntity>(cql, args)).ToList();
} }
public virtual async Task<TEntity> InsertAsync(TEntity entity) public virtual async Task<TEntity> InsertAsync(TEntity entity)

View File

@ -1,7 +1,4 @@
using Cassandra; using Microsoft.Extensions.DependencyInjection;
using Cassandra.Mapping;
using JiShe.CollectBus.Cassandra.Mappers;
using Microsoft.Extensions.DependencyInjection;
using Volo.Abp; using Volo.Abp;
using Volo.Abp.Autofac; using Volo.Abp.Autofac;
using Volo.Abp.Modularity; using Volo.Abp.Modularity;

View File

@ -1,10 +1,4 @@
using Autofac.Core; using JiShe.CollectBus.Cassandra;
using Cassandra;
using Cassandra.Mapping;
using JiShe.CollectBus.Cassandra;
using JiShe.CollectBus.Cassandra.Mappers;
using Microsoft.Extensions.Options;
using System.Reflection;
using Volo.Abp; using Volo.Abp;
using Volo.Abp.Modularity; using Volo.Abp.Modularity;
@ -26,8 +20,6 @@ namespace Microsoft.Extensions.DependencyInjection
public static void AddCassandra(this ServiceConfigurationContext context) public static void AddCassandra(this ServiceConfigurationContext context)
{ {
context.Services.AddTransient(typeof(ICassandraRepository<,>), typeof(CassandraRepository<,>)); context.Services.AddTransient(typeof(ICassandraRepository<,>), typeof(CassandraRepository<,>));
context.Services.AddSingleton(new MappingConfiguration()
.Define(new CollectBusMapping()));
} }
} }
} }

View File

@ -3,9 +3,7 @@ using System.Text;
using Cassandra; using Cassandra;
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
using JiShe.CollectBus.Common.Attributes; using JiShe.CollectBus.Common.Attributes;
using Cassandra.Mapping; using Volo.Abp.Data;
using Cassandra.Data.Linq;
using Thrift.Protocol.Entities;
namespace JiShe.CollectBus.Cassandra.Extensions namespace JiShe.CollectBus.Cassandra.Extensions
{ {
@ -16,17 +14,26 @@ namespace JiShe.CollectBus.Cassandra.Extensions
var type = typeof(TEntity); var type = typeof(TEntity);
var tableAttribute = type.GetCustomAttribute<CassandraTableAttribute>(); var tableAttribute = type.GetCustomAttribute<CassandraTableAttribute>();
var tableName = tableAttribute?.Name ?? type.Name.ToLower(); var tableName = tableAttribute?.Name ?? type.Name.ToLower();
//var tableKeyspace = tableAttribute?.Keyspace ?? defaultKeyspace;
var tableKeyspace = session.Keyspace; var tableKeyspace = session.Keyspace;
var properties = type.GetProperties(); var properties = type.GetProperties();
var primaryKey = properties.FirstOrDefault(p => p.GetCustomAttribute<KeyAttribute>() != null);
// 分区键设置
var primaryKey = properties.FirstOrDefault(p => p.GetCustomAttribute<PartitionKeyAttribute>() != null);
if (primaryKey == null) if (primaryKey == null)
{ {
throw new InvalidOperationException($"No primary key defined for type {type.Name}"); throw new InvalidOperationException($"No primary key defined for type {type.Name}");
} }
// 集群键设置
var clusteringKeys = properties.Where(p => p.GetCustomAttribute<ClusteringKeyAttribute>() != null).Select(a=>a.Name).ToList();
var clusteringKeyCql = string.Empty;
if (clusteringKeys.Any())
{
clusteringKeyCql = $", {string.Join(", ", clusteringKeys)}";
}
var cql = new StringBuilder(); var cql = new StringBuilder();
cql.Append($"CREATE TABLE IF NOT EXISTS {tableKeyspace}.{tableName} ("); cql.Append($"CREATE TABLE IF NOT EXISTS {tableKeyspace}.{tableName} (");
@ -40,7 +47,7 @@ namespace JiShe.CollectBus.Cassandra.Extensions
cql.Append($"{columnName} {cqlType}, "); cql.Append($"{columnName} {cqlType}, ");
} }
cql.Length -= 2; // Remove last comma and space cql.Length -= 2; // Remove last comma and space
cql.Append($", PRIMARY KEY ({primaryKey.Name.ToLower()}))"); cql.Append($", PRIMARY KEY (({primaryKey.Name.ToLower()}){clusteringKeyCql}))");
session.Execute(cql.ToString()); session.Execute(cql.ToString());
} }
@ -61,6 +68,7 @@ namespace JiShe.CollectBus.Cassandra.Extensions
if (type == typeof(Guid)) return "uuid"; if (type == typeof(Guid)) return "uuid";
if (type == typeof(DateTimeOffset)) return "timestamp"; if (type == typeof(DateTimeOffset)) return "timestamp";
if (type == typeof(Byte[])) return "blob"; if (type == typeof(Byte[])) return "blob";
if (type == typeof(ExtraPropertyDictionary)) return "map<text,text>";
// 处理集合类型 // 处理集合类型
if (type.IsGenericType) if (type.IsGenericType)
@ -72,6 +80,8 @@ namespace JiShe.CollectBus.Cassandra.Extensions
return $"list<{GetCassandraType(elementType)}>"; return $"list<{GetCassandraType(elementType)}>";
if (genericType == typeof(HashSet<>)) if (genericType == typeof(HashSet<>))
return $"set<{GetCassandraType(elementType)}>"; return $"set<{GetCassandraType(elementType)}>";
if (genericType == typeof(Nullable<>))
return GetCassandraType(elementType);
if (genericType == typeof(Dictionary<,>)) if (genericType == typeof(Dictionary<,>))
{ {
var keyType = type.GetGenericArguments()[0]; var keyType = type.GetGenericArguments()[0];

View File

@ -10,7 +10,10 @@ namespace JiShe.CollectBus.Cassandra
public interface ICassandraRepository<TEntity, TKey> where TEntity : class public interface ICassandraRepository<TEntity, TKey> where TEntity : class
{ {
Task<TEntity> GetAsync(TKey id); Task<TEntity> GetAsync(TKey id);
Task<List<TEntity>> GetListAsync(); Task<TEntity?> GetAsync(string cql, params object[] args);
Task<TEntity> FirstOrDefaultAsync(TKey id);
Task<TEntity?> FirstOrDefaultAsync(string cql, params object[] args);
Task<List<TEntity>?> GetListAsync(string? cql = null, params object[] args);
Task<TEntity> InsertAsync(TEntity entity); Task<TEntity> InsertAsync(TEntity entity);
Task<TEntity> UpdateAsync(TEntity entity); Task<TEntity> UpdateAsync(TEntity entity);
Task DeleteAsync(TEntity entity); Task DeleteAsync(TEntity entity);

View File

@ -15,8 +15,8 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\..\services\JiShe.CollectBus.Domain\JiShe.CollectBus.Domain.csproj" />
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj" /> <ProjectReference Include="..\..\shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj" />
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Domain.Shared\JiShe.CollectBus.Domain.Shared.csproj" />
</ItemGroup> </ItemGroup>
</Project> </Project>

View File

@ -0,0 +1,19 @@
using JiShe.CollectBus.IoTDB.Enums;
namespace JiShe.CollectBus.IoTDB.Attribute
{
/// <summary>
/// IoTDB实体类型特性
/// </summary>
[AttributeUsage(AttributeTargets.Class)]
public class EntityTypeAttribute : System.Attribute
{
public EntityTypeEnum EntityType { get; }
public EntityTypeAttribute(EntityTypeEnum entityType)
{
EntityType = entityType;
}
}
}

View File

@ -1,7 +1,7 @@
namespace JiShe.CollectBus.IoTDB.Attribute namespace JiShe.CollectBus.IoTDB.Attribute
{ {
/// <summary> /// <summary>
/// 用于标识当前实体为单侧点模式单侧点模式只有一个Filed标识字段,类型是Tuple<string,object>,Item1=>测点名称Item2=>测点值,泛型 /// 用于标识当前实体为单侧点模式单侧点模式只有一个Filed标识字段,类型是Tuple<string,T>,Item1=>测点名称Item2=>测点值,泛型
/// </summary> /// </summary>
[AttributeUsage(AttributeTargets.Property)] [AttributeUsage(AttributeTargets.Property)]
public class SingleMeasuringAttribute : System.Attribute public class SingleMeasuringAttribute : System.Attribute

View File

@ -0,0 +1,18 @@
using JiShe.CollectBus.IoTDB.Enums;
namespace JiShe.CollectBus.IoTDB.Attribute
{
/// <summary>
/// IoTDB实体存储路径或表名称一般用于已经明确的存储路径或表名称例如日志存储
/// </summary>
[AttributeUsage(AttributeTargets.Class)]
public class TableNameOrTreePathAttribute : System.Attribute
{
public string TableNameOrTreePath { get; }
public TableNameOrTreePathAttribute(string tableNameOrTreePath)
{
TableNameOrTreePath = tableNameOrTreePath;
}
}
}

View File

@ -1,33 +1,22 @@
using JiShe.CollectBus.IoTDB.Context; using JiShe.CollectBus.IoTDB.Context;
using JiShe.CollectBus.IoTDB.Interface;
using JiShe.CollectBus.IoTDB.Options; using JiShe.CollectBus.IoTDB.Options;
using JiShe.CollectBus.IoTDB.Provider;
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Volo.Abp.Modularity; using Volo.Abp.Modularity;
namespace JiShe.CollectBus.IoTDB namespace JiShe.CollectBus.IoTDB;
{
public class CollectBusIoTDBModule : AbpModule /// <summary>
/// CollectBusIoTDBModule
/// </summary>
public class CollectBusIoTDbModule : AbpModule
{ {
public override void ConfigureServices(ServiceConfigurationContext context) public override void ConfigureServices(ServiceConfigurationContext context)
{ {
var configuration = context.Services.GetConfiguration(); var configuration = context.Services.GetConfiguration();
Configure<IoTDBOptions>(options => Configure<IoTDbOptions>(options => { configuration.GetSection(nameof(IoTDbOptions)).Bind(options); });
{
configuration.GetSection(nameof(IoTDBOptions)).Bind(options);
});
// 注册上下文为Scoped // 注册上下文为Scoped
context.Services.AddScoped<IoTDBRuntimeContext>(); context.Services.AddScoped<IoTDbRuntimeContext>();
// 注册Session工厂
context.Services.AddSingleton<IIoTDBSessionFactory, IoTDBSessionFactory>();
// 注册Provider
context.Services.AddScoped<IIoTDBProvider, IoTDBProvider>();
}
} }
} }

View File

@ -6,18 +6,18 @@ namespace JiShe.CollectBus.IoTDB.Context
/// <summary> /// <summary>
/// IoTDB SessionPool 运行时上下文 /// IoTDB SessionPool 运行时上下文
/// </summary> /// </summary>
public class IoTDBRuntimeContext public class IoTDbRuntimeContext
{ {
private readonly bool _defaultValue; private readonly bool _defaultValue;
public IoTDBRuntimeContext(IOptions<IoTDBOptions> options) public IoTDbRuntimeContext(IOptions<IoTDbOptions> options)
{ {
_defaultValue = options.Value.UseTableSessionPoolByDefault; _defaultValue = options.Value.UseTableSessionPoolByDefault;
UseTableSessionPool = _defaultValue; UseTableSessionPool = _defaultValue;
} }
/// <summary> /// <summary>
/// 是否使用表模型存储, 默认false使用tree模型存储 /// 存储模型切换标识是否使用table模型存储, 默认为false标识tree模型存储
/// </summary> /// </summary>
public bool UseTableSessionPool { get; set; } public bool UseTableSessionPool { get; set; }

View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDB.Enums
{
/// <summary>
/// IoTDB实体类型枚举
/// </summary>
public enum EntityTypeEnum
{
/// <summary>
/// 树模型
/// </summary>
TreeModel = 1,
/// <summary>
/// 表模型
/// </summary>
TableModel = 2,
}
}

View File

@ -1,13 +1,13 @@
using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.Common.Models;
using JiShe.CollectBus.IoTDB.Model;
using JiShe.CollectBus.IoTDB.Options; using JiShe.CollectBus.IoTDB.Options;
using JiShe.CollectBus.IoTDB.Provider;
namespace JiShe.CollectBus.IoTDB.Interface namespace JiShe.CollectBus.IoTDB.Interface
{ {
/// <summary> /// <summary>
/// IoTDB数据源,数据库能同时存多个时序模型,但数据是完全隔离的,不能跨时序模型查询,通过连接字符串配置 /// IoTDB数据源,数据库能同时存多个时序模型,但数据是完全隔离的,不能跨时序模型查询,通过连接字符串配置
/// </summary> /// </summary>
public interface IIoTDBProvider public interface IIoTDbProvider
{ {
///// <summary> ///// <summary>
///// 切换 SessionPool ///// 切换 SessionPool

View File

@ -3,8 +3,8 @@
/// <summary> /// <summary>
/// Session 工厂接口 /// Session 工厂接口
/// </summary> /// </summary>
public interface IIoTDBSessionFactory:IDisposable public interface IIoTDbSessionFactory:IDisposable
{ {
IIoTDBSessionPool GetSessionPool(bool useTableSession); IIoTDbSessionPool GetSessionPool(bool useTableSession);
} }
} }

View File

@ -5,7 +5,7 @@ namespace JiShe.CollectBus.IoTDB.Interface
/// <summary> /// <summary>
/// Session 连接池 /// Session 连接池
/// </summary> /// </summary>
public interface IIoTDBSessionPool : IDisposable public interface IIoTDbSessionPool : IDisposable
{ {
/// <summary> /// <summary>
/// 打开连接池 /// 打开连接池

View File

@ -6,7 +6,6 @@
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<!--<PackageReference Include="Apache.IoTDB" Version="1.3.3.1" />-->
<PackageReference Include="Apache.IoTDB" Version="2.0.2" /> <PackageReference Include="Apache.IoTDB" Version="2.0.2" />
<PackageReference Include="Volo.Abp" Version="8.3.3" /> <PackageReference Include="Volo.Abp" Version="8.3.3" />
</ItemGroup> </ItemGroup>

View File

@ -1,9 +1,9 @@
using JiShe.CollectBus.IoTDB.Attribute; using JiShe.CollectBus.IoTDB.Attribute;
namespace JiShe.CollectBus.IoTDB.Provider namespace JiShe.CollectBus.IoTDB.Model
{ {
/// <summary> /// <summary>
/// IoT实体基类 /// IoT实体基类,此类适用于多个数据测点记录场景,单个测点请使用子类 SingleMeasuring
/// </summary> /// </summary>
public abstract class IoTEntity public abstract class IoTEntity
{ {
@ -11,29 +11,29 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// 系统名称 /// 系统名称
/// </summary> /// </summary>
[TAGColumn] [TAGColumn]
public string SystemName { get; set; } public required string SystemName { get; set; }
/// <summary> /// <summary>
/// 项目编码 /// 项目编码
/// </summary> /// </summary>
[TAGColumn] [TAGColumn]
public string ProjectCode { get; set; } public required string ProjectCode { get; set; }
/// <summary> /// <summary>
/// 设备类型集中器、电表、水表、流量计、传感器等 /// 设备类型集中器、电表、水表、流量计、传感器等
/// </summary> /// </summary>
[TAGColumn] [TAGColumn]
public string DeviceType { get; set; } public required string DeviceType { get; set; }
/// <summary> /// <summary>
/// 设备ID /// 设备ID
/// </summary> /// </summary>
[TAGColumn] [TAGColumn]
public string DeviceId { get; set; } public required string DeviceId { get; set; }
/// <summary> /// <summary>
/// 当前时间戳,单位毫秒 /// 当前时间戳,单位毫秒,必须通过DateTimeOffset获取
/// </summary> /// </summary>
public long Timestamps { get; set; } = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); public required long Timestamps { get; set; } = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
} }
} }

View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using JiShe.CollectBus.IoTDB.Attribute;
using JiShe.CollectBus.IoTDB.Enums;
using JiShe.CollectBus.IoTDB.Provider;
namespace JiShe.CollectBus.IoTDB.Model
{
/// <summary>
/// Table模型单项数据实体
/// </summary>
[EntityType(EntityTypeEnum.TableModel)]
public class TableModelSingleMeasuringEntity<T> : IoTEntity
{
/// <summary>
/// 单项数据键值对
/// </summary>
[SingleMeasuring(nameof(SingleColumn))]
public required Tuple<string, T> SingleColumn { get; set; }
}
}

View File

@ -4,19 +4,21 @@ using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using JiShe.CollectBus.IoTDB.Attribute; using JiShe.CollectBus.IoTDB.Attribute;
using JiShe.CollectBus.IoTDB.Enums;
using JiShe.CollectBus.IoTDB.Provider; using JiShe.CollectBus.IoTDB.Provider;
namespace JiShe.CollectBus.IotSystems.AFNEntity namespace JiShe.CollectBus.IoTDB.Model
{ {
/// <summary> /// <summary>
/// AFN单项数据实体 /// Tree模型单项数据实体
/// </summary> /// </summary>
public class SingleMeasuringAFNDataEntity<T> : IoTEntity [EntityType(EntityTypeEnum.TreeModel)]
public class TreeModelSingleMeasuringEntity<T> : IoTEntity
{ {
/// <summary> /// <summary>
/// 单项数据 /// 单项数据键值
/// </summary> /// </summary>
[SingleMeasuring(nameof(SingleMeasuring))] [SingleMeasuring(nameof(SingleMeasuring))]
public Tuple<string, T> SingleMeasuring { get; set; } public required Tuple<string, T> SingleMeasuring { get; set; }
} }
} }

View File

@ -3,7 +3,7 @@
/// <summary> /// <summary>
/// IOTDB配置 /// IOTDB配置
/// </summary> /// </summary>
public class IoTDBOptions public class IoTDbOptions
{ {
/// <summary> /// <summary>
/// 数据库名称,表模型才有,树模型为空 /// 数据库名称,表模型才有,树模型为空

View File

@ -1,4 +1,5 @@
using Apache.IoTDB; using Apache.IoTDB;
using JiShe.CollectBus.IoTDB.Enums;
namespace JiShe.CollectBus.IoTDB.Provider namespace JiShe.CollectBus.IoTDB.Provider
{ {
@ -7,6 +8,11 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// </summary> /// </summary>
public class DeviceMetadata public class DeviceMetadata
{ {
/// <summary>
/// IoTDB实体类型枚举
/// </summary>
public EntityTypeEnum EntityType { get; set; }
/// <summary> /// <summary>
/// 是否有单测量值 /// 是否有单测量值
/// </summary> /// </summary>

View File

@ -1,4 +1,6 @@
namespace JiShe.CollectBus.IoTDB.Provider using JiShe.CollectBus.IoTDB.Model;
namespace JiShe.CollectBus.IoTDB.Provider
{ {
/// <summary> /// <summary>
/// 设备路径构建器 /// 设备路径构建器
@ -13,7 +15,7 @@
/// <returns></returns> /// <returns></returns>
public static string GetDevicePath<T>(T entity) where T : IoTEntity public static string GetDevicePath<T>(T entity) where T : IoTEntity
{ {
return $"root.{entity.SystemName.ToLower()}.`{entity.ProjectCode}`.`{entity.DeviceId}`"; return $"root.{entity.SystemName.ToLower()}.`{entity.ProjectCode}`.`{entity.DeviceType}`.`{entity.DeviceId}`";
} }
@ -28,6 +30,17 @@
var type = typeof(T); var type = typeof(T);
return $"{type.Name.ToLower()}"; return $"{type.Name.ToLower()}";
} }
/// <summary>
/// 获取表名称,用作单侧点表模型特殊处理。
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entity"></param>
/// <returns></returns>
public static string GetDeviceTableName<T>(T entity) where T : IoTEntity
{
return $"{entity.SystemName.ToLower()}.`{entity.ProjectCode}`.`{entity.DeviceType}`.`{entity.DeviceId}`";
}
} }
} }

View File

@ -1,4 +1,5 @@
using System.Collections.Concurrent; using System;
using System.Collections.Concurrent;
using System.Reflection; using System.Reflection;
using System.Text; using System.Text;
using Apache.IoTDB; using Apache.IoTDB;
@ -7,28 +8,37 @@ using JiShe.CollectBus.Common.Models;
using JiShe.CollectBus.IoTDB.Attribute; using JiShe.CollectBus.IoTDB.Attribute;
using JiShe.CollectBus.IoTDB.Context; using JiShe.CollectBus.IoTDB.Context;
using JiShe.CollectBus.IoTDB.Interface; using JiShe.CollectBus.IoTDB.Interface;
using JiShe.CollectBus.IoTDB.Model;
using JiShe.CollectBus.IoTDB.Options; using JiShe.CollectBus.IoTDB.Options;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Volo.Abp.DependencyInjection;
using Volo.Abp.Domain.Entities;
namespace JiShe.CollectBus.IoTDB.Provider namespace JiShe.CollectBus.IoTDB.Provider
{ {
/// <summary> /// <summary>
/// IoTDB数据源 /// IoTDB数据源
/// </summary> /// </summary>
public class IoTDBProvider : IIoTDBProvider public class IoTDbProvider : IIoTDbProvider, IScopedDependency
{ {
private static readonly ConcurrentDictionary<Type, DeviceMetadata> _metadataCache = new(); private static readonly ConcurrentDictionary<Type, DeviceMetadata> MetadataCache = new();
private readonly ILogger<IoTDBProvider> _logger; private readonly ILogger<IoTDbProvider> _logger;
private readonly IIoTDBSessionFactory _sessionFactory; private readonly IIoTDbSessionFactory _sessionFactory;
private readonly IoTDBRuntimeContext _runtimeContext; private readonly IoTDbRuntimeContext _runtimeContext;
private IIoTDBSessionPool CurrentSession => private IIoTDbSessionPool CurrentSession =>
_sessionFactory.GetSessionPool(_runtimeContext.UseTableSessionPool); _sessionFactory.GetSessionPool(_runtimeContext.UseTableSessionPool);
public IoTDBProvider( /// <summary>
ILogger<IoTDBProvider> logger, /// IoTDbProvider
IIoTDBSessionFactory sessionFactory, /// </summary>
IoTDBRuntimeContext runtimeContext) /// <param name="logger"></param>
/// <param name="sessionFactory"></param>
/// <param name="runtimeContext"></param>
public IoTDbProvider(
ILogger<IoTDbProvider> logger,
IIoTDbSessionFactory sessionFactory,
IoTDbRuntimeContext runtimeContext)
{ {
_logger = logger; _logger = logger;
_sessionFactory = sessionFactory; _sessionFactory = sessionFactory;
@ -49,12 +59,6 @@ namespace JiShe.CollectBus.IoTDB.Provider
var tablet = BuildTablet(new[] { entity }, metadata); var tablet = BuildTablet(new[] { entity }, metadata);
await CurrentSession.InsertAsync(tablet); await CurrentSession.InsertAsync(tablet);
//int result = await _currentSession.InsertAsync(tablet);
//if (result <= 0)
//{
// _logger.LogError($"{typeof(T).Name}插入数据没有成功");
//}
} }
/// <summary> /// <summary>
@ -73,11 +77,6 @@ namespace JiShe.CollectBus.IoTDB.Provider
{ {
var tablet = BuildTablet(batch, metadata); var tablet = BuildTablet(batch, metadata);
await CurrentSession.InsertAsync(tablet); await CurrentSession.InsertAsync(tablet);
//var result = await _currentSession.InsertAsync(tablet);
//if (result <= 0)
//{
// _logger.LogWarning($"{typeof(T).Name} 批量插入数据第{batch}批次没有成功,共{batches}批次。");
//}
} }
} }
@ -139,10 +138,39 @@ namespace JiShe.CollectBus.IoTDB.Provider
List<string> tempColumnNames = new List<string>(); List<string> tempColumnNames = new List<string>();
tempColumnNames.AddRange(metadata.ColumnNames); tempColumnNames.AddRange(metadata.ColumnNames);
var entityTypeAttribute = typeof(T).GetCustomAttribute<EntityTypeAttribute>();
if (entityTypeAttribute == null)
{
throw new ArgumentException($"{nameof(BuildTablet)} 构建存储结构{nameof(Tablet)}时 {nameof(T)}的EntityType 没有指定,属于异常情况,-101");
}
if (metadata.EntityType != entityTypeAttribute.EntityType)
{
throw new ArgumentException($"{nameof(BuildTablet)} 构建存储结构{nameof(Tablet)}时 {nameof(T)}的EntityType 和{nameof(DeviceMetadata)}的 EntityType 不一致,属于异常情况,-102");
}
if (metadata.EntityType == Enums.EntityTypeEnum.TreeModel && _runtimeContext.UseTableSessionPool == true)
{
throw new ArgumentException($"{nameof(BuildTablet)} 构建存储结构{nameof(Tablet)}时 tree模型不能使用table模型Session连接属于异常情况-103");
}
else if (metadata.EntityType == Enums.EntityTypeEnum.TableModel && _runtimeContext.UseTableSessionPool == false)
{
throw new ArgumentException($"{nameof(BuildTablet)} 构建存储结构{nameof(Tablet)}时 table模型不能使用tree模型Session连接属于异常情况-104");
}
string tableNameOrTreePath = string.Empty;
var tableNameOrTreePathAttribute = typeof(T).GetCustomAttribute<TableNameOrTreePathAttribute>();
if (tableNameOrTreePathAttribute != null)
{
tableNameOrTreePath = tableNameOrTreePathAttribute.TableNameOrTreePath;
}
foreach (var entity in entities) foreach (var entity in entities)
{ {
timestamps.Add(entity.Timestamps); timestamps.Add(entity.Timestamps);
var rowValues = new List<object>(); var rowValues = new List<object>();
foreach (var measurement in tempColumnNames) foreach (var measurement in tempColumnNames)
{ {
@ -153,7 +181,9 @@ namespace JiShe.CollectBus.IoTDB.Provider
} }
var value = propertyInfo.GetValue(entity); var value = propertyInfo.GetValue(entity);
if (propertyInfo.IsDefined(typeof(SingleMeasuringAttribute), false) && value != null)//表示当前对象是单测点模式 if (propertyInfo.IsDefined(typeof(SingleMeasuringAttribute), false) && metadata.IsSingleMeasuring == true)//表示当前对象是单测点模式
{
if (value != null)
{ {
Type tupleType = value.GetType(); Type tupleType = value.GetType();
Type[] tupleArgs = tupleType.GetGenericArguments(); Type[] tupleArgs = tupleType.GetGenericArguments();
@ -169,27 +199,35 @@ namespace JiShe.CollectBus.IoTDB.Provider
metadata.ColumnNames[indexOf] = (string)item1!; metadata.ColumnNames[indexOf] = (string)item1!;
rowValues.Add(item2); rowValues.Add(item2);
} }
else else
{ {
if (value != null) rowValues.Add(null);
}
//同时如果是单测点模式且是table模型存储路径只能通过DevicePathBuilder.GetDeviceTableName(entity)获取
if (_runtimeContext.UseTableSessionPool)
{ {
tableNameOrTreePath = DevicePathBuilder.GetDeviceTableName(entity);
}
}
else
{
rowValues.Add(value); rowValues.Add(value);
} }
else
{
//填充默认数据值
DataTypeDefaultValueMap.TryGetValue(propertyInfo.PropertyType.Name, out object defaultValue);
rowValues.Add(defaultValue);
}
}
} }
values.Add(rowValues); values.Add(rowValues);
//如果指定了路径
if (!string.IsNullOrWhiteSpace(tableNameOrTreePath))
{
devicePaths.Add(tableNameOrTreePath);
}
else
{
if (!_runtimeContext.UseTableSessionPool)//树模型 if (!_runtimeContext.UseTableSessionPool)//树模型
{ {
devicePaths.Add(DevicePathBuilder.GetDevicePath(entity)); devicePaths.Add(DevicePathBuilder.GetDevicePath(entity));
@ -200,6 +238,8 @@ namespace JiShe.CollectBus.IoTDB.Provider
} }
} }
}
if (devicePaths.Count > 1) if (devicePaths.Count > 1)
{ {
throw new Exception($"{nameof(BuildTablet)} 构建Tablet《{typeof(T).Name}》时,批量插入的设备路径不一致。"); throw new Exception($"{nameof(BuildTablet)} 构建Tablet《{typeof(T).Name}》时,批量插入的设备路径不一致。");
@ -213,14 +253,16 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary> /// <summary>
/// 构建tree模型的Tablet /// 构建tree模型的Tablet
/// </summary> /// </summary>
/// <param name="metadata"></param> /// <param name="metadata">已解析的设备数据元数据</param>
/// <param name="devicePath"></param> /// <param name="devicePath">设备路径</param>
/// <param name="values"></param> /// <param name="values">数据集合</param>
/// <param name="timestamps"></param> /// <param name="timestamps">时间戳集合</param>
/// <returns></returns> /// <returns></returns>
private Tablet BuildSessionTablet(DeviceMetadata metadata, string devicePath, private Tablet BuildSessionTablet(DeviceMetadata metadata, string devicePath,
List<List<object>> values, List<long> timestamps) List<List<object>> values, List<long> timestamps)
{ {
//todo 树模型需要去掉TAG类型和ATTRIBUTE类型的字段只需要保留FIELD类型字段即可
return new Tablet( return new Tablet(
devicePath, devicePath,
metadata.ColumnNames, metadata.ColumnNames,
@ -233,16 +275,16 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary> /// <summary>
/// 构建表模型的Tablet /// 构建表模型的Tablet
/// </summary> /// </summary>
/// <param name="metadata"></param> /// <param name="metadata">已解析的设备数据元数据</param>
/// <param name="devicePath"></param> /// <param name="tableName">表名称</param>
/// <param name="values"></param> /// <param name="values">数据集合</param>
/// <param name="timestamps"></param> /// <param name="timestamps">时间戳集合</param>
/// <returns></returns> /// <returns></returns>
private Tablet BuildTableSessionTablet(DeviceMetadata metadata, string devicePath, private Tablet BuildTableSessionTablet(DeviceMetadata metadata, string tableName,
List<List<object>> values, List<long> timestamps) List<List<object>> values, List<long> timestamps)
{ {
var tablet = new Tablet( var tablet = new Tablet(
devicePath, tableName,
metadata.ColumnNames, metadata.ColumnNames,
metadata.ColumnCategories, metadata.ColumnCategories,
metadata.DataTypes, metadata.DataTypes,
@ -392,34 +434,21 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <returns></returns> /// <returns></returns>
private DeviceMetadata GetMetadata<T>() where T : IoTEntity private DeviceMetadata GetMetadata<T>() where T : IoTEntity
{ {
var columns = CollectColumnMetadata(typeof(T)); var columns = CollectColumnMetadata(typeof(T));
var metadata = BuildDeviceMetadata(columns); var metadata = BuildDeviceMetadata<T>(columns);
return MetadataCache.AddOrUpdate(
return _metadataCache.AddOrUpdate(
typeof(T), typeof(T),
addValueFactory: t => metadata, // 如果键不存在,用此值添加 addValueFactory: t => metadata, // 如果键不存在,用此值添加
updateValueFactory: (t, existingValue) => updateValueFactory: (t, existingValue) =>
{ {
var columns = CollectColumnMetadata(t); var columns = CollectColumnMetadata(t);
var metadata = BuildDeviceMetadata(columns); var metadata = BuildDeviceMetadata<T>(columns);
//对现有值 existingValue 进行修改,返回新值 //对现有值 existingValue 进行修改,返回新值
existingValue.ColumnNames = metadata.ColumnNames; existingValue.ColumnNames = metadata.ColumnNames;
return existingValue; return existingValue;
} }
); );
//return _metadataCache.GetOrAdd(typeof(T), type =>
//{
// var columns = CollectColumnMetadata(type);
// var metadata = BuildDeviceMetadata(columns);
// //if (metadata.IsSingleMeasuring)
// //{
// // _metadataCache.Remove(typeof(T));
// //}
// return metadata;
//});
} }
/// <summary> /// <summary>
@ -433,21 +462,36 @@ namespace JiShe.CollectBus.IoTDB.Provider
foreach (var prop in type.GetProperties()) foreach (var prop in type.GetProperties())
{ {
string typeName = string.Empty;
Type declaredType = prop.PropertyType;
// 处理可空类型
if (declaredType.IsGenericType && declaredType.GetGenericTypeDefinition() == typeof(Nullable<>))
{
Type underlyingType = Nullable.GetUnderlyingType(declaredType);
typeName = underlyingType.Name;
}
else
{
typeName = declaredType.Name;
}
//先获取Tag标签和属性标签 //先获取Tag标签和属性标签
ColumnInfo? column = prop.GetCustomAttribute<TAGColumnAttribute>() is not null ? new ColumnInfo( ColumnInfo? column = prop.GetCustomAttribute<TAGColumnAttribute>() is not null ? new ColumnInfo(
name: prop.Name, name: prop.Name,
category: ColumnCategory.TAG, category: ColumnCategory.TAG,
dataType: GetDataTypeFromTypeName(prop.PropertyType.Name), dataType: GetDataTypeFromTypeName(typeName),
false false
) : prop.GetCustomAttribute<ATTRIBUTEColumnAttribute>() is not null ? new ColumnInfo( ) : prop.GetCustomAttribute<ATTRIBUTEColumnAttribute>() is not null ? new ColumnInfo(
prop.Name, prop.Name,
ColumnCategory.ATTRIBUTE, ColumnCategory.ATTRIBUTE,
GetDataTypeFromTypeName(prop.PropertyType.Name), GetDataTypeFromTypeName(typeName),
false false
) : prop.GetCustomAttribute<FIELDColumnAttribute>() is not null ? new ColumnInfo( ) : prop.GetCustomAttribute<FIELDColumnAttribute>() is not null ? new ColumnInfo(
prop.Name, prop.Name,
ColumnCategory.FIELD, ColumnCategory.FIELD,
GetDataTypeFromTypeName(prop.PropertyType.Name), GetDataTypeFromTypeName(typeName),
false) false)
: null; : null;
@ -483,9 +527,10 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary> /// <summary>
/// 构建设备元数据 /// 构建设备元数据
/// </summary> /// </summary>
/// <param name="columns"></param> /// <param name="typeInfo">待解析的类</param>
/// <param name="columns">已处理好的数据列</param>
/// <returns></returns> /// <returns></returns>
private DeviceMetadata BuildDeviceMetadata(List<ColumnInfo> columns) private DeviceMetadata BuildDeviceMetadata<T>(List<ColumnInfo> columns) where T : IoTEntity
{ {
var metadata = new DeviceMetadata(); var metadata = new DeviceMetadata();
@ -504,6 +549,15 @@ namespace JiShe.CollectBus.IoTDB.Provider
ProcessCategory(groupedColumns, ColumnCategory.ATTRIBUTE, metadata); ProcessCategory(groupedColumns, ColumnCategory.ATTRIBUTE, metadata);
ProcessCategory(groupedColumns, ColumnCategory.FIELD, metadata); ProcessCategory(groupedColumns, ColumnCategory.FIELD, metadata);
var entityTypeAttribute = typeof(T).GetCustomAttribute<EntityTypeAttribute>();
if (entityTypeAttribute == null)
{
throw new ArgumentException($"{nameof(BuildDeviceMetadata)} 构建设备元数据时 {nameof(IoTEntity)} 的EntityType 没有指定,属于异常情况,-101");
}
metadata.EntityType = entityTypeAttribute.EntityType;
return metadata; return metadata;
} }

View File

@ -2,6 +2,7 @@
using JiShe.CollectBus.IoTDB.Interface; using JiShe.CollectBus.IoTDB.Interface;
using JiShe.CollectBus.IoTDB.Options; using JiShe.CollectBus.IoTDB.Options;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Volo.Abp.DependencyInjection;
namespace JiShe.CollectBus.IoTDB.Provider namespace JiShe.CollectBus.IoTDB.Provider
{ {
@ -9,25 +10,29 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary> /// <summary>
/// 实现带缓存的Session工厂 /// 实现带缓存的Session工厂
/// </summary> /// </summary>
public class IoTDBSessionFactory : IIoTDBSessionFactory public class IoTDbSessionFactory : IIoTDbSessionFactory, ISingletonDependency
{ {
private readonly IoTDBOptions _options; private readonly IoTDbOptions _options;
private readonly ConcurrentDictionary<bool, IIoTDBSessionPool> _pools = new(); private readonly ConcurrentDictionary<bool, IIoTDbSessionPool> _pools = new();
private bool _disposed; private bool _disposed;
public IoTDBSessionFactory(IOptions<IoTDBOptions> options) /// <summary>
/// IoTDbSessionFactory
/// </summary>
/// <param name="options"></param>
public IoTDbSessionFactory(IOptions<IoTDbOptions> options)
{ {
_options = options.Value; _options = options.Value;
} }
public IIoTDBSessionPool GetSessionPool(bool useTableSession) public IIoTDbSessionPool GetSessionPool(bool useTableSession)
{ {
if (_disposed) throw new ObjectDisposedException(nameof(IoTDBSessionFactory)); if (_disposed) throw new ObjectDisposedException(nameof(IoTDbSessionFactory));
return _pools.GetOrAdd(useTableSession, key => return _pools.GetOrAdd(useTableSession, key =>
{ {
var pool = key var pool = key
? (IIoTDBSessionPool)new TableSessionPoolAdapter(_options) ? (IIoTDbSessionPool)new TableSessionPoolAdapter(_options)
: new SessionPoolAdapter(_options); : new SessionPoolAdapter(_options);
pool.OpenAsync().ConfigureAwait(false).GetAwaiter().GetResult(); ; pool.OpenAsync().ConfigureAwait(false).GetAwaiter().GetResult(); ;

View File

@ -9,12 +9,16 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary> /// <summary>
/// 树模型连接池 /// 树模型连接池
/// </summary> /// </summary>
public class SessionPoolAdapter : IIoTDBSessionPool public class SessionPoolAdapter : IIoTDbSessionPool
{ {
private readonly SessionPool _sessionPool; private readonly SessionPool _sessionPool;
private readonly IoTDBOptions _options; private readonly IoTDbOptions _options;
public SessionPoolAdapter(IoTDBOptions options) /// <summary>
/// SessionPoolAdapter
/// </summary>
/// <param name="options"></param>
public SessionPoolAdapter(IoTDbOptions options)
{ {
_options = options; _options = options;
_sessionPool = new SessionPool.Builder() _sessionPool = new SessionPool.Builder()
@ -52,7 +56,7 @@ namespace JiShe.CollectBus.IoTDB.Provider
var result = await _sessionPool.InsertAlignedTabletAsync(tablet); var result = await _sessionPool.InsertAlignedTabletAsync(tablet);
if (result != 0) if (result != 0)
{ {
throw new Exception($"{nameof(TableSessionPoolAdapter)} "); throw new Exception($"{nameof(SessionPoolAdapter)} Tree模型数据入库没有成功返回结果为{result}");
} }
return result; return result;

View File

@ -9,12 +9,16 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary> /// <summary>
/// 表模型Session连接池 /// 表模型Session连接池
/// </summary> /// </summary>
public class TableSessionPoolAdapter : IIoTDBSessionPool public class TableSessionPoolAdapter : IIoTDbSessionPool
{ {
private readonly TableSessionPool _sessionPool; private readonly TableSessionPool _sessionPool;
private readonly IoTDBOptions _options; private readonly IoTDbOptions _options;
public TableSessionPoolAdapter(IoTDBOptions options) /// <summary>
/// TableSessionPoolAdapter
/// </summary>
/// <param name="options"></param>
public TableSessionPoolAdapter(IoTDbOptions options)
{ {
_options = options; _options = options;
_sessionPool = new TableSessionPool.Builder() _sessionPool = new TableSessionPool.Builder()
@ -50,7 +54,7 @@ namespace JiShe.CollectBus.IoTDB.Provider
var result = await _sessionPool.InsertAsync(tablet); var result = await _sessionPool.InsertAsync(tablet);
if (result != 0) if (result != 0)
{ {
throw new Exception($"{nameof(TableSessionPoolAdapter)} "); throw new Exception($"{nameof(TableSessionPoolAdapter)} table模型数据入库没有成功返回结果为{result}");
} }
return result; return result;

View File

@ -1,30 +1,24 @@
using Confluent.Kafka; using Confluent.Kafka;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Confluent.Kafka.Admin; using Confluent.Kafka.Admin;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Volo.Abp.DependencyInjection; using Volo.Abp.DependencyInjection;
namespace JiShe.CollectBus.Kafka.AdminClient namespace JiShe.CollectBus.Kafka.AdminClient;
{
public class AdminClientService : IAdminClientService, IDisposable, ISingletonDependency public class AdminClientService : IAdminClientService, IDisposable, ISingletonDependency
{ {
private readonly ILogger<AdminClientService> _logger; private readonly ILogger<AdminClientService> _logger;
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="AdminClientService" /> class. /// Initializes a new instance of the <see cref="AdminClientService" /> class.
/// </summary> /// </summary>
/// <param name="configuration">The configuration.</param> /// <param name="configuration"></param>
/// <param name="logger">The logger.</param> /// <param name="logger"></param>
public AdminClientService(IConfiguration configuration, ILogger<AdminClientService> logger) public AdminClientService(IConfiguration configuration, ILogger<AdminClientService> logger)
{ {
_logger = logger; _logger = logger;
GetInstance(configuration); Instance = GetInstance(configuration);
} }
/// <summary> /// <summary>
@ -33,70 +27,17 @@ namespace JiShe.CollectBus.Kafka.AdminClient
/// <value> /// <value>
/// The instance. /// The instance.
/// </value> /// </value>
public IAdminClient Instance { get; set; } = default; public IAdminClient Instance { get; set; }
/// <summary> /// <summary>
/// Gets the instance.
/// </summary>
/// <param name="configuration">The configuration.</param>
/// <returns></returns>
public IAdminClient GetInstance(IConfiguration configuration)
{
ArgumentNullException.ThrowIfNullOrWhiteSpace(configuration["Kafka:EnableAuthorization"]);
var enableAuthorization = bool.Parse(configuration["Kafka:EnableAuthorization"]!);
var adminClientConfig = new AdminClientConfig()
{
BootstrapServers = configuration["Kafka:BootstrapServers"],
};
if (enableAuthorization)
{
adminClientConfig.SecurityProtocol = SecurityProtocol.SaslPlaintext;
adminClientConfig.SaslMechanism = SaslMechanism.Plain;
adminClientConfig.SaslUsername = configuration["Kafka:SaslUserName"];
adminClientConfig.SaslPassword = configuration["Kafka:SaslPassword"];
}
Instance = new AdminClientBuilder(adminClientConfig).Build();
return Instance;
}
/// <summary>
/// Checks the topic asynchronous.
/// </summary>
/// <param name="topic">The topic.</param>
/// <returns></returns>
public async Task<bool> CheckTopicAsync(string topic)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(5));
return await Task.FromResult(metadata.Topics.Exists(a => a.Topic == topic));
}
/// <summary>
/// 判断Kafka主题是否存在
/// </summary>
/// <param name="topic">主题名称</param>
/// <param name="numPartitions">副本数量不能高于Brokers数量</param>
/// <returns></returns>
public async Task<bool> CheckTopicAsync(string topic,int numPartitions)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(5));
if(numPartitions > metadata.Brokers.Count)
{
throw new Exception($"{nameof(CheckTopicAsync)} 主题检查时,副本数量大于了节点数量。") ;
}
return await Task.FromResult(metadata.Topics.Exists(a => a.Topic == topic));
}
//// <summary>
/// 创建Kafka主题 /// 创建Kafka主题
/// </summary> /// </summary>
/// <param name="topic">主题名称</param> /// <param name="topic"></param>
/// <param name="numPartitions">主题分区数量</param> /// <param name="numPartitions"></param>
/// <param name="replicationFactor">副本数量不能高于Brokers数量</param> /// <param name="replicationFactor"></param>
/// <returns></returns> /// <returns></returns>
public async Task CreateTopicAsync(string topic, int numPartitions, short replicationFactor) public async Task CreateTopicAsync(string topic, int numPartitions, short replicationFactor)
{ {
try try
{ {
if (await CheckTopicAsync(topic)) return; if (await CheckTopicAsync(topic)) return;
@ -114,10 +55,7 @@ namespace JiShe.CollectBus.Kafka.AdminClient
} }
catch (CreateTopicsException e) catch (CreateTopicsException e)
{ {
if (e.Results[0].Error.Code != ErrorCode.TopicAlreadyExists) if (e.Results[0].Error.Code != ErrorCode.TopicAlreadyExists) throw;
{
throw;
}
} }
} }
@ -200,5 +138,53 @@ namespace JiShe.CollectBus.Kafka.AdminClient
{ {
Instance?.Dispose(); Instance?.Dispose();
} }
/// <summary>
/// Gets the instance.
/// </summary>
/// <param name="configuration">The configuration.</param>
/// <returns></returns>
public IAdminClient GetInstance(IConfiguration configuration)
{
ArgumentException.ThrowIfNullOrWhiteSpace(configuration["Kafka:EnableAuthorization"]);
var enableAuthorization = bool.Parse(configuration["Kafka:EnableAuthorization"]!);
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = configuration["Kafka:BootstrapServers"]
};
if (enableAuthorization)
{
adminClientConfig.SecurityProtocol = SecurityProtocol.SaslPlaintext;
adminClientConfig.SaslMechanism = SaslMechanism.Plain;
adminClientConfig.SaslUsername = configuration["Kafka:SaslUserName"];
adminClientConfig.SaslPassword = configuration["Kafka:SaslPassword"];
}
return new AdminClientBuilder(adminClientConfig).Build();
}
/// <summary>
/// Checks the topic asynchronous.
/// </summary>
/// <param name="topic">The topic.</param>
/// <returns></returns>
public async Task<bool> CheckTopicAsync(string topic)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(5));
return await Task.FromResult(metadata.Topics.Exists(a => a.Topic == topic));
}
/// <summary>
/// 判断Kafka主题是否存在
/// </summary>
/// <param name="topic">主题名称</param>
/// <param name="numPartitions">副本数量不能高于Brokers数量</param>
/// <returns></returns>
public async Task<bool> CheckTopicAsync(string topic, int numPartitions)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(5));
if (numPartitions > metadata.Brokers.Count)
throw new Exception($"{nameof(CheckTopicAsync)} 主题检查时,副本数量大于了节点数量。");
return await Task.FromResult(metadata.Topics.Exists(a => a.Topic == topic));
} }
} }

View File

@ -1,14 +1,26 @@
using System; namespace JiShe.CollectBus.Kafka.Attributes;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Attributes
{
[AttributeUsage(AttributeTargets.Method)] [AttributeUsage(AttributeTargets.Method)]
public class KafkaSubscribeAttribute : Attribute public class KafkaSubscribeAttribute : Attribute
{ {
/// <summary>
/// 订阅主题
/// </summary>
/// <param name="batchTimeout"></param>
public KafkaSubscribeAttribute(string topic)
{
Topic = topic;
}
/// <summary>
/// 订阅主题
/// </summary>
public KafkaSubscribeAttribute(string topic, int partition)
{
Topic = topic;
Partition = partition;
}
/// <summary> /// <summary>
/// 订阅的主题 /// 订阅的主题
/// </summary> /// </summary>
@ -45,24 +57,4 @@ namespace JiShe.CollectBus.Kafka.Attributes
/// 格式:("00:05:00") /// 格式:("00:05:00")
/// </summary> /// </summary>
public TimeSpan? BatchTimeout { get; set; } = null; public TimeSpan? BatchTimeout { get; set; } = null;
/// <summary>
/// 订阅主题
/// </summary>
/// <param name="batchTimeout"></param>
public KafkaSubscribeAttribute(string topic)
{
this.Topic = topic;
}
/// <summary>
/// 订阅主题
/// </summary>
public KafkaSubscribeAttribute(string topic, int partition)
{
this.Topic = topic;
this.Partition = partition;
}
}
} }

View File

@ -1,11 +1,5 @@
using System; namespace JiShe.CollectBus.Kafka.Attributes;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Attributes
{
[AttributeUsage(AttributeTargets.Class, Inherited = false)] [AttributeUsage(AttributeTargets.Class, Inherited = false)]
public class TopicAttribute : Attribute public class TopicAttribute : Attribute
{ {
@ -26,4 +20,3 @@ namespace JiShe.CollectBus.Kafka.Attributes
/// </value> /// </value>
public string Name { get; set; } public string Name { get; set; }
} }
}

View File

@ -13,15 +13,18 @@ namespace JiShe.CollectBus.Kafka.Consumer
public class ConsumerService : IConsumerService, IDisposable public class ConsumerService : IConsumerService, IDisposable
{ {
private readonly ILogger<ConsumerService> _logger; private readonly ILogger<ConsumerService> _logger;
private readonly IConfiguration _configuration;
private readonly ConcurrentDictionary<Type, (object Consumer, CancellationTokenSource CTS)> private readonly ConcurrentDictionary<Type, (object Consumer, CancellationTokenSource CTS)>
_consumerStore = new(); _consumerStore = new();
private readonly KafkaOptionConfig _kafkaOptionConfig; private readonly KafkaOptionConfig _kafkaOptionConfig;
private class KafkaConsumer<TKey, TValue> where TKey : notnull where TValue : class { } private class KafkaConsumer<TKey, TValue> where TKey : notnull where TValue : class { }
public ConsumerService(IConfiguration configuration, ILogger<ConsumerService> logger, IOptions<KafkaOptionConfig> kafkaOptionConfig) /// <summary>
/// ConsumerService
/// </summary>
/// <param name="logger"></param>
/// <param name="kafkaOptionConfig"></param>
public ConsumerService(ILogger<ConsumerService> logger, IOptions<KafkaOptionConfig> kafkaOptionConfig)
{ {
_configuration = configuration;
_logger = logger; _logger = logger;
_kafkaOptionConfig = kafkaOptionConfig.Value; _kafkaOptionConfig = kafkaOptionConfig.Value;
} }
@ -165,10 +168,10 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <summary> /// <summary>
/// 订阅消息 /// 订阅消息
/// </summary> /// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TValue"></typeparam> /// <typeparam name="TValue"></typeparam>
/// <param name="topics"></param> /// <param name="topics"></param>
/// <param name="messageHandler"></param> /// <param name="messageHandler"></param>
/// <param name="groupId"></param>
/// <returns></returns> /// <returns></returns>
public async Task SubscribeAsync<TValue>(string[] topics, Func<TValue, Task<bool>> messageHandler, string? groupId) where TValue : class public async Task SubscribeAsync<TValue>(string[] topics, Func<TValue, Task<bool>> messageHandler, string? groupId) where TValue : class
{ {
@ -387,7 +390,7 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <param name="consumeTimeout">消费等待时间</param> /// <param name="consumeTimeout">消费等待时间</param>
public async Task SubscribeBatchAsync<TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class public async Task SubscribeBatchAsync<TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class
{ {
await SubscribeBatchAsync<TValue>(new[] { topic }, messageBatchHandler, groupId, batchSize, batchTimeout, consumeTimeout); await SubscribeBatchAsync(new[] { topic }, messageBatchHandler, groupId, batchSize, batchTimeout, consumeTimeout);
} }

View File

@ -1,15 +1,9 @@
using Confluent.Kafka; namespace JiShe.CollectBus.Kafka.Consumer;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Consumer
{
public interface IConsumerService public interface IConsumerService
{ {
Task SubscribeAsync<TKey, TValue>(string topic, Func<TKey, TValue, Task<bool>> messageHandler, string? groupId=null) where TKey : notnull where TValue : class; Task SubscribeAsync<TKey, TValue>(string topic, Func<TKey, TValue, Task<bool>> messageHandler,
string? groupId = null) where TKey : notnull where TValue : class;
/// <summary> /// <summary>
/// 订阅消息 /// 订阅消息
@ -18,9 +12,11 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <param name="topic"></param> /// <param name="topic"></param>
/// <param name="messageHandler"></param> /// <param name="messageHandler"></param>
/// <returns></returns> /// <returns></returns>
Task SubscribeAsync<TValue>(string topic, Func<TValue, Task<bool>> messageHandler, string? groupId = null) where TValue : class; Task SubscribeAsync<TValue>(string topic, Func<TValue, Task<bool>> messageHandler, string? groupId = null)
where TValue : class;
Task SubscribeAsync<TKey, TValue>(string[] topics, Func<TKey, TValue, Task<bool>> messageHandler, string? groupId) where TKey : notnull where TValue : class; Task SubscribeAsync<TKey, TValue>(string[] topics, Func<TKey, TValue, Task<bool>> messageHandler, string? groupId)
where TKey : notnull where TValue : class;
/// <summary> /// <summary>
@ -31,16 +27,24 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <param name="topics"></param> /// <param name="topics"></param>
/// <param name="messageHandler"></param> /// <param name="messageHandler"></param>
/// <returns></returns> /// <returns></returns>
Task SubscribeAsync<TValue>(string[] topics, Func<TValue, Task<bool>> messageHandler, string? groupId = null) where TValue : class; Task SubscribeAsync<TValue>(string[] topics, Func<TValue, Task<bool>> messageHandler, string? groupId = null)
where TValue : class;
Task SubscribeBatchAsync<TKey, TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class; Task SubscribeBatchAsync<TKey, TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler,
string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null)
where TKey : notnull where TValue : class;
Task SubscribeBatchAsync<TKey, TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class; Task SubscribeBatchAsync<TKey, TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler,
string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null)
where TKey : notnull where TValue : class;
Task SubscribeBatchAsync<TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class; Task SubscribeBatchAsync<TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler,
string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null)
where TValue : class;
Task SubscribeBatchAsync<TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class; Task SubscribeBatchAsync<TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler,
string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null)
where TValue : class;
void Unsubscribe<TKey, TValue>() where TKey : notnull where TValue : class; void Unsubscribe<TKey, TValue>() where TKey : notnull where TValue : class;
} }
}

View File

@ -1,12 +1,7 @@
using Confluent.Kafka; using Confluent.Kafka;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal namespace JiShe.CollectBus.Kafka.Internal;
{
/// <summary> /// <summary>
/// 消息头过滤器 /// 消息头过滤器
/// </summary> /// </summary>
@ -20,11 +15,8 @@ namespace JiShe.CollectBus.Kafka.Internal
public bool Match(Headers headers) public bool Match(Headers headers)
{ {
foreach (var kvp in this) foreach (var kvp in this)
{
if (!headers.TryGetLastBytes(kvp.Key, out var value) || !value.SequenceEqual(kvp.Value)) if (!headers.TryGetLastBytes(kvp.Key, out var value) || !value.SequenceEqual(kvp.Value))
return false; return false;
}
return true; return true;
} }
} }
}

View File

@ -1,11 +1,5 @@
using System; namespace JiShe.CollectBus.Kafka.Internal;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal
{
/// <summary> /// <summary>
/// Kafka订阅者 /// Kafka订阅者
/// <para> /// <para>
@ -15,4 +9,3 @@ namespace JiShe.CollectBus.Kafka.Internal
public interface IKafkaSubscribe public interface IKafkaSubscribe
{ {
} }
}

View File

@ -1,11 +1,5 @@
using System; namespace JiShe.CollectBus.Kafka.Internal;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal
{
public interface ISubscribeAck public interface ISubscribeAck
{ {
/// <summary> /// <summary>
@ -18,4 +12,3 @@ namespace JiShe.CollectBus.Kafka.Internal
/// </summary> /// </summary>
string? Msg { get; set; } string? Msg { get; set; }
} }
}

View File

@ -1,12 +1,7 @@
using Confluent.Kafka; using Confluent.Kafka;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal namespace JiShe.CollectBus.Kafka.Internal;
{
public class KafkaOptionConfig public class KafkaOptionConfig
{ {
/// <summary> /// <summary>
@ -63,6 +58,4 @@ namespace JiShe.CollectBus.Kafka.Internal
/// 首次采集时间 /// 首次采集时间
/// </summary> /// </summary>
public DateTime FirstCollectionTime { get; set; } public DateTime FirstCollectionTime { get; set; }
}
} }

View File

@ -1,14 +1,8 @@
using Newtonsoft.Json; using System.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Reflection; using System.Reflection;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal namespace JiShe.CollectBus.Kafka.Internal;
{
/// <summary> /// <summary>
/// 反射辅助类 /// 反射辅助类
/// </summary> /// </summary>
@ -27,18 +21,15 @@ namespace JiShe.CollectBus.Kafka.Internal
if (parameterIndex < 0 || parameterIndex >= parameters.Length) if (parameterIndex < 0 || parameterIndex >= parameters.Length)
throw new ArgumentOutOfRangeException(nameof(parameterIndex)); throw new ArgumentOutOfRangeException(nameof(parameterIndex));
ParameterInfo param = parameters[parameterIndex]; var param = parameters[parameterIndex];
Type paramType = param.ParameterType; var paramType = param.ParameterType;
Type? elementType = null; Type? elementType = null;
// 判断是否是集合类型(排除字符串) // 判断是否是集合类型(排除字符串)
if (paramType != typeof(string) && IsEnumerableType(paramType)) if (paramType != typeof(string) && IsEnumerableType(paramType))
{
elementType = GetEnumerableElementType(paramType); elementType = GetEnumerableElementType(paramType);
}
return Tuple.Create(paramType, elementType); return Tuple.Create(paramType, elementType);
} }
/// <summary> /// <summary>
@ -50,7 +41,7 @@ namespace JiShe.CollectBus.Kafka.Internal
|| (type.IsGenericType && type.GetInterfaces() || (type.IsGenericType && type.GetInterfaces()
.Any(t => t.IsGenericType .Any(t => t.IsGenericType
&& t.GetGenericTypeDefinition() == typeof(IEnumerable<>))) && t.GetGenericTypeDefinition() == typeof(IEnumerable<>)))
|| type.GetInterfaces().Any(t => t == typeof(System.Collections.IEnumerable)); || type.GetInterfaces().Any(t => t == typeof(IEnumerable));
} }
/// <summary> /// <summary>
@ -81,20 +72,20 @@ namespace JiShe.CollectBus.Kafka.Internal
} }
// <summary> /// <summary>
/// 判断是否使用强转换 /// 判断是否使用强转换
/// </summary> /// </summary>
/// <param name="targetType">目标类型</param> /// <param name="targetType"></param>
/// <returns></returns> /// <returns></returns>
public static bool IsConvertType(this Type targetType) public static bool IsConvertType(this Type targetType)
{ {
// 处理可空类型 // 处理可空类型
Type underlyingType = Nullable.GetUnderlyingType(targetType) ?? targetType; var underlyingType = Nullable.GetUnderlyingType(targetType) ?? targetType;
// 情况1值类型或基元类型如 int、DateTime // 情况1值类型或基元类型如 int、DateTime
if (underlyingType.IsValueType || underlyingType.IsPrimitive) if (underlyingType.IsValueType || underlyingType.IsPrimitive)
return true; return true;
// 情况2字符串类型直接赋值 // 情况2字符串类型直接赋值
else if (underlyingType == typeof(string)) if (underlyingType == typeof(string))
return true; return true;
// 情况3枚举类型处理 // 情况3枚举类型处理
@ -110,4 +101,3 @@ namespace JiShe.CollectBus.Kafka.Internal
return false; return false;
} }
} }
}

View File

@ -1,13 +1,5 @@
using Confluent.Kafka; namespace JiShe.CollectBus.Kafka.Internal;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace JiShe.CollectBus.Kafka.Internal
{
public class SubscribeResult : ISubscribeAck public class SubscribeResult : ISubscribeAck
{ {
/// <summary> /// <summary>
@ -35,9 +27,7 @@ namespace JiShe.CollectBus.Kafka.Internal
/// <summary> /// <summary>
/// 失败 /// 失败
/// </summary> /// </summary>
/// <param name="code"></param>
/// <param name="msg"></param> /// <param name="msg"></param>
/// <param name="data"></param>
/// <returns></returns> /// <returns></returns>
public SubscribeResult Fail(string? msg = null) public SubscribeResult Fail(string? msg = null)
{ {
@ -47,9 +37,8 @@ namespace JiShe.CollectBus.Kafka.Internal
} }
} }
public static partial class SubscribeAck public static class SubscribeAck
{ {
/// <summary> /// <summary>
/// 成功 /// 成功
/// </summary> /// </summary>
@ -71,5 +60,3 @@ namespace JiShe.CollectBus.Kafka.Internal
return new SubscribeResult().Fail(msg); return new SubscribeResult().Fail(msg);
} }
} }
}

View File

@ -8,26 +8,17 @@ using JiShe.CollectBus.Kafka.Consumer;
using JiShe.CollectBus.Kafka.Internal; using JiShe.CollectBus.Kafka.Internal;
using JiShe.CollectBus.Kafka.Serialization; using JiShe.CollectBus.Kafka.Serialization;
using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Mvc.Abstractions;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using System;
using System.Collections; using System.Collections;
using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
using System.Linq.Expressions;
using System.Reflection; using System.Reflection;
using System.Text.Json;
using System.Threading.Tasks;
using YamlDotNet.Core.Tokens;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace JiShe.CollectBus.Kafka namespace JiShe.CollectBus.Kafka
{ {
public static class KafkaSubcribesExtensions public static class KafkaSubscribeExtensions
{ {
public static void UseInitKafkaTopic(this IServiceProvider provider) public static void UseInitKafkaTopic(this IServiceProvider provider)
@ -36,7 +27,7 @@ namespace JiShe.CollectBus.Kafka
var kafkaAdminClient = provider.GetRequiredService<IAdminClientService>(); var kafkaAdminClient = provider.GetRequiredService<IAdminClientService>();
var kafkaOptions = provider.GetRequiredService<IOptions<KafkaOptionConfig>>(); var kafkaOptions = provider.GetRequiredService<IOptions<KafkaOptionConfig>>();
List<string> topics = ProtocolConstExtensions.GetAllTopicNamesByIssued(); var topics = ProtocolConstExtensions.GetAllTopicNamesByIssued();
topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived()); topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived());
foreach (var item in topics) foreach (var item in topics)
@ -48,8 +39,6 @@ namespace JiShe.CollectBus.Kafka
/// <summary> /// <summary>
/// 添加Kafka订阅 /// 添加Kafka订阅
/// </summary> /// </summary>
/// <param name="app"></param>
/// <param name="assembly"></param>
public static void UseKafkaSubscribe(this IServiceProvider provider) public static void UseKafkaSubscribe(this IServiceProvider provider)
{ {
var lifetime = provider.GetRequiredService<IHostApplicationLifetime>(); var lifetime = provider.GetRequiredService<IHostApplicationLifetime>();
@ -57,8 +46,8 @@ namespace JiShe.CollectBus.Kafka
lifetime.ApplicationStarted.Register(() => lifetime.ApplicationStarted.Register(() =>
{ {
var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>(); var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>();
int threadCount = 0; var threadCount = 0;
int topicCount = 0; var topicCount = 0;
var assemblyPath = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location); var assemblyPath = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location);
if (string.IsNullOrWhiteSpace(assemblyPath)) if (string.IsNullOrWhiteSpace(assemblyPath))
{ {
@ -98,6 +87,9 @@ namespace JiShe.CollectBus.Kafka
}); });
} }
/// <summary>
/// 添加Kafka订阅
/// </summary>
public static void UseKafkaSubscribersAsync(this IApplicationBuilder app, Assembly assembly) public static void UseKafkaSubscribersAsync(this IApplicationBuilder app, Assembly assembly)
{ {
var provider = app.ApplicationServices; var provider = app.ApplicationServices;
@ -134,8 +126,6 @@ namespace JiShe.CollectBus.Kafka
/// <summary> /// <summary>
/// 构建Kafka订阅 /// 构建Kafka订阅
/// </summary> /// </summary>
/// <param name="subscribe"></param>
/// <param name="provider"></param>
private static Tuple<int, int> BuildKafkaSubscribe(object subscribe, IServiceProvider provider, ILogger<CollectBusKafkaModule> logger, KafkaOptionConfig kafkaOptionConfig) private static Tuple<int, int> BuildKafkaSubscribe(object subscribe, IServiceProvider provider, ILogger<CollectBusKafkaModule> logger, KafkaOptionConfig kafkaOptionConfig)
{ {
var subscribedMethods = subscribe.GetType().GetMethods() var subscribedMethods = subscribe.GetType().GetMethods()
@ -169,11 +159,6 @@ namespace JiShe.CollectBus.Kafka
/// <summary> /// <summary>
/// 启动后台消费线程 /// 启动后台消费线程
/// </summary> /// </summary>
/// <param name="config"></param>
/// <param name="attr"></param>
/// <param name="method"></param>
/// <param name="consumerInstance"></param>
/// <returns></returns>
private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr, MethodInfo method, object subscribe, ILogger<CollectBusKafkaModule> logger) private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr, MethodInfo method, object subscribe, ILogger<CollectBusKafkaModule> logger)
{ {
var consumerService = provider.GetRequiredService<IConsumerService>(); var consumerService = provider.GetRequiredService<IConsumerService>();
@ -225,10 +210,6 @@ namespace JiShe.CollectBus.Kafka
/// <summary> /// <summary>
/// 处理消息 /// 处理消息
/// </summary> /// </summary>
/// <param name="message"></param>
/// <param name="method"></param>
/// <param name="subscribe"></param>
/// <returns></returns>
private static async Task<bool> ProcessMessageAsync(List<dynamic> messages, MethodInfo method, object subscribe) private static async Task<bool> ProcessMessageAsync(List<dynamic> messages, MethodInfo method, object subscribe)
{ {
var parameters = method.GetParameters(); var parameters = method.GetParameters();
@ -351,9 +332,6 @@ namespace JiShe.CollectBus.Kafka
} }
return false; return false;
} }
} }

View File

@ -1,9 +1,4 @@
using Confluent.Kafka; using Confluent.Kafka;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Producer namespace JiShe.CollectBus.Kafka.Producer
{ {

View File

@ -23,6 +23,13 @@ namespace JiShe.CollectBus.Kafka.Producer
private readonly ConcurrentDictionary<Type, object> _producerCache = new(); private readonly ConcurrentDictionary<Type, object> _producerCache = new();
private class KafkaProducer<TKey, TValue> where TKey : notnull where TValue : class { } private class KafkaProducer<TKey, TValue> where TKey : notnull where TValue : class { }
private readonly KafkaOptionConfig _kafkaOptionConfig; private readonly KafkaOptionConfig _kafkaOptionConfig;
/// <summary>
/// ProducerService
/// </summary>
/// <param name="configuration"></param>
/// <param name="logger"></param>
/// <param name="kafkaOptionConfig"></param>
public ProducerService(IConfiguration configuration,ILogger<ProducerService> logger, IOptions<KafkaOptionConfig> kafkaOptionConfig) public ProducerService(IConfiguration configuration,ILogger<ProducerService> logger, IOptions<KafkaOptionConfig> kafkaOptionConfig)
{ {
_configuration = configuration; _configuration = configuration;

View File

@ -4,14 +4,12 @@ using Volo.Abp.EventBus;
namespace JiShe.CollectBus.Samples; namespace JiShe.CollectBus.Samples;
[EventName("Sample.Kafka.Test")] [EventName("Sample.Kafka.Test")]
[TopicName("Test1")]
public class SampleDto public class SampleDto
{ {
public int Value { get; set; } public int Value { get; set; }
} }
[EventName("Sample.Kafka.Test2")] [EventName("Sample.Kafka.Test2")]
[TopicName("Test2")]
public class SampleDto2 public class SampleDto2
{ {
public int Value { get; set; } public int Value { get; set; }

View File

@ -11,9 +11,11 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Reflection; using System.Reflection;
using System.Threading.Tasks; using System.Threading.Tasks;
using Cassandra.Mapping;
using JiShe.CollectBus.Cassandra; using JiShe.CollectBus.Cassandra;
using JiShe.CollectBus.FreeRedis; using JiShe.CollectBus.FreeRedis;
using JiShe.CollectBus.IoTDB; using JiShe.CollectBus.IoTDB;
using JiShe.CollectBus.Mappers;
using Volo.Abp; using Volo.Abp;
using Volo.Abp.Application; using Volo.Abp.Application;
using Volo.Abp.Autofac; using Volo.Abp.Autofac;
@ -24,6 +26,8 @@ using Volo.Abp.EventBus;
using Volo.Abp.Modularity; using Volo.Abp.Modularity;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using JiShe.CollectBus.Kafka.Internal; using JiShe.CollectBus.Kafka.Internal;
using JiShe.CollectBus.Interceptors;
using JiShe.CollectBus.Common.Attributes;
namespace JiShe.CollectBus; namespace JiShe.CollectBus;
@ -37,7 +41,7 @@ namespace JiShe.CollectBus;
typeof(CollectBusFreeRedisModule), typeof(CollectBusFreeRedisModule),
typeof(CollectBusFreeSqlModule), typeof(CollectBusFreeSqlModule),
typeof(CollectBusKafkaModule), typeof(CollectBusKafkaModule),
typeof(CollectBusIoTDBModule), typeof(CollectBusIoTDbModule),
typeof(CollectBusCassandraModule) typeof(CollectBusCassandraModule)
)] )]
public class CollectBusApplicationModule : AbpModule public class CollectBusApplicationModule : AbpModule
@ -51,6 +55,23 @@ public class CollectBusApplicationModule : AbpModule
{ {
options.AddMaps<CollectBusApplicationModule>(validate: true); options.AddMaps<CollectBusApplicationModule>(validate: true);
}); });
context.Services.AddSingleton(new MappingConfiguration()
.Define(new CollectBusMapping()));
// 注册拦截器
context.Services.OnRegistered(ctx =>
{
var methods = ctx.ImplementationType.GetMethods();
foreach (var method in methods)
{
var attr = method.GetCustomAttribute(typeof(LogInterceptAttribute), true);
if (attr != null)
{
ctx.Interceptors.TryAdd<LogInterceptor>();
}
}
});
} }
public override async Task OnApplicationInitializationAsync( public override async Task OnApplicationInitializationAsync(
@ -70,10 +91,9 @@ public class CollectBusApplicationModule : AbpModule
//初始化主题信息 //初始化主题信息
var kafkaAdminClient = context.ServiceProvider.GetRequiredService<IAdminClientService>(); var kafkaAdminClient = context.ServiceProvider.GetRequiredService<IAdminClientService>();
var configuration = context.ServiceProvider.GetRequiredService<IConfiguration>();
var kafkaOptions = context.ServiceProvider.GetRequiredService<IOptions<KafkaOptionConfig>>(); var kafkaOptions = context.ServiceProvider.GetRequiredService<IOptions<KafkaOptionConfig>>();
List<string> topics = ProtocolConstExtensions.GetAllTopicNamesByIssued(); var topics = ProtocolConstExtensions.GetAllTopicNamesByIssued();
topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived()); topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived());
foreach (var item in topics) foreach (var item in topics)

View File

@ -0,0 +1,9 @@
using System;
namespace JiShe.CollectBus.Interceptors
{
[AttributeUsage(AttributeTargets.Method)]
public class LogInterceptAttribute : Attribute
{
}
}

View File

@ -0,0 +1,60 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Volo.Abp.DependencyInjection;
using Volo.Abp.DynamicProxy;
namespace JiShe.CollectBus.Interceptors
{
public class LogInterceptor : AbpInterceptor, ITransientDependency
{
public override async Task InterceptAsync(IAbpMethodInvocation invocation)
{
// 方法执行前的逻辑(如果需要)
try
{
// 执行原始方法
await invocation.ProceedAsync();
// 方法执行成功后,返回前的逻辑
await OnSuccessAsync(invocation);
}
catch (Exception ex)
{
// 出现异常时的逻辑
await OnExceptionAsync(invocation, ex);
throw;
}
finally
{
// 方法结束前一定会执行的逻辑
await OnCompleteAsync(invocation);
}
}
private Task OnSuccessAsync(IAbpMethodInvocation invocation)
{
// 方法执行成功后的逻辑
// 可以访问 invocation.ReturnValue 获取返回值
Console.WriteLine($"方法 {invocation.Method.Name} 成功执行,返回值:{invocation.ReturnValue}");
return Task.CompletedTask;
}
private Task OnExceptionAsync(IAbpMethodInvocation invocation, Exception ex)
{
// 方法执行异常时的逻辑
Console.WriteLine($"方法 {invocation.Method.Name} 执行异常:{ex.Message}");
return Task.CompletedTask;
}
private Task OnCompleteAsync(IAbpMethodInvocation invocation)
{
// 无论成功失败,方法执行完毕后的逻辑
Console.WriteLine($"方法 {invocation.Method.Name} 执行完毕");
return Task.CompletedTask;
}
}
}

View File

@ -1,13 +1,8 @@
using Cassandra.Mapping; using Cassandra.Mapping;
using System; using JiShe.CollectBus.IotSystems.Devices;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using JiShe.CollectBus.IotSystems.MessageIssueds; using JiShe.CollectBus.IotSystems.MessageIssueds;
using static Cassandra.QueryTrace;
namespace JiShe.CollectBus.Cassandra.Mappers namespace JiShe.CollectBus.Mappers
{ {
public class CollectBusMapping: Mappings public class CollectBusMapping: Mappings
{ {
@ -15,6 +10,8 @@ namespace JiShe.CollectBus.Cassandra.Mappers
{ {
For<MessageIssued>() For<MessageIssued>()
.Column(e => e.Type, cm => cm.WithName("type").WithDbType<int>()); .Column(e => e.Type, cm => cm.WithName("type").WithDbType<int>());
For<Device>()
.Column(e => e.Status, cm => cm.WithName("status").WithDbType<int>());
} }
} }
} }

View File

@ -42,6 +42,8 @@ namespace JiShe.CollectBus.RedisDataCache
Instance = _freeRedisProvider.Instance; Instance = _freeRedisProvider.Instance;
} }
//todo 单个数据查询
/// <summary> /// <summary>
/// 单个添加数据 /// 单个添加数据
/// </summary> /// </summary>

View File

@ -1,46 +1,41 @@
using System; using JiShe.CollectBus.Ammeters;
using System.Collections.Generic;
using System.Threading.Tasks;
using Apache.IoTDB.DataStructure;
using Apache.IoTDB;
using Confluent.Kafka;
using JiShe.CollectBus.Ammeters;
using JiShe.CollectBus.FreeSql;
using JiShe.CollectBus.IotSystems.PrepayModel;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Logging;
using JiShe.CollectBus.IotSystems.AFNEntity;
using JiShe.CollectBus.Protocol.Contracts.Interfaces;
using Microsoft.Extensions.DependencyInjection;
using JiShe.CollectBus.Common.Consts;
using JiShe.CollectBus.Common.Enums;
using System.Diagnostics.Metrics;
using JiShe.CollectBus.Common.DeviceBalanceControl;
using JiShe.CollectBus.Kafka.Attributes;
using System.Text.Json;
using JiShe.CollectBus.Application.Contracts; using JiShe.CollectBus.Application.Contracts;
using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.Common.Consts;
using System.Diagnostics; using JiShe.CollectBus.Common.DeviceBalanceControl;
using JiShe.CollectBus.Common.Enums;
using JiShe.CollectBus.Common.Extensions;
using JiShe.CollectBus.FreeSql;
using JiShe.CollectBus.IoTDB.Context; using JiShe.CollectBus.IoTDB.Context;
using JiShe.CollectBus.IoTDB.Interface; using JiShe.CollectBus.IoTDB.Interface;
using JiShe.CollectBus.IoTDB.Model;
using JiShe.CollectBus.IoTDB.Options; using JiShe.CollectBus.IoTDB.Options;
using JiShe.CollectBus.IoTDB.Provider;
using JiShe.CollectBus.IotSystems.PrepayModel;
using JiShe.CollectBus.Kafka.Attributes;
using JiShe.CollectBus.Kafka.Internal; using JiShe.CollectBus.Kafka.Internal;
using JiShe.CollectBus.Common.Extensions; using JiShe.CollectBus.Protocol.Contracts.Interfaces;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Samples; namespace JiShe.CollectBus.Samples;
public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaSubscribe public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaSubscribe
{ {
private readonly ILogger<SampleAppService> _logger; private readonly ILogger<SampleAppService> _logger;
private readonly IIoTDBProvider _iotDBProvider; private readonly IIoTDbProvider _iotDBProvider;
private readonly IoTDBRuntimeContext _dbContext; private readonly IoTDbRuntimeContext _dbContext;
private readonly IoTDBOptions _options; private readonly IoTDbOptions _options;
private readonly IRedisDataCacheService _redisDataCacheService; private readonly IRedisDataCacheService _redisDataCacheService;
public SampleAppService(IIoTDBProvider iotDBProvider, IOptions<IoTDBOptions> options, public SampleAppService(IIoTDbProvider iotDBProvider, IOptions<IoTDbOptions> options,
IoTDBRuntimeContext dbContext, ILogger<SampleAppService> logger, IRedisDataCacheService redisDataCacheService) IoTDbRuntimeContext dbContext, ILogger<SampleAppService> logger, IRedisDataCacheService redisDataCacheService)
{ {
_iotDBProvider = iotDBProvider; _iotDBProvider = iotDBProvider;
_options = options.Value; _options = options.Value;
@ -52,23 +47,13 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS
/// <summary> /// <summary>
/// 测试 UseSessionPool /// 测试 UseSessionPool
/// </summary> /// </summary>
/// <param name="timestamps"></param> /// <param name="testTime"></param>
/// <returns></returns> /// <returns></returns>
[HttpGet] [HttpGet]
public async Task UseSessionPool(long timestamps) public async Task UseSessionPool(DateTime testTime)
{ {
string? messageHexString = null;
if (timestamps == 0)
{
timestamps = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
_logger.LogError($"timestamps_{timestamps}");
}
else
{
messageHexString = messageHexString + timestamps;
}
ElectricityMeter meter = new ElectricityMeter() ElectricityMeterTreeModel meter = new ElectricityMeterTreeModel()
{ {
SystemName = "energy", SystemName = "energy",
DeviceId = "402440506", DeviceId = "402440506",
@ -77,8 +62,8 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS
MeterModel = "DDZY-1980", MeterModel = "DDZY-1980",
ProjectCode = "10059", ProjectCode = "10059",
Voltage = 10, Voltage = 10,
IssuedMessageHexString = messageHexString, IssuedMessageHexString = "messageHexString",
Timestamps = timestamps, Timestamps = testTime.GetDateTimeOffset().ToUnixTimeMilliseconds(),
}; };
await _iotDBProvider.InsertAsync(meter); await _iotDBProvider.InsertAsync(meter);
} }
@ -88,9 +73,10 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
[HttpGet] [HttpGet]
public async Task UseTableSessionPool() public async Task UseTableSessionPool(DateTime time)
{ {
ElectricityMeter meter2 = new ElectricityMeter() var testTime = time;
ElectricityMeterTreeModel meter2 = new ElectricityMeterTreeModel()
{ {
SystemName = "energy", SystemName = "energy",
DeviceId = "402440506", DeviceId = "402440506",
@ -99,7 +85,7 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS
MeterModel = "DDZY-1980", MeterModel = "DDZY-1980",
ProjectCode = "10059", ProjectCode = "10059",
Voltage = 10, Voltage = 10,
Timestamps = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), Timestamps = testTime.GetDateTimeOffset().ToUnixTimeMilliseconds(),
}; };
await _iotDBProvider.InsertAsync(meter2); await _iotDBProvider.InsertAsync(meter2);
@ -115,9 +101,135 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS
MeterModel = "DDZY-1980", MeterModel = "DDZY-1980",
ProjectCode = "10059", ProjectCode = "10059",
Voltage = 10, Voltage = 10,
Timestamps = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), Timestamps = testTime.GetDateTimeOffset().ToUnixTimeMilliseconds(),
}; };
await _iotDBProvider.InsertAsync(meter); await _iotDBProvider.InsertAsync(meter);
}
/// <summary>
/// 测试Session切换3
/// </summary>
/// <returns></returns>
[HttpGet]
public async Task UseTableSessionPool3(DateTime time)
{
var testTime = time;
ElectricityMeterTreeModel meter2 = new ElectricityMeterTreeModel()
{
SystemName = "energy",
DeviceId = "402440506",
DeviceType = "Ammeter",
Current = 10,
MeterModel = "DDZY-1980",
ProjectCode = "10059",
Voltage = 10,
IssuedMessageHexString = "dsdfsfd",
Timestamps = testTime.GetDateTimeOffset().ToUnixTimeMilliseconds(),
};
await _iotDBProvider.InsertAsync(meter2);
_dbContext.UseTableSessionPool = true;
ElectricityMeter meter3 = new ElectricityMeter()
{
SystemName = "energy",
DeviceId = "402440506",
DeviceType = "Ammeter",
Current = 10,
MeterModel = "DDZY-1980",
ProjectCode = "10059",
Voltage = 10,
Currentd = 22,
IssuedMessageHexString = "dsdfsfd",
Timestamps = testTime.GetDateTimeOffset().ToUnixTimeMilliseconds(),
};
await _iotDBProvider.InsertAsync(meter3);
}
/// <summary>
/// 测试树模型单个测点数据项
/// </summary>
/// <param name="measuring"></param>
/// <returns></returns>
[HttpGet]
public async Task TestTreeModelSingleMeasuringEntity(string measuring, string value, DateTime time)
{
var meter = new TreeModelSingleMeasuringEntity<string>()
{
SystemName = "energy",
DeviceId = "402440506",
DeviceType = "Ammeter",
ProjectCode = "10059",
Timestamps = time.GetDateTimeOffset().ToUnixTimeMilliseconds(),
SingleMeasuring = new Tuple<string, string>(measuring, value)
};
await _iotDBProvider.InsertAsync(meter);
}
/// <summary>
/// 测试树模型单个测点数据项2
/// </summary>
/// <param name="measuring"></param>
/// <returns></returns>
[HttpGet]
public async Task TestTreeModelSingleMeasuringEntity2(string measuring, int value, DateTime time)
{
var meter = new TreeModelSingleMeasuringEntity<int>()
{
SystemName = "energy",
DeviceId = "402440506",
DeviceType = "Ammeter",
ProjectCode = "10059",
Timestamps = time.GetDateTimeOffset().ToUnixTimeMilliseconds(),
SingleMeasuring = new Tuple<string, int>(measuring, value)
};
await _iotDBProvider.InsertAsync(meter);
}
/// <summary>
/// 测试表模型单个测点数据项
/// </summary>
/// <param name="measuring"></param>
/// <returns></returns>
[HttpGet]
public async Task TestTableModelSingleMeasuringEntity(string measuring, string value, DateTime time)
{
var meter = new TableModelSingleMeasuringEntity<string>()
{
SystemName = "energy",
DeviceId = "402440506",
DeviceType = "Ammeter",
ProjectCode = "10059",
Timestamps = time.GetDateTimeOffset().ToUnixTimeMilliseconds(),
SingleColumn = new Tuple<string, string>(measuring, value)
};
_dbContext.UseTableSessionPool = true;
await _iotDBProvider.InsertAsync(meter);
}
/// <summary>
/// 测试表模型单个测点数据项2
/// </summary>
/// <param name="measuring"></param>
/// <returns></returns>
[HttpGet]
public async Task TestTableModelSingleMeasuringEntity2(string measuring, int value, DateTime time)
{
var meter = new TableModelSingleMeasuringEntity<int>()
{
SystemName = "energy",
DeviceId = "402440506",
DeviceType = "Ammeter",
ProjectCode = "10059",
Timestamps = time.GetDateTimeOffset().ToUnixTimeMilliseconds(),
SingleColumn = new Tuple<string, int>(measuring, value)
};
_dbContext.UseTableSessionPool = true;
await _iotDBProvider.InsertAsync(meter);
} }
/// <summary> /// <summary>
@ -172,26 +284,6 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS
} }
/// <summary>
/// 测试单个测点数据项
/// </summary>
/// <param name="measuring"></param>
/// <returns></returns>
[HttpGet]
public async Task TestSingleMeasuringAFNData(string measuring, string value)
{
var meter = new SingleMeasuringAFNDataEntity<string>()
{
SystemName = "energy",
DeviceId = "402440506",
DeviceType = "Ammeter",
ProjectCode = "10059",
Timestamps = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
SingleMeasuring = new Tuple<string, string>(measuring, value)
};
await _iotDBProvider.InsertAsync(meter);
}
/// <summary> /// <summary>
/// 测试Redis批量读取10万条数据性能 /// 测试Redis批量读取10万条数据性能
/// </summary> /// </summary>

View File

@ -12,7 +12,6 @@ using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Common.Helpers;
using JiShe.CollectBus.IotSystems.AFNEntity;
using JiShe.CollectBus.Protocol.Contracts.Interfaces; using JiShe.CollectBus.Protocol.Contracts.Interfaces;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using JiShe.CollectBus.Cassandra; using JiShe.CollectBus.Cassandra;
@ -25,6 +24,7 @@ using Volo.Abp.Domain.Repositories;
using System.Diagnostics; using System.Diagnostics;
using System.Linq; using System.Linq;
using Cassandra; using Cassandra;
using JiShe.CollectBus.Interceptors;
namespace JiShe.CollectBus.Samples; namespace JiShe.CollectBus.Samples;
@ -35,11 +35,10 @@ public class TestAppService : CollectBusAppService
private readonly ICassandraRepository<MessageIssued, string> _messageReceivedCassandraRepository; private readonly ICassandraRepository<MessageIssued, string> _messageReceivedCassandraRepository;
private readonly ICassandraProvider _cassandraProvider; private readonly ICassandraProvider _cassandraProvider;
public TestAppService( public TestAppService(
ILogger<TestAppService> logger, ILogger<TestAppService> logger,
ICassandraRepository<MessageIssued, string> messageReceivedCassandraRepository, ICassandraProvider cassandraProvider) ICassandraRepository<MessageIssued, string> messageReceivedCassandraRepository,
ICassandraProvider cassandraProvider)
{ {
_logger = logger; _logger = logger;
_messageReceivedCassandraRepository = messageReceivedCassandraRepository; _messageReceivedCassandraRepository = messageReceivedCassandraRepository;
@ -122,4 +121,11 @@ public class TestAppService : CollectBusAppService
// 等待所有批处理完成 // 等待所有批处理完成
await Task.WhenAll(tasks); await Task.WhenAll(tasks);
} }
[LogIntercept]
public async Task<string> LogInterceptorTest(string str)
{
_logger.LogWarning(str);
return str;
}
} }

View File

@ -37,7 +37,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading
public abstract class BasicScheduledMeterReadingService : CollectBusAppService, IScheduledMeterReadingService public abstract class BasicScheduledMeterReadingService : CollectBusAppService, IScheduledMeterReadingService
{ {
private readonly ILogger<BasicScheduledMeterReadingService> _logger; private readonly ILogger<BasicScheduledMeterReadingService> _logger;
private readonly IIoTDBProvider _dbProvider; private readonly IIoTDbProvider _dbProvider;
private readonly IMeterReadingRecordRepository _meterReadingRecordRepository; private readonly IMeterReadingRecordRepository _meterReadingRecordRepository;
private readonly IProducerService _producerService; private readonly IProducerService _producerService;
private readonly IRedisDataCacheService _redisDataCacheService; private readonly IRedisDataCacheService _redisDataCacheService;
@ -48,7 +48,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading
IMeterReadingRecordRepository meterReadingRecordRepository, IMeterReadingRecordRepository meterReadingRecordRepository,
IProducerService producerService, IProducerService producerService,
IRedisDataCacheService redisDataCacheService, IRedisDataCacheService redisDataCacheService,
IIoTDBProvider dbProvider, IIoTDbProvider dbProvider,
IOptions<KafkaOptionConfig> kafkaOptions) IOptions<KafkaOptionConfig> kafkaOptions)
{ {
_logger = logger; _logger = logger;

View File

@ -37,7 +37,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading
string serverTagName = string.Empty; string serverTagName = string.Empty;
public EnergySystemScheduledMeterReadingService( public EnergySystemScheduledMeterReadingService(
ILogger<EnergySystemScheduledMeterReadingService> logger, ILogger<EnergySystemScheduledMeterReadingService> logger,
IIoTDBProvider dbProvider, IIoTDbProvider dbProvider,
IMeterReadingRecordRepository meterReadingRecordRepository, IMeterReadingRecordRepository meterReadingRecordRepository,
IOptions<KafkaOptionConfig> kafkaOptions, IOptions<KafkaOptionConfig> kafkaOptions,
IProducerService producerService, IProducerService producerService,

View File

@ -30,10 +30,8 @@ namespace JiShe.CollectBus.Subscribers
private readonly IServiceProvider _serviceProvider; private readonly IServiceProvider _serviceProvider;
private readonly IRepository<MessageReceivedLogin, Guid> _messageReceivedLoginEventRepository; private readonly IRepository<MessageReceivedLogin, Guid> _messageReceivedLoginEventRepository;
private readonly IRepository<MessageReceivedHeartbeat, Guid> _messageReceivedHeartbeatEventRepository; private readonly IRepository<MessageReceivedHeartbeat, Guid> _messageReceivedHeartbeatEventRepository;
private readonly IRepository<MessageReceived, Guid> _messageReceivedEventRepository;
private readonly IRepository<Device, Guid> _deviceRepository;
private readonly IMeterReadingRecordRepository _meterReadingRecordsRepository; private readonly IMeterReadingRecordRepository _meterReadingRecordsRepository;
private readonly IIoTDBProvider _dbProvider; private readonly IIoTDbProvider _dbProvider;
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="SubscriberAppService"/> class. /// Initializes a new instance of the <see cref="SubscriberAppService"/> class.
@ -43,16 +41,13 @@ namespace JiShe.CollectBus.Subscribers
/// <param name="serviceProvider">The service provider.</param> /// <param name="serviceProvider">The service provider.</param>
/// <param name="messageReceivedLoginEventRepository">The message received login event repository.</param> /// <param name="messageReceivedLoginEventRepository">The message received login event repository.</param>
/// <param name="messageReceivedHeartbeatEventRepository">The message received heartbeat event repository.</param> /// <param name="messageReceivedHeartbeatEventRepository">The message received heartbeat event repository.</param>
/// <param name="messageReceivedEventRepository">The message received event repository.</param>
/// <param name="deviceRepository">The device repository.</param>
/// <param name="meterReadingRecordsRepository">The device repository.</param> /// <param name="meterReadingRecordsRepository">The device repository.</param>
public SubscriberAppService(ILogger<SubscriberAppService> logger, public SubscriberAppService(ILogger<SubscriberAppService> logger,
ITcpService tcpService, IServiceProvider serviceProvider, ITcpService tcpService,
IServiceProvider serviceProvider,
IRepository<MessageReceivedLogin, Guid> messageReceivedLoginEventRepository, IRepository<MessageReceivedLogin, Guid> messageReceivedLoginEventRepository,
IRepository<MessageReceivedHeartbeat, Guid> messageReceivedHeartbeatEventRepository, IRepository<MessageReceivedHeartbeat, Guid> messageReceivedHeartbeatEventRepository,
IRepository<MessageReceived, Guid> messageReceivedEventRepository, IIoTDbProvider dbProvider,
IRepository<Device, Guid> deviceRepository,
IIoTDBProvider dbProvider,
IMeterReadingRecordRepository meterReadingRecordsRepository) IMeterReadingRecordRepository meterReadingRecordsRepository)
{ {
_logger = logger; _logger = logger;
@ -60,8 +55,6 @@ namespace JiShe.CollectBus.Subscribers
_serviceProvider = serviceProvider; _serviceProvider = serviceProvider;
_messageReceivedLoginEventRepository = messageReceivedLoginEventRepository; _messageReceivedLoginEventRepository = messageReceivedLoginEventRepository;
_messageReceivedHeartbeatEventRepository = messageReceivedHeartbeatEventRepository; _messageReceivedHeartbeatEventRepository = messageReceivedHeartbeatEventRepository;
_messageReceivedEventRepository = messageReceivedEventRepository;
_deviceRepository = deviceRepository;
_meterReadingRecordsRepository = meterReadingRecordsRepository; _meterReadingRecordsRepository = meterReadingRecordsRepository;
_dbProvider = dbProvider; _dbProvider = dbProvider;
} }

View File

@ -10,7 +10,6 @@ using Volo.Abp.Uow;
namespace JiShe.CollectBus.Workers namespace JiShe.CollectBus.Workers
{ {
[IgnoreJob]
public class EpiCollectWorker : HangfireBackgroundWorkerBase, ITransientDependency,ICollectWorker public class EpiCollectWorker : HangfireBackgroundWorkerBase, ITransientDependency,ICollectWorker
{ {
private readonly ILogger<EpiCollectWorker> _logger; private readonly ILogger<EpiCollectWorker> _logger;

View File

@ -4,10 +4,12 @@ using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using JiShe.CollectBus.IoTDB.Attribute; using JiShe.CollectBus.IoTDB.Attribute;
using JiShe.CollectBus.IoTDB.Provider; using JiShe.CollectBus.IoTDB.Enums;
using JiShe.CollectBus.IoTDB.Model;
namespace JiShe.CollectBus.Ammeters namespace JiShe.CollectBus.Ammeters
{ {
[EntityType(EntityTypeEnum.TableModel)]
public class ElectricityMeter : IoTEntity public class ElectricityMeter : IoTEntity
{ {
[ATTRIBUTEColumn] [ATTRIBUTEColumn]
@ -33,5 +35,8 @@ namespace JiShe.CollectBus.Ammeters
[FIELDColumn] [FIELDColumn]
public double Power => Voltage * Current; public double Power => Voltage * Current;
[FIELDColumn]
public double? Currentd { get; set; }
} }
} }

View File

@ -0,0 +1,37 @@
using JiShe.CollectBus.IoTDB.Attribute;
using JiShe.CollectBus.IoTDB.Enums;
using JiShe.CollectBus.IoTDB.Model;
namespace JiShe.CollectBus.Ammeters
{
[EntityType(EntityTypeEnum.TreeModel)]
public class ElectricityMeterTreeModel : IoTEntity
{
[ATTRIBUTEColumn]
public string MeterModel { get; set; }
/// <summary>
/// 下发消息内容
/// </summary>
[FIELDColumn]
public string IssuedMessageHexString { get; set; }
///// <summary>
///// 下发消息Id
///// </summary>
//[FIELDColumn]
//public string IssuedMessageId { get; set; }
[FIELDColumn]
public double Voltage { get; set; }
[FIELDColumn]
public double Current { get; set; }
[FIELDColumn]
public double Power => Voltage * Current;
[FIELDColumn]
public double? Currentd { get; set; }
}
}

View File

@ -1,14 +1,18 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using JiShe.CollectBus.Common.Attributes;
using JiShe.CollectBus.Enums; using JiShe.CollectBus.Enums;
using Volo.Abp.Auditing;
using Volo.Abp.Domain.Entities; using Volo.Abp.Domain.Entities;
using Volo.Abp.Logging;
namespace JiShe.CollectBus.IotSystems.Devices namespace JiShe.CollectBus.IotSystems.Devices
{ {
public class Device : AggregateRoot<Guid> public class Device : BasicAggregateRoot<Guid>
{ {
/// <summary> /// <summary>
/// Device /// Device
@ -20,6 +24,7 @@ namespace JiShe.CollectBus.IotSystems.Devices
/// <param name="status"></param> /// <param name="status"></param>
public Device(string number, string clientId, DateTime firstOnlineTime, DateTime lastOnlineTime, DeviceStatus status) public Device(string number, string clientId, DateTime firstOnlineTime, DateTime lastOnlineTime, DeviceStatus status)
{ {
Id = Guid.NewGuid();
Number = number; Number = number;
FirstOnlineTime = firstOnlineTime; FirstOnlineTime = firstOnlineTime;
LastOnlineTime = lastOnlineTime; LastOnlineTime = lastOnlineTime;
@ -30,6 +35,7 @@ namespace JiShe.CollectBus.IotSystems.Devices
/// <summary> /// <summary>
/// 集中器编号在集中器登录时解析获取并会更新为当前TCP连接的最新ClientId /// 集中器编号在集中器登录时解析获取并会更新为当前TCP连接的最新ClientId
/// </summary> /// </summary>
[PartitionKey]
public string Number { get; set; } public string Number { get; set; }
/// <summary> /// <summary>
@ -55,6 +61,7 @@ namespace JiShe.CollectBus.IotSystems.Devices
/// <summary> /// <summary>
/// 设备状态 /// 设备状态
/// </summary> /// </summary>
[PartitionKey]
public DeviceStatus Status { get; set; } public DeviceStatus Status { get; set; }
/// <summary> /// <summary>

View File

@ -11,19 +11,14 @@ using Volo.Abp.Domain.Entities;
namespace JiShe.CollectBus.IotSystems.MessageIssueds namespace JiShe.CollectBus.IotSystems.MessageIssueds
{ {
[CassandraTable] [CassandraTable]
public class MessageIssued:IEntity<string> public class MessageIssued: ICassandraEntity<string>
{ {
public string ClientId { get; set; } public string ClientId { get; set; }
public byte[] Message { get; set; } public byte[] Message { get; set; }
public string DeviceNo { get; set; } public string DeviceNo { get; set; }
public IssuedEventType Type { get; set; } public IssuedEventType Type { get; set; }
public string MessageId { get; set; } public string MessageId { get; set; }
[Key] [PartitionKey]
public string Id { get; set; } public string Id { get; set; }
public object?[] GetKeys()
{
return new object[] { Id };
}
} }
} }

View File

@ -21,4 +21,20 @@ namespace JiShe.CollectBus.Common.Attributes
{ {
} }
} }
[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field)]
public class PartitionKeyAttribute : Attribute
{
public PartitionKeyAttribute()
{
}
}
[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field)]
public class ClusteringKeyAttribute : Attribute
{
public ClusteringKeyAttribute()
{
}
}
} }

View File

@ -1,13 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Common.Attributes
{
[AttributeUsage(AttributeTargets.Class, Inherited = false)]
public class IgnoreJobAttribute : Attribute
{
}
}

View File

@ -4,7 +4,7 @@ using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
namespace JiShe.CollectBus.Common.AttributeInfo namespace JiShe.CollectBus.Common.Attributes
{ {
/// <summary> /// <summary>
/// 排序序号 /// 排序序号

View File

@ -1,23 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Volo.Abp.EventBus;
using Volo.Abp;
namespace JiShe.CollectBus.Common.Attributes
{
[AttributeUsage(AttributeTargets.Class, Inherited = false)]
public class TopicNameAttribute : Attribute
{
public virtual string Name { get; }
public TopicNameAttribute(string name)
{
this.Name = Check.NotNullOrWhiteSpace(name, nameof(name));
}
public string GetName(Type eventType) => this.Name;
}
}

View File

@ -233,5 +233,22 @@ namespace JiShe.CollectBus.Common.Extensions
.AddHours(hours) .AddHours(hours)
.AddMinutes(minutes); .AddMinutes(minutes);
} }
/// <summary>
/// 将 DateTime 时间转换为 DateTimeOffset 时间
/// </summary>
/// <param name="rawDateTime"></param>
/// <returns></returns>
public static DateTimeOffset GetDateTimeOffset(this DateTime rawDateTime)
{
//确保 Kind 为 Local如果是 Unspecified
DateTime localDateTime = rawDateTime.Kind == DateTimeKind.Unspecified
? DateTime.SpecifyKind(rawDateTime, DateTimeKind.Local)
: rawDateTime;
// 转换为 DateTimeOffset自动应用本地时区偏移
return new DateTimeOffset(localDateTime);
}
} }
} }

View File

@ -8,7 +8,7 @@ using System.Runtime.InteropServices;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using JiShe.CollectBus.Common.AttributeInfo; using JiShe.CollectBus.Common.Attributes;
namespace JiShe.CollectBus.Common.Helpers namespace JiShe.CollectBus.Common.Helpers
{ {

View File

@ -0,0 +1,14 @@
using System;
using System.ComponentModel.DataAnnotations;
using JiShe.CollectBus.Common.Attributes;
namespace JiShe.CollectBus
{
public class CassandraBaseEntity<TKey>: ICassandraEntity<TKey>
{
/// <summary>
/// Id
/// </summary>
public TKey Id { get; set; }
}
}

View File

@ -0,0 +1,19 @@
using System;
namespace JiShe.CollectBus
{
public interface ICassandraEntity<TKey>
{
TKey Id { get; set; }
}
public interface IHasCreationTime
{
DateTime CreationTime { get; set; }
}
public interface IHasLastModificationTime
{
DateTime? LastModificationTime { get; set; }
}
}

View File

@ -31,4 +31,8 @@
<None Remove="JiShe.CollectBus.Domain.Shared.abppkg" /> <None Remove="JiShe.CollectBus.Domain.Shared.abppkg" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj" />
</ItemGroup>
</Project> </Project>

View File

@ -84,7 +84,7 @@
"SaslPassword": "lixiao1980", "SaslPassword": "lixiao1980",
"KafkaReplicationFactor": 3, "KafkaReplicationFactor": 3,
"NumPartitions": 30, "NumPartitions": 30,
"ServerTagName": "JiSheCollectBus99" "ServerTagName": "JiSheCollectBus20"
}, },
"IoTDBOptions": { "IoTDBOptions": {
"UserName": "root", "UserName": "root",
@ -95,7 +95,6 @@
"OpenDebugMode": true, "OpenDebugMode": true,
"UseTableSessionPoolByDefault": false "UseTableSessionPoolByDefault": false
}, },
"ServerTagName": "JiSheCollectBus3",
"Cassandra": { "Cassandra": {
"ReplicationStrategy": { "ReplicationStrategy": {
"Class": "NetworkTopologyStrategy", //NetworkTopologyStrategySimpleStrategy "Class": "NetworkTopologyStrategy", //NetworkTopologyStrategySimpleStrategy