From 7b1f04a45207f0826aaeb46da36c59ea8b3f4638 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Tue, 15 Apr 2025 16:45:10 +0800 Subject: [PATCH 01/27] =?UTF-8?q?=E6=9A=82=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Samples/SampleAppService.cs | 4 +- .../ISubscribeAck.cs | 21 ++++++ .../KafkaSubcribesExtensions.cs | 12 ++- .../SubscribeResult.cs | 75 +++++++++++++++++++ 4 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 src/JiShe.CollectBus.KafkaProducer/ISubscribeAck.cs create mode 100644 src/JiShe.CollectBus.KafkaProducer/SubscribeResult.cs diff --git a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs index e4c078d..3e1c2ea 100644 --- a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs +++ b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs @@ -217,9 +217,9 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS [KafkaSubscribe(["test-topic"])] - public async Task KafkaSubscribeAsync(string obj) + public async Task KafkaSubscribeAsync(object obj) { _logger.LogWarning($"收到订阅消息： {obj}"); - return await Task.FromResult(true); + return SubscribeAck.Success(); } } diff --git a/src/JiShe.CollectBus.KafkaProducer/ISubscribeAck.cs b/src/JiShe.CollectBus.KafkaProducer/ISubscribeAck.cs new file mode 100644 index 0000000..ffb30ef --- /dev/null +++ b/src/JiShe.CollectBus.KafkaProducer/ISubscribeAck.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JiShe.CollectBus.Kafka +{ + public interface ISubscribeAck + { + /// + /// 是否成功标记 + /// + bool Ack { get; set; } + + /// + /// 消息 + /// + string? Msg { get; set; } + } +} diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index 6a5f5cb..6cf2b60 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -115,14 +115,18 @@ namespace JiShe.CollectBus.Kafka if (method.ReturnType == typeof(Task)) { object? result = await (Task)method.Invoke(subscribe, new[] { messageObj })!; - if (result is bool success) - return success; + if (result is ISubscribeAck ackResult) + { + return ackResult.Ack; + } } else { object? result = method.Invoke(subscribe, new[] { messageObj }); - if (result is bool success) - return success; + if (result is ISubscribeAck ackResult) + { + return ackResult.Ack; + } } return false; diff --git a/src/JiShe.CollectBus.KafkaProducer/SubscribeResult.cs b/src/JiShe.CollectBus.KafkaProducer/SubscribeResult.cs new file mode 100644 index 0000000..83eaa49 --- /dev/null +++ b/src/JiShe.CollectBus.KafkaProducer/SubscribeResult.cs @@ -0,0 +1,75 @@ +using Confluent.Kafka; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using static System.Runtime.InteropServices.JavaScript.JSType; + +namespace JiShe.CollectBus.Kafka +{ + public class SubscribeResult: ISubscribeAck + { + /// + /// 是否成功 + /// + public bool Ack { get; set; } + + /// + /// 消息 + /// + public string? Msg { get; set; } + + + /// + /// 成功 + /// + /// 消息 + public SubscribeResult Success(string? msg = null) + { + Ack = true; + Msg = msg; + return this; + } + + /// + /// 失败 + /// + /// + /// + /// + /// + public SubscribeResult Fail(string? msg = null) + { + Msg = msg; + Ack = false; + return this; + } + } + + public static partial class SubscribeAck + { + + /// + /// 成功 + /// + /// 消息 + /// + public static ISubscribeAck Success(string? msg = null) + { + return new SubscribeResult().Success(msg); + } + + + /// + /// 失败 + /// + /// 消息 + /// + public static ISubscribeAck Fail(string? msg = null) + { + return new SubscribeResult().Fail(msg); + } + } + +} From abde2b51616f4a7783d4ac2165e0e69844209a1b Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Tue, 15 Apr 2025 16:48:35 +0800 Subject: [PATCH 02/27] =?UTF-8?q?=E4=BC=98=E5=8C=96Redis=E5=A4=A7=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E6=95=B0=E6=8D=AE=E6=B7=BB=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BasicScheduledMeterReadingService.cs | 58 ++--- ...nergySystemScheduledMeterReadingService.cs | 22 +- ...Result.cs => BusCacheGlobalPagedResult.cs} | 6 +- .../Models/DeviceCacheBasicModel.cs | 7 +- .../Ammeters/AmmeterInfo.cs | 17 +- .../IotSystems/Watermeter/WatermeterInfo.cs | 17 +- .../FreeRedisProvider.cs | 238 ++++++------------ .../IFreeRedisProvider.cs | 39 +++ src/JiShe.CollectBus.Host/appsettings.json | 2 +- 9 files changed, 178 insertions(+), 228 deletions(-) rename src/JiShe.CollectBus.Common/Models/{BusGlobalPagedResult.cs => BusCacheGlobalPagedResult.cs} (80%) diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index 287158f..5a665ff 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -207,26 +207,26 @@ namespace JiShe.CollectBus.ScheduledMeterReading /// public virtual async Task InitAmmeterCacheData(string gatherCode = "") { -#if DEBUG - var timeDensity = "15"; - //获取缓存中的电表信息 - var redisKeyList = $"{string.Format(RedisConst.CacheMeterInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}*"; +//#if DEBUG +// var timeDensity = "15"; +// //获取缓存中的电表信息 +// var redisKeyList = $"{string.Format(RedisConst.CacheMeterInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}*"; - var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); - var meterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); - List focusAddressDataLista = new List(); - foreach (var item in meterInfos) - { - focusAddressDataLista.Add(item.FocusAddress); - } +// var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); +// var meterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); +// List focusAddressDataLista = new List(); +// foreach (var item in meterInfos) +// { +// focusAddressDataLista.Add(item.FocusAddress); +// } + +// DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); +// return; +//#else +// var meterInfos = await GetAmmeterInfoList(gatherCode); +//#endif - DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); - return; -#else var meterInfos = await GetAmmeterInfoList(gatherCode); -#endif - - if (meterInfos == null || meterInfos.Count <= 0) { throw new NullReferenceException($"{nameof(InitAmmeterCacheData)} 初始化电表缓存数据时，电表数据为空"); @@ -311,7 +311,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading } } - keyValuePairs.TryAdd($"{ammeter.ID}", ammeter); + keyValuePairs.TryAdd($"{ammeter.MeterId}", ammeter); } await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs); } @@ -757,10 +757,10 @@ namespace JiShe.CollectBus.ScheduledMeterReading PendingCopyReadTime = pendingCopyReadTime, CreationTime = currentTime, MeterAddress = ammeterInfo.AmmerterAddress, - MeterId = ammeterInfo.ID, + MeterId = ammeterInfo.MeterId, MeterType = MeterTypeEnum.Ammeter, FocusAddress = ammeterInfo.FocusAddress, - FocusID = ammeterInfo.FocusID, + FocusID = ammeterInfo.FocusId, AFN = aFN, Fn = fn, ItemCode = tempItem, @@ -772,7 +772,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading }; meterReadingRecords.CreateDataId(GuidGenerator.Create()); - keyValuePairs.TryAdd($"{ammeterInfo.ID}_{tempItem}", meterReadingRecords); + keyValuePairs.TryAdd($"{ammeterInfo.MeterId}_{tempItem}", meterReadingRecords); } //TimeSpan timeSpan = TimeSpan.FromMicroseconds(5); //await Task.Delay(timeSpan); @@ -910,22 +910,22 @@ namespace JiShe.CollectBus.ScheduledMeterReading if (string.IsNullOrWhiteSpace(ammeter.AreaCode)) { - _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeter.ID},集中器通信区号为空"); + _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeter.MeterId},集中器通信区号为空"); continue; } if (string.IsNullOrWhiteSpace(ammeter.Address)) { - _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeter.ID},集中器通信地址为空"); + _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeter.MeterId},集中器通信地址为空"); continue; } if (Convert.ToInt32(ammeter.Address) > 65535) { - _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeter.ID},集中器通信地址无效,确保大于65535"); + _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeter.MeterId},集中器通信地址无效,确保大于65535"); continue; } if (ammeter.MeteringCode <= 0 || ammeter.MeteringCode > 2033) { - _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeter.ID},非有效测量点号({ammeter.MeteringCode})"); + _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeter.MeterId},非有效测量点号({ammeter.MeteringCode})"); continue; } @@ -1023,10 +1023,10 @@ namespace JiShe.CollectBus.ScheduledMeterReading PendingCopyReadTime = pendingCopyReadTime, CreationTime = currentTime, MeterAddress = ammeter.AmmerterAddress, - MeterId = ammeter.ID, + MeterId = ammeter.MeterId, MeterType = MeterTypeEnum.Ammeter, FocusAddress = ammeter.FocusAddress, - FocusID = ammeter.FocusID, + FocusID = ammeter.FocusId, AFN = aFN, Fn = fn, ItemCode = tempItem, @@ -1038,7 +1038,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading }; meterReadingRecords.CreateDataId(GuidGenerator.Create()); - keyValuePairs.TryAdd($"{ammeter.ID}_{tempItem}", meterReadingRecords); + keyValuePairs.TryAdd($"{ammeter.MeterId}_{tempItem}", meterReadingRecords); } await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs); } @@ -1097,7 +1097,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading foreach (var subItem in item) { - keyValuePairs.TryAdd($"{subItem.ID}", subItem); + keyValuePairs.TryAdd($"{subItem.MeterId}", subItem); } await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs); } diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs index 91f7d4a..07f5526 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs @@ -80,11 +80,11 @@ namespace JiShe.CollectBus.ScheduledMeterReading // Baudrate = 2400, // FocusAddress = "402440506", // Name = "张家祠工务(三相电表)", - // FocusID = 95780, + // FocusId = 95780, // DatabaseBusiID = 1, // MeteringCode = 1, // AmmerterAddress = "402410040506", - // ID = 127035, + // MeterId = 127035, // TypeName = 3, // DataTypes = "449,503,581,582,583,584,585,586,587,588,589,590,591,592,593,594,597,598,599,600,601,602,603,604,605,606,607,608,661,663,677,679", // TimeDensity = 15, @@ -94,11 +94,11 @@ namespace JiShe.CollectBus.ScheduledMeterReading // Baudrate = 2400, // FocusAddress = "542400504", // Name = "五号配（长芦二所四排）(单相电表)", - // FocusID = 69280, + // FocusId = 69280, // DatabaseBusiID = 1, // MeteringCode = 2, // AmmerterAddress = "542410000504", - // ID = 95594, + // MeterId = 95594, // TypeName = 1, // DataTypes = "581,589,592,597,601", // TimeDensity = 15, @@ -106,7 +106,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading //return ammeterInfos; - string sql = $@"SELECT C.ID,C.Name,C.FocusID,C.SingleRate,C.MeteringCode,C.Code AS BrandType,C.Baudrate,C.Password,C.MeteringPort,C.[Address] AS AmmerterAddress,C.TypeName,C.Protocol,C.TripState,C.[State],B.[Address],B.AreaCode,B.AutomaticReport,D.DataTypes,B.TimeDensity,A.GatherCode,C.Special,C.[ProjectID],B.AbnormalState,B.LastTime,CONCAT(B.AreaCode, B.[Address]) AS FocusAddress,(select top 1 DatabaseBusiID from TB_Project where ID = B.ProjectID) AS DatabaseBusiID + string sql = $@"SELECT C.ID as MeterId,C.Name,C.FocusID as FocusId,C.SingleRate,C.MeteringCode,C.Code AS BrandType,C.Baudrate,C.Password,C.MeteringPort,C.[Address] AS AmmerterAddress,C.TypeName,C.Protocol,C.TripState,C.[State],B.[Address],B.AreaCode,B.AutomaticReport,D.DataTypes,B.TimeDensity,A.GatherCode,C.Special,C.[ProjectID],B.AbnormalState,B.LastTime,CONCAT(B.AreaCode, B.[Address]) AS FocusAddress,(select top 1 DatabaseBusiID from TB_Project where ID = B.ProjectID) AS DatabaseBusiID FROM TB_GatherInfo(NOLOCK) AS A INNER JOIN TB_FocusInfo(NOLOCK) AS B ON A.ID = B.GatherInfoID AND B.RemoveState >= 0 AND B.State>=0 INNER JOIN TB_AmmeterInfo(NOLOCK) AS C ON B.ID = C.FocusID AND C.State>= 0 AND C.State<100 @@ -114,10 +114,10 @@ namespace JiShe.CollectBus.ScheduledMeterReading WHERE 1=1 and C.Special = 0 "; //TODO 记得移除特殊表过滤 - //if (!string.IsNullOrWhiteSpace(gatherCode)) - //{ - // sql = $@"{sql} AND A.GatherCode = '{gatherCode}'"; - //} + if (!string.IsNullOrWhiteSpace(gatherCode)) + { + sql = $@"{sql} AND A.GatherCode = '{gatherCode}'"; + } return await SqlProvider.Instance.Change(DbEnum.EnergyDB) .Ado .QueryAsync(sql); @@ -133,9 +133,9 @@ namespace JiShe.CollectBus.ScheduledMeterReading public override async Task> GetWatermeterInfoList(string gatherCode = "V4-Gather-8890") { string sql = $@"SELECT - A.ID, + A.ID as MeterId, A.Name, - A.FocusID, + A.FocusID as FocusId, A.MeteringCode, A.Baudrate, A.MeteringPort, diff --git a/src/JiShe.CollectBus.Common/Models/BusGlobalPagedResult.cs b/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs similarity index 80% rename from src/JiShe.CollectBus.Common/Models/BusGlobalPagedResult.cs rename to src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs index 6ae8b91..ed42529 100644 --- a/src/JiShe.CollectBus.Common/Models/BusGlobalPagedResult.cs +++ b/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs @@ -6,7 +6,11 @@ using System.Threading.Tasks; namespace JiShe.CollectBus.Common.Models { - public class GlobalPagedResult + /// + /// 缓存全局分页结果 + /// + /// + public class BusCacheGlobalPagedResult { /// /// 数据集合 diff --git a/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs b/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs index 05b654d..06a427b 100644 --- a/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs +++ b/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs @@ -9,7 +9,7 @@ namespace JiShe.CollectBus.Common.Models /// /// 设备缓存基础模型 /// - public class DeviceCacheBasicModel + public abstract class DeviceCacheBasicModel { /// /// 集中器Id @@ -20,5 +20,10 @@ namespace JiShe.CollectBus.Common.Models /// 表Id /// public int MeterId { get; set; } + + /// + /// 唯一标识 + /// + public virtual string UniqueId => $"{FocusId}:{MeterId}"; } } diff --git a/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs b/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs index fa21071..4ac667d 100644 --- a/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs +++ b/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs @@ -1,4 +1,5 @@ -using System; +using JiShe.CollectBus.Common.Models; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -6,23 +7,13 @@ using System.Threading.Tasks; namespace JiShe.CollectBus.Ammeters { - public class AmmeterInfo - { - /// - /// 电表ID - /// - public int ID { get; set; } - + public class AmmeterInfo: DeviceCacheBasicModel + { /// /// 电表名称 /// public string Name { get; set; } - /// - /// 集中器ID - /// - public int FocusID { get; set; } - /// /// 集中器地址 /// diff --git a/src/JiShe.CollectBus.Domain/IotSystems/Watermeter/WatermeterInfo.cs b/src/JiShe.CollectBus.Domain/IotSystems/Watermeter/WatermeterInfo.cs index c735a9a..966192b 100644 --- a/src/JiShe.CollectBus.Domain/IotSystems/Watermeter/WatermeterInfo.cs +++ b/src/JiShe.CollectBus.Domain/IotSystems/Watermeter/WatermeterInfo.cs @@ -1,4 +1,5 @@ using JiShe.CollectBus.Common.Enums; +using JiShe.CollectBus.Common.Models; using System; using System.Collections.Generic; using System.Linq; @@ -10,13 +11,8 @@ namespace JiShe.CollectBus.IotSystems.Watermeter /// /// 水表信息 /// - public class WatermeterInfo - { - /// - /// 水表ID - /// - public int ID { get; set; } - + public class WatermeterInfo: DeviceCacheBasicModel + { /// /// 水表名称 /// @@ -25,12 +21,7 @@ namespace JiShe.CollectBus.IotSystems.Watermeter /// 表密码 /// public string Password { get; set; } - - /// - /// 集中器ID - /// - public int FocusID { get; set; } - + /// /// 集中器地址 /// diff --git a/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs b/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs index c76f9e6..1e07e20 100644 --- a/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs +++ b/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs @@ -1,11 +1,13 @@ using FreeRedis; using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Common.Models; +using JiShe.CollectBus.Common.Extensions; using JiShe.CollectBus.FreeRedisProvider.Options; using Microsoft.Extensions.Options; using System.Diagnostics; using System.Text.Json; using Volo.Abp.DependencyInjection; +using static System.Runtime.InteropServices.JavaScript.JSType; namespace JiShe.CollectBus.FreeRedisProvider { @@ -41,139 +43,38 @@ namespace JiShe.CollectBus.FreeRedisProvider return Instance; } - - //public async Task AddMeterZSetCacheData(string redisCacheKey, string redisCacheIndexKey, decimal score, T data) - //{ - // if (score < 0 || data == null || string.IsNullOrWhiteSpace(redisCacheKey) || string.IsNullOrWhiteSpace(redisCacheIndexKey)) - // { - // throw new Exception($"{nameof(AddMeterZSetCacheData)} 参数异常，-101"); - // } - - // // 生成唯一member标识 - // var member = data.Serialize(); - - // // 计算score范围 - // decimal dataScore = (long)score << 32; - - // //// 事务操作 - // //using (var tran = FreeRedisProvider.Instance.Multi()) - // //{ - // // await tran.ZAddAsync(cacheKey, score,member); - // // await tran.SAddAsync($"cat_index:{categoryId}", member); - // // object[] ret = tran.Exec(); - // //} - - // using (var pipe = Instance.StartPipe()) - // { - // pipe.ZAdd(redisCacheKey, dataScore, member); - // pipe.SAdd(redisCacheIndexKey, member); - // object[] ret = pipe.EndPipe(); - // } - - // await Task.CompletedTask; - //} - - //public async Task> GetMeterZSetPagedData( - //string redisCacheKey, - //string redisCacheIndexKey, - //decimal score, - //int pageSize = 10, - //int pageIndex = 1) - //{ - // if (score < 0 || string.IsNullOrWhiteSpace(redisCacheKey) || string.IsNullOrWhiteSpace(redisCacheIndexKey)) - // { - // throw new Exception($"{nameof(GetMeterZSetPagedData)} 参数异常，-101"); - // } - - // // 计算score范围 - // decimal minScore = (long)score << 32; - // decimal maxScore = ((long)score + 1) << 32; - - // // 分页参数 - // int start = (pageIndex - 1) * pageSize; - - // // 查询主数据 - // var members = await Instance.ZRevRangeByScoreAsync( - // redisCacheKey, - // maxScore, - // minScore, - // offset: start, - // count: pageSize - // ); - - // if (members == null) - // { - // throw new Exception($"{nameof(GetMeterZSetPagedData)} 获取缓存的信息失败，第 {pageIndex + 1} 页数据未返回，-102"); - // } - - // // 查询总数 - // var total = await Instance.ZCountAsync(redisCacheKey, minScore, maxScore); - - // return new BusPagedResult - // { - // Items = members.Select(m => - // BusJsonSerializer.Deserialize(m)!).ToList(), - // TotalCount = total, - // PageIndex = pageIndex, - // PageSize = pageSize - // }; - //} - - ///// - ///// 删除数据示例 - ///// - ///// - ///// 分类 - ///// - ///// - ///// - //public async Task RemoveMeterZSetData( - //string redisCacheKey, - //string redisCacheIndexKey, - //T data) - //{ - - // // 查询需要删除的member - // var members = await Instance.SMembersAsync(redisCacheIndexKey); - // var target = members.FirstOrDefault(m => - // BusJsonSerializer.Deserialize(m) == data);//泛型此处该如何处理？ - - // if (target != null) - // { - // using (var trans = Instance.Multi()) - // { - // trans.ZRem(redisCacheKey, target); - // trans.SRem(redisCacheIndexKey, target); - // trans.Exec(); - // } - // } - - // await Task.CompletedTask; - //} - - - public async Task AddMeterZSetCacheData( + /// + /// 单个添加数据 + /// + /// + /// 主数据存储Hash缓存Key + /// 集中器索引Set缓存Key + /// 集中器排序索引ZSET缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 表计信息 + /// 可选时间戳 + /// + public async Task AddMeterCacheData( string redisCacheKey, - string redisCacheIndexKey, - int categoryId, // 新增分类ID参数 + string redisCacheFocusIndexKey, + string redisCacheScoresIndexKey, + string redisCacheGlobalIndexKey, T data, - DateTimeOffset? timestamp = null) + DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel { // 参数校验增强 if (data == null || string.IsNullOrWhiteSpace(redisCacheKey) - || string.IsNullOrWhiteSpace(redisCacheIndexKey)) + || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey) + || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) + || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) { - throw new ArgumentException("Invalid parameters"); + throw new ArgumentException($"{nameof(AddMeterCacheData)} 参数异常，-101"); } - - // 生成唯一member标识（带数据指纹） - var member = $"{categoryId}:{Guid.NewGuid()}"; - var serializedData = data.Serialize(); - + // 计算组合score（分类ID + 时间戳） var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow; - long scoreValue = ((long)categoryId << 32) | (uint)actualTimestamp.Ticks; + long scoreValue = ((long)data.FocusId << 32) | (uint)actualTimestamp.Ticks; //全局索引写入 long globalScore = actualTimestamp.ToUnixTimeMilliseconds(); @@ -182,64 +83,83 @@ namespace JiShe.CollectBus.FreeRedisProvider using (var trans = Instance.Multi()) { // 主数据存储Hash - trans.HSet(redisCacheKey, member, serializedData); + trans.HSet(redisCacheKey, data.UniqueId, data.Serialize()); + + // 分类索引 + trans.SAdd(redisCacheFocusIndexKey, data.UniqueId); // 排序索引使用ZSET - trans.ZAdd($"{redisCacheKey}_scores", scoreValue, member); - - // 分类索引 - trans.SAdd(redisCacheIndexKey, member); + trans.ZAdd(redisCacheScoresIndexKey, scoreValue, data.UniqueId); //全局索引 - trans.ZAdd("global_data_all", globalScore, member); + trans.ZAdd(redisCacheGlobalIndexKey, globalScore, data.UniqueId); var results = trans.Exec(); if (results == null || results.Length <= 0) - throw new Exception("Transaction failed"); + throw new Exception($"{nameof(AddMeterCacheData)} 事务提交失败，-102"); } await Task.CompletedTask; } + /// + /// 批量添加数据 + /// + /// + /// 主数据存储Hash缓存Key + /// 集中器索引Set缓存Key + /// 集中器排序索引ZSET缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 数据集合 + /// 可选时间戳 + /// public async Task BatchAddMeterData( string redisCacheKey, - string indexKey, - IEnumerable items) where T : DeviceCacheBasicModel + string redisCacheFocusIndexKey, + string redisCacheScoresIndexKey, + string redisCacheGlobalIndexKey, + IEnumerable items, + DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel { const int BATCH_SIZE = 1000; // 每批1000条 var semaphore = new SemaphoreSlim(Environment.ProcessorCount * 2); - //foreach (var batch in items.Batch(BATCH_SIZE)) - //{ - // await semaphore.WaitAsync(); + foreach (var batch in items.Batch(BATCH_SIZE)) + { + await semaphore.WaitAsync(); - // _ = Task.Run(async () => - // { - // using (var pipe = FreeRedisProvider.Instance.StartPipe()) - // { - // foreach (var item in batch) - // { - // var member = $"{item.CategoryId}:{Guid.NewGuid()}"; - // long score = ((long)item.CategoryId << 32) | (uint)item.Timestamp.Ticks; + _ = Task.Run(() => + { + using (var pipe = Instance.StartPipe()) + { + foreach (var item in batch) + { + // 计算组合score（分类ID + 时间戳） + var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow; - // // Hash主数据 - // pipe.HSet(redisCacheKey, member, item.Data.Serialize()); + long scoreValue = ((long)item.FocusId << 32) | (uint)actualTimestamp.Ticks; - // // 分类索引 - // pipe.ZAdd($"{redisCacheKey}_scores", score, member); + //全局索引写入 + long globalScore = actualTimestamp.ToUnixTimeMilliseconds(); - // // 全局索引 - // pipe.ZAdd("global_data_all", item.Timestamp.ToUnixTimeMilliseconds(), member); + // 主数据存储Hash + pipe.HSet(redisCacheKey, item.UniqueId, item.Serialize()); - // // 分类快速索引 - // pipe.SAdd(indexKey, member); - // } - // pipe.EndPipe(); - // } - // semaphore.Release(); - // }); - //} + // 分类索引 + pipe.SAdd(redisCacheFocusIndexKey, item.UniqueId); + + // 排序索引使用ZSET + pipe.ZAdd(redisCacheScoresIndexKey, scoreValue, item.UniqueId); + + //全局索引 + pipe.ZAdd(redisCacheGlobalIndexKey, globalScore, item.UniqueId); + } + pipe.EndPipe(); + } + semaphore.Release(); + }); + } await Task.CompletedTask; } @@ -411,7 +331,7 @@ namespace JiShe.CollectBus.FreeRedisProvider throw new Exception("删除操作失败"); } - public async Task> GetGlobalPagedData( + public async Task> GetGlobalPagedData( string redisCacheKey, int pageSize = 10, long? lastScore = null, @@ -461,7 +381,7 @@ namespace JiShe.CollectBus.FreeRedisProvider ? await GetNextCursor(zsetKey, actualMembers.Last(), descending) : (null, null); - return new GlobalPagedResult + return new BusCacheGlobalPagedResult { Items = dataTasks.Select(t => t.Result).ToList(), HasNext = hasNext, diff --git a/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs b/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs index cc3ff02..36f9a81 100644 --- a/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs +++ b/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs @@ -1,4 +1,5 @@ using FreeRedis; +using JiShe.CollectBus.Common.Models; namespace JiShe.CollectBus.FreeRedisProvider { @@ -9,6 +10,44 @@ namespace JiShe.CollectBus.FreeRedisProvider /// /// RedisClient Instance { get; set; } + + /// + /// 单个添加数据 + /// + /// + /// 主数据存储Hash缓存Key + /// 集中器索引Set缓存Key + /// 集中器排序索引ZSET缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 表计信息 + /// 可选时间戳 + /// + Task AddMeterCacheData( + string redisCacheKey, + string redisCacheFocusIndexKey, + string redisCacheScoresIndexKey, + string redisCacheGlobalIndexKey, + T data, + DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel; + + /// + /// 批量添加数据 + /// + /// + /// 主数据存储Hash缓存Key + /// 集中器索引Set缓存Key + /// 集中器排序索引ZSET缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 数据集合 + /// 可选时间戳 + /// + Task BatchAddMeterData( + string redisCacheKey, + string redisCacheFocusIndexKey, + string redisCacheScoresIndexKey, + string redisCacheGlobalIndexKey, + IEnumerable items, + DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel; } } diff --git a/src/JiShe.CollectBus.Host/appsettings.json b/src/JiShe.CollectBus.Host/appsettings.json index a5b2e15..c922f8d 100644 --- a/src/JiShe.CollectBus.Host/appsettings.json +++ b/src/JiShe.CollectBus.Host/appsettings.json @@ -128,7 +128,7 @@ "OpenDebugMode": true, "UseTableSessionPoolByDefault": false }, - "ServerTagName": "JiSheCollectBus", + "ServerTagName": "JiSheCollectBus2", "KafkaReplicationFactor": 3, "NumPartitions": 30 } \ No newline at end of file From b5f1f1f50bc5f2e3266be1f1f18258f36c2076b2 Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Tue, 15 Apr 2025 17:40:17 +0800 Subject: [PATCH 03/27] =?UTF-8?q?=E5=A4=A7=E6=89=B9=E9=87=8F=E6=8F=92?= =?UTF-8?q?=E5=85=A5=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BasicScheduledMeterReadingService.cs | 68 +++++--- ...nergySystemScheduledMeterReadingService.cs | 8 +- .../Consts/RedisConst.cs | 33 +++- .../JiShe.CollectBus.Common.csproj | 1 + .../Ammeters/AmmeterInfoTemp.cs | 152 ++++++++++++++++++ 5 files changed, 228 insertions(+), 34 deletions(-) create mode 100644 src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfoTemp.cs diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index 5a665ff..33e70f9 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -14,12 +14,14 @@ using JiShe.CollectBus.IotSystems.Watermeter; using JiShe.CollectBus.Kafka.Producer; using JiShe.CollectBus.Protocol.Contracts; using JiShe.CollectBus.Repository.MeterReadingRecord; +using Mapster; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using static FreeSql.Internal.GlobalFilter; namespace JiShe.CollectBus.ScheduledMeterReading { @@ -207,26 +209,30 @@ namespace JiShe.CollectBus.ScheduledMeterReading /// public virtual async Task InitAmmeterCacheData(string gatherCode = "") { -//#if DEBUG -// var timeDensity = "15"; -// //获取缓存中的电表信息 -// var redisKeyList = $"{string.Format(RedisConst.CacheMeterInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}*"; +#if DEBUG + var timeDensity = "15"; + string tempCacheMeterInfoKey = $"CollectBus:{"{0}:{1}"}:MeterInfo:{"{2}"}:{"{3}"}"; + //获取缓存中的电表信息 + var redisKeyList = $"{string.Format(tempCacheMeterInfoKey, SystemType, "JiSheCollectBus", MeterTypeEnum.Ammeter, timeDensity)}*"; -// var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); -// var meterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); -// List focusAddressDataLista = new List(); -// foreach (var item in meterInfos) -// { -// focusAddressDataLista.Add(item.FocusAddress); -// } - -// DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); -// return; -//#else -// var meterInfos = await GetAmmeterInfoList(gatherCode); -//#endif + var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); + var tempMeterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); + //List focusAddressDataLista = new List(); + List meterInfos = new List(); + foreach (var item in tempMeterInfos) + { + var tempData = item.Adapt(); + tempData.FocusId = item.FocusID; + tempData.MeterId = item.Id; + meterInfos.Add(tempData); + //focusAddressDataLista.Add(item.FocusAddress); + } + //DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); +#else var meterInfos = await GetAmmeterInfoList(gatherCode); +#endif + if (meterInfos == null || meterInfos.Count <= 0) { throw new NullReferenceException($"{nameof(InitAmmeterCacheData)} 初始化电表缓存数据时，电表数据为空"); @@ -238,6 +244,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading { throw new NullReferenceException($"{nameof(InitAmmeterCacheData)} 初始化电表缓存数据时，采集项类型数据为空"); } + var timer = Stopwatch.StartNew(); List focusAddressDataList = new List();//用于处理Kafka主题分区数据的分发和处理。 @@ -245,6 +252,12 @@ namespace JiShe.CollectBus.ScheduledMeterReading var meterInfoGroupByTimeDensity = meterInfos.GroupBy(d => d.TimeDensity); foreach (var itemTimeDensity in meterInfoGroupByTimeDensity) { + var redisCacheKey = $"{string.Format(RedisConst.CacheMeterInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; + var redisCacheFocusIndexKey = $"{string.Format(RedisConst.CacheMeterInfoFocusIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; + var redisCacheScoresIndexKey = $"{string.Format(RedisConst.CacheMeterInfoScoresIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; + var redisCacheGlobalIndexKey = $"{string.Format(RedisConst.CacheMeterInfoGlobalIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; + + List ammeterInfos = new List(); //将表计信息根据集中器分组，获得集中器号 var meterInfoGroup = itemTimeDensity.GroupBy(x => x.FocusAddress).ToList(); foreach (var item in meterInfoGroup) @@ -256,17 +269,17 @@ namespace JiShe.CollectBus.ScheduledMeterReading focusAddressDataList.Add(item.Key); - var redisCacheKey = $"{string.Format(RedisConst.CacheMeterInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}{item.Key}"; + // var redisCacheKey = $"{string.Format(RedisConst.CacheMeterInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}{item.Key}"; #if DEBUG //每次缓存时，删除缓存，避免缓存数据有不准确的问题 //await FreeRedisProvider.Instance.DelAsync(redisCacheKey); #else //每次缓存时，删除缓存，避免缓存数据有不准确的问题 - await FreeRedisProvider.Instance.DelAsync(redisCacheKey); + //await FreeRedisProvider.Instance.DelAsync(redisCacheKey); #endif - Dictionary keyValuePairs = new Dictionary(); + //Dictionary keyValuePairs = new Dictionary(); foreach (var ammeter in item) { //处理ItemCode @@ -311,11 +324,18 @@ namespace JiShe.CollectBus.ScheduledMeterReading } } - keyValuePairs.TryAdd($"{ammeter.MeterId}", ammeter); + ammeterInfos.Add(ammeter); + //keyValuePairs.TryAdd($"{ammeter.MeterId}", ammeter); } - await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs); + //await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs); } + await FreeRedisProvider.BatchAddMeterData( + redisCacheKey, + redisCacheFocusIndexKey, + redisCacheScoresIndexKey, + redisCacheGlobalIndexKey, ammeterInfos); + //在缓存表信息数据的时候，新增下一个时间的自动处理任务，1分钟后执行所有的采集频率任务 TasksToBeIssueModel nextTask = new TasksToBeIssueModel() { @@ -338,7 +358,9 @@ namespace JiShe.CollectBus.ScheduledMeterReading DeviceGroupBalanceControl.InitializeCache(focusAddressDataList); } - _logger.LogInformation($"{nameof(InitAmmeterCacheData)} 初始化电表缓存数据完成"); + timer.Stop(); + + _logger.LogInformation($"{nameof(InitAmmeterCacheData)} 初始化电表缓存数据完成，耗时{timer.ElapsedMilliseconds}毫秒"); } /// diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs index 07f5526..d44fe56 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs @@ -114,10 +114,10 @@ namespace JiShe.CollectBus.ScheduledMeterReading WHERE 1=1 and C.Special = 0 "; //TODO 记得移除特殊表过滤 - if (!string.IsNullOrWhiteSpace(gatherCode)) - { - sql = $@"{sql} AND A.GatherCode = '{gatherCode}'"; - } + //if (!string.IsNullOrWhiteSpace(gatherCode)) + //{ + // sql = $@"{sql} AND A.GatherCode = '{gatherCode}'"; + //} return await SqlProvider.Instance.Change(DbEnum.EnergyDB) .Ado .QueryAsync(sql); diff --git a/src/JiShe.CollectBus.Common/Consts/RedisConst.cs b/src/JiShe.CollectBus.Common/Consts/RedisConst.cs index ea4323b..9377056 100644 --- a/src/JiShe.CollectBus.Common/Consts/RedisConst.cs +++ b/src/JiShe.CollectBus.Common/Consts/RedisConst.cs @@ -28,25 +28,44 @@ namespace JiShe.CollectBus.Common.Consts /// public const string FifteenMinuteAcquisitionTimeInterval = "Fifteen"; + public const string MeterInfo = "MeterInfo"; /// /// 缓存表计信息,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 /// - public const string CacheMeterInfoKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:MeterInfo:{"{2}"}:{"{3}"}:"; + public const string CacheMeterInfoKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:{"{3}"}"; + /// + /// 缓存表计信息集中器索引Set缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// + public const string CacheMeterInfoFocusIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:FocusIndex:{"{3}"}"; + + /// + /// 缓存表计信息集中器排序索引ZSET缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// + public const string CacheMeterInfoScoresIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:ScoresIndex:{"{3}"}"; + + /// + /// 缓存表计信息集中器采集频率分组全局索引ZSet缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// + public const string CacheMeterInfoGlobalIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:GlobalIndex:{"{3}"}"; + + + public const string TaskInfo = "TaskInfo"; /// /// 缓存待下发的指令生产任务数据,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 /// - public const string CacheTasksToBeIssuedKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:TaskInfo:{"{2}"}:{"{3}"}"; + public const string CacheTasksToBeIssuedKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TaskInfo}:{"{2}"}{"{3}"}"; + public const string TelemetryPacket = "TelemetryPacket"; /// /// 缓存表计下发指令数据集,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 /// - public const string CacheTelemetryPacketInfoKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:TelemetryPacket:{"{2}"}:{"{3}"}:"; + public const string CacheTelemetryPacketInfoKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:{"{3}"}"; - /// - /// 缓存设备平衡关系映射结果,{0}=>系统类型，{1}=>应用服务部署标记 - /// - public const string CacheDeviceBalanceRelationMapResultKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:RelationMap"; + ///// + ///// 缓存设备平衡关系映射结果,{0}=>系统类型，{1}=>应用服务部署标记 + ///// + //public const string CacheDeviceBalanceRelationMapResultKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:RelationMap"; public const string CacheAmmeterFocusKey = "CacheAmmeterFocusKey"; } diff --git a/src/JiShe.CollectBus.Common/JiShe.CollectBus.Common.csproj b/src/JiShe.CollectBus.Common/JiShe.CollectBus.Common.csproj index b22e8ca..05645ef 100644 --- a/src/JiShe.CollectBus.Common/JiShe.CollectBus.Common.csproj +++ b/src/JiShe.CollectBus.Common/JiShe.CollectBus.Common.csproj @@ -17,6 +17,7 @@ + diff --git a/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfoTemp.cs b/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfoTemp.cs new file mode 100644 index 0000000..fce33fa --- /dev/null +++ b/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfoTemp.cs @@ -0,0 +1,152 @@ +using JiShe.CollectBus.Common.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JiShe.CollectBus.Ammeters +{ + public class AmmeterInfoTemp + { + /// + /// 集中器Id + /// + public int FocusID { get; set; } + + /// + /// 表Id + /// + public int Id { get; set; } + + /// + /// 电表名称 + /// + public string Name { get; set; } + + /// + /// 集中器地址 + /// + public string FocusAddress { get; set; } + + /// + /// 集中器地址 + /// + public string Address { get; set; } + + /// + /// 集中器区域代码 + /// + public string AreaCode { get; set; } + + /// + /// 电表类别 （1单相、2三相三线、3三相四线）, + /// 07协议： 开合闸指令(1A开闸断电,1C单相表合闸,1B多相表合闸) 645 2007 表 + /// 97协议：//true(合闸);false(跳闸) 545 1997 没有单相多相 之分 "true" ? "9966" : "3355" + /// + public int TypeName { get; set; } + + /// + /// 跳合闸状态字段： 0 合闸，1 跳闸 + /// 电表：TripState （0 合闸-通电, 1 断开、跳闸）； + /// + public int TripState { get; set; } + + /// + /// 规约 -电表default(30) 1：97协议，30：07协议 + /// + public int? Protocol { get; set; } + + /// + /// 一个集中器下的[MeteringCode]必须唯一。 PN + /// + public int MeteringCode { get; set; } + + /// + /// 电表通信地址 + /// + public string AmmerterAddress { get; set; } + + /// + /// 波特率 default(2400) + /// + public int Baudrate { get; set; } + + /// + /// MeteringPort 端口就几个可以枚举。 + /// + public int MeteringPort { get; set; } + + /// + /// 电表密码 + /// + public string Password { get; set; } + + /// + /// 采集时间间隔(分钟，如15) + /// + public int TimeDensity { get; set; } + + /// + /// 该电表方案下采集项，JSON格式，如：["0D_80","0D_80"] + /// + public string ItemCodes { get; set; } + + /// + /// State表状态: + /// 0新装（未下发），1运行(档案下发成功时设置状态值1)， 2暂停, 100销表（销表后是否重新启用） + /// 特定：State -1 已删除 + /// + public int State { get; set; } + + /// + /// 是否自动采集(0:主动采集，1：自动采集) + /// + public int AutomaticReport { get; set; } + + /// + /// 该电表方案下采集项编号 + /// + public string DataTypes { get; set; } + + /// + /// 品牌型号 + /// + public string BrandType { get; set; } + + /// + /// 采集器编号 + /// + public string GatherCode { get; set; } + + /// + /// 是否特殊表，1是特殊电表 + /// + public int Special { get; set; } + + /// + /// 费率类型，单、多 (SingleRate :单费率（单相表1），多费率（其他0） ，与TypeName字段无关) + /// SingleRate ? "单" : "复" + /// [SingleRate] --0 复费率 false ， 1 单费率 true （与PayPlanID保持一致） + ///对应 TB_PayPlan.Type: 1复费率，2单费率 + /// + public bool SingleRate { get; set; } + + /// + /// 项目ID + /// + public int ProjectID { get; set; } + + /// + /// 数据库业务ID + /// + public int DatabaseBusiID { get; set; } + + /// + /// 是否异常集中器 0:正常，1异常 + /// + public int AbnormalState { get; set; } + + public DateTime LastTime { get; set; } + } +} From 72d1b9f623ee392decfa8daafea9bd1f0367ee72 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Tue, 15 Apr 2025 18:03:51 +0800 Subject: [PATCH 04/27] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=AE=A2=E9=98=85?= =?UTF-8?q?=E5=9B=9E=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Samples/SampleAppService.cs | 12 +++++++-- .../Pages/Monitor.cshtml | 1 + .../KafkaSubcribesExtensions.cs | 27 ++++++++++++------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs index 3e1c2ea..8f5449a 100644 --- a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs +++ b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs @@ -215,11 +215,19 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS return aa == null; } - [KafkaSubscribe(["test-topic"])] + [KafkaSubscribe(["test-topic1"])] - public async Task KafkaSubscribeAsync(object obj) + public async Task KafkaSubscribeAsync() // TestSubscribe obj { + var obj=string.Empty; _logger.LogWarning($"收到订阅消息： {obj}"); return SubscribeAck.Success(); } } + +public class TestSubscribe +{ + public string Topic { get; set; } + public int Val { get; set; } +} + diff --git a/src/JiShe.CollectBus.Host/Pages/Monitor.cshtml b/src/JiShe.CollectBus.Host/Pages/Monitor.cshtml index aaadf3f..b438e18 100644 --- a/src/JiShe.CollectBus.Host/Pages/Monitor.cshtml +++ b/src/JiShe.CollectBus.Host/Pages/Monitor.cshtml @@ -16,6 +16,7 @@ 后端服务 + diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index 6cf2b60..d3ce904 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -1,4 +1,7 @@ using Confluent.Kafka; +using DeviceDetectorNET; +using JiShe.CollectBus.Common.Enums; +using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Kafka.Attributes; using JiShe.CollectBus.Kafka.Consumer; using Microsoft.AspNetCore.Builder; @@ -8,6 +11,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Primitives; using Newtonsoft.Json; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -96,6 +100,7 @@ namespace JiShe.CollectBus.Kafka }); } + /// /// 处理消息 /// @@ -106,15 +111,18 @@ namespace JiShe.CollectBus.Kafka private static async Task ProcessMessageAsync(string message, MethodInfo method, object subscribe) { var parameters = method.GetParameters(); - if (parameters.Length != 1) - return true; - - var paramType = parameters[0].ParameterType; - var messageObj = paramType == typeof(string)? message: JsonConvert.DeserializeObject(message, paramType); - - if (method.ReturnType == typeof(Task)) + bool isGenericTask = method.ReturnType.IsGenericType + && method.ReturnType.GetGenericTypeDefinition() == typeof(Task<>); + bool existParameters = parameters.Length > 0; + dynamic? messageObj= null; + if (existParameters) { - object? result = await (Task)method.Invoke(subscribe, new[] { messageObj })!; + var paramType = parameters[0].ParameterType; + messageObj = paramType == typeof(string) ? message : message.Deserialize(paramType); + } + if (isGenericTask) + { + object? result = await (Task)method.Invoke(subscribe, existParameters? new[] { messageObj }:null)!; if (result is ISubscribeAck ackResult) { return ackResult.Ack; @@ -122,13 +130,12 @@ namespace JiShe.CollectBus.Kafka } else { - object? result = method.Invoke(subscribe, new[] { messageObj }); + object? result = method.Invoke(subscribe, existParameters ? new[] { messageObj } : null); if (result is ISubscribeAck ackResult) { return ackResult.Ack; } } - return false; } From 11d3fcf162ce187880a8470ad3646d37f50cc548 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Tue, 15 Apr 2025 18:58:38 +0800 Subject: [PATCH 05/27] =?UTF-8?q?=E7=A7=BB=E9=99=A4CAP=E7=BB=84=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...he.CollectBus.Application.Contracts.csproj | 1 + .../Subscribers/ISubscriberAppService.cs | 11 +++--- .../IWorkerSubscriberAppService.cs | 9 +++-- .../JiShe.CollectBus.Application.csproj | 1 - .../Plugins/TcpMonitor.cs | 18 +++++---- .../Samples/SampleAppService.cs | 11 +----- .../BasicScheduledMeterReadingService.cs | 13 +++---- ...nergySystemScheduledMeterReadingService.cs | 2 +- .../Subscribers/SubscriberAppService.cs | 39 ++++++++++++------- .../Subscribers/WorkerSubscriberAppService.cs | 24 +++++++----- .../CollectBusHostModule.cs | 2 +- .../Pages/Monitor.cshtml | 1 - .../Consumer/ConsumerService.cs | 6 +-- .../Producer/IProducerService.cs | 2 +- .../Producer/ProducerService.cs | 10 ++--- .../Abstracts/BaseProtocolPlugin.cs | 10 +++-- ...JiShe.CollectBus.Protocol.Contracts.csproj | 3 +- 17 files changed, 89 insertions(+), 74 deletions(-) diff --git a/src/JiShe.CollectBus.Application.Contracts/JiShe.CollectBus.Application.Contracts.csproj b/src/JiShe.CollectBus.Application.Contracts/JiShe.CollectBus.Application.Contracts.csproj index de972d1..6e4fed5 100644 --- a/src/JiShe.CollectBus.Application.Contracts/JiShe.CollectBus.Application.Contracts.csproj +++ b/src/JiShe.CollectBus.Application.Contracts/JiShe.CollectBus.Application.Contracts.csproj @@ -26,6 +26,7 @@ + diff --git a/src/JiShe.CollectBus.Application.Contracts/Subscribers/ISubscriberAppService.cs b/src/JiShe.CollectBus.Application.Contracts/Subscribers/ISubscriberAppService.cs index f9bc706..658ff29 100644 --- a/src/JiShe.CollectBus.Application.Contracts/Subscribers/ISubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application.Contracts/Subscribers/ISubscriberAppService.cs @@ -1,16 +1,17 @@ using System.Threading.Tasks; using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.IotSystems.MessageReceiveds; +using JiShe.CollectBus.Kafka; using Volo.Abp.Application.Services; namespace JiShe.CollectBus.Subscribers { public interface ISubscriberAppService : IApplicationService { - Task LoginIssuedEvent(IssuedEventMessage issuedEventMessage); - Task HeartbeatIssuedEvent(IssuedEventMessage issuedEventMessage); - Task ReceivedEvent(MessageReceived receivedMessage); - Task ReceivedHeartbeatEvent(MessageReceivedHeartbeat receivedHeartbeatMessage); - Task ReceivedLoginEvent(MessageReceivedLogin receivedLoginMessage); + Task LoginIssuedEvent(IssuedEventMessage issuedEventMessage); + Task HeartbeatIssuedEvent(IssuedEventMessage issuedEventMessage); + Task ReceivedEvent(MessageReceived receivedMessage); + Task ReceivedHeartbeatEvent(MessageReceivedHeartbeat receivedHeartbeatMessage); + Task ReceivedLoginEvent(MessageReceivedLogin receivedLoginMessage); } } diff --git a/src/JiShe.CollectBus.Application.Contracts/Subscribers/IWorkerSubscriberAppService.cs b/src/JiShe.CollectBus.Application.Contracts/Subscribers/IWorkerSubscriberAppService.cs index abba774..9a37167 100644 --- a/src/JiShe.CollectBus.Application.Contracts/Subscribers/IWorkerSubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application.Contracts/Subscribers/IWorkerSubscriberAppService.cs @@ -1,6 +1,7 @@ using JiShe.CollectBus.IotSystems.MessageIssueds; using JiShe.CollectBus.IotSystems.MessageReceiveds; using JiShe.CollectBus.IotSystems.MeterReadingRecords; +using JiShe.CollectBus.Kafka; using System.Collections.Generic; using System.Threading.Tasks; using Volo.Abp.Application.Services; @@ -19,19 +20,19 @@ namespace JiShe.CollectBus.Subscribers /// 1分钟采集电表数据下行消息消费订阅 /// /// - Task AmmeterScheduledMeterOneMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage issuedEventMessage); + Task AmmeterScheduledMeterOneMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage issuedEventMessage); /// /// 5分钟采集电表数据下行消息消费订阅 /// /// - Task AmmeterScheduledMeterFiveMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage issuedEventMessage); + Task AmmeterScheduledMeterFiveMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage issuedEventMessage); /// /// 15分钟采集电表数据下行消息消费订阅 /// /// - Task AmmeterScheduledMeterFifteenMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage issuedEventMessage); + Task AmmeterScheduledMeterFifteenMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage issuedEventMessage); #endregion #region 水表消息采集 @@ -39,7 +40,7 @@ namespace JiShe.CollectBus.Subscribers /// 1分钟采集水表数据下行消息消费订阅 /// /// - Task WatermeterSubscriberWorkerAutoReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage issuedEventMessage); + Task WatermeterSubscriberWorkerAutoReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage issuedEventMessage); #endregion } diff --git a/src/JiShe.CollectBus.Application/JiShe.CollectBus.Application.csproj b/src/JiShe.CollectBus.Application/JiShe.CollectBus.Application.csproj index a2cc613..24db5a5 100644 --- a/src/JiShe.CollectBus.Application/JiShe.CollectBus.Application.csproj +++ b/src/JiShe.CollectBus.Application/JiShe.CollectBus.Application.csproj @@ -23,7 +23,6 @@ - diff --git a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs index ff4de0c..29427fc 100644 --- a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs +++ b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs @@ -7,10 +7,12 @@ using DotNetCore.CAP; using JiShe.CollectBus.Ammeters; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; +using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Enums; using JiShe.CollectBus.Interceptors; using JiShe.CollectBus.IotSystems.Devices; using JiShe.CollectBus.IotSystems.MessageReceiveds; +using JiShe.CollectBus.Kafka.Producer; using JiShe.CollectBus.Protocol.Contracts; using MassTransit; using Microsoft.Extensions.Logging; @@ -26,7 +28,7 @@ namespace JiShe.CollectBus.Plugins { public partial class TcpMonitor : PluginBase, ITransientDependency, ITcpReceivedPlugin, ITcpConnectingPlugin, ITcpConnectedPlugin, ITcpClosedPlugin { - private readonly ICapPublisher _producerBus; + private readonly IProducerService _producerService; private readonly ILogger _logger; private readonly IRepository _deviceRepository; private readonly IDistributedCache _ammeterInfoCache; @@ -34,16 +36,16 @@ namespace JiShe.CollectBus.Plugins /// /// /// - /// + /// /// /// /// - public TcpMonitor(ICapPublisher producerBus, + public TcpMonitor(IProducerService producerService, ILogger logger, IRepository deviceRepository, IDistributedCache ammeterInfoCache) { - _producerBus = producerBus; + _producerService = producerService; _logger = logger; _deviceRepository = deviceRepository; _ammeterInfoCache = ammeterInfoCache; @@ -170,7 +172,7 @@ namespace JiShe.CollectBus.Plugins DeviceNo = deviceNo, MessageId = NewId.NextGuid().ToString() }; - await _producerBus.PublishAsync(ProtocolConst.SubscriberLoginReceivedEventName, messageReceivedLoginEvent); + await _producerService.ProduceAsync(ProtocolConst.SubscriberLoginReceivedEventName, messageReceivedLoginEvent.Serialize()); //await _producerBus.Publish( messageReceivedLoginEvent); } @@ -217,7 +219,7 @@ namespace JiShe.CollectBus.Plugins DeviceNo = deviceNo, MessageId = NewId.NextGuid().ToString() }; - await _producerBus.PublishAsync(ProtocolConst.SubscriberHeartbeatReceivedEventName, messageReceivedHeartbeatEvent); + await _producerService.ProduceAsync(ProtocolConst.SubscriberHeartbeatReceivedEventName, messageReceivedHeartbeatEvent.Serialize()); //await _producerBus.Publish(messageReceivedHeartbeatEvent); } @@ -245,7 +247,7 @@ namespace JiShe.CollectBus.Plugins //string topicName = string.Format(ProtocolConst.AFNTopicNameFormat, aFn); //todo 如何确定时标？目前集中器的采集频率，都是固定，数据上报的时候，根据当前时间，往后推测出应当采集的时间点作为时标。但是如果由于网络问题，数据一直没上报的情况改怎么计算？ - await _producerBus.PublishAsync(ProtocolConst.SubscriberReceivedEventName, new MessageReceived + await _producerService.ProduceAsync(ProtocolConst.SubscriberReceivedEventName, new MessageReceived { ClientId = client.Id, ClientIp = client.IP, @@ -253,7 +255,7 @@ namespace JiShe.CollectBus.Plugins MessageHexString = messageHexString, DeviceNo = deviceNo, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); } } } diff --git a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs index 8f5449a..78b472d 100644 --- a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs +++ b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs @@ -215,19 +215,12 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS return aa == null; } - [KafkaSubscribe(["test-topic1"])] + [KafkaSubscribe(["test-topic"])] - public async Task KafkaSubscribeAsync() // TestSubscribe obj + public async Task KafkaSubscribeAsync(object obj) { - var obj=string.Empty; _logger.LogWarning($"收到订阅消息： {obj}"); return SubscribeAck.Success(); } } -public class TestSubscribe -{ - public string Topic { get; set; } - public int Val { get; set; } -} - diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index 287158f..f2a08b1 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -29,7 +29,6 @@ namespace JiShe.CollectBus.ScheduledMeterReading public abstract class BasicScheduledMeterReadingService : CollectBusAppService, IScheduledMeterReadingService { private readonly ILogger _logger; - private readonly ICapPublisher _producerBus; private readonly IIoTDBProvider _dbProvider; private readonly IMeterReadingRecordRepository _meterReadingRecordRepository; private readonly IProducerService _producerService; @@ -37,12 +36,10 @@ namespace JiShe.CollectBus.ScheduledMeterReading public BasicScheduledMeterReadingService( ILogger logger, - ICapPublisher producerBus, IMeterReadingRecordRepository meterReadingRecordRepository, IProducerService producerService, IIoTDBProvider dbProvider) { - _producerBus = producerBus; _logger = logger; _dbProvider = dbProvider; _meterReadingRecordRepository = meterReadingRecordRepository; @@ -381,7 +378,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading FocusAddress = ammerterItem.Value.FocusAddress, TimeDensity = timeDensity.ToString(), }; - _ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName, tempMsg); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName, tempMsg.Serialize()); //_= _producerBus.Publish(tempMsg); @@ -445,7 +442,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading FocusAddress = ammerterItem.Value.FocusAddress, TimeDensity = timeDensity.ToString(), }; - _ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName, tempMsg); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName, tempMsg.Serialize()); //_ = _producerBus.Publish(tempMsg); @@ -510,7 +507,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading TimeDensity = timeDensity.ToString(), }; - _ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg.Serialize()); //_ = _producerBus.Publish(tempMsg); @@ -805,7 +802,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading } int partition = DeviceGroupBalanceControl.GetDeviceGroupId(taskRecord.FocusAddress); - await _producerService.ProduceAsync(topicName, partition, taskRecord); + await _producerService.ProduceAsync(topicName, partition, taskRecord.Serialize()); } private async Task AmmerterCreatePublishTask(int timeDensity, MeterTypeEnum meterType) @@ -846,7 +843,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading TimeDensity = timeDensity.ToString(), }; - _ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg.Serialize()); //_ = _producerBus.Publish(tempMsg); diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs index 91f7d4a..8593dfd 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs @@ -36,7 +36,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading { string serverTagName = string.Empty; public EnergySystemScheduledMeterReadingService(ILogger logger, - ICapPublisher producerBus, IIoTDBProvider dbProvider, IMeterReadingRecordRepository meterReadingRecordRepository,IConfiguration configuration, IProducerService producerService) : base(logger, producerBus, meterReadingRecordRepository, producerService,dbProvider) + IIoTDBProvider dbProvider, IMeterReadingRecordRepository meterReadingRecordRepository,IConfiguration configuration, IProducerService producerService) : base(logger, meterReadingRecordRepository, producerService,dbProvider) { serverTagName = configuration.GetValue(CommonConst.ServerTagName)!; } diff --git a/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs b/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs index d5e9caf..9bf297f 100644 --- a/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs @@ -6,6 +6,8 @@ using JiShe.CollectBus.IoTDBProvider; using JiShe.CollectBus.IotSystems.Devices; using JiShe.CollectBus.IotSystems.MessageReceiveds; using JiShe.CollectBus.IotSystems.MeterReadingRecords; +using JiShe.CollectBus.Kafka; +using JiShe.CollectBus.Kafka.Attributes; using JiShe.CollectBus.Protocol.Contracts; using JiShe.CollectBus.Protocol.Contracts.Interfaces; using JiShe.CollectBus.Protocol.Contracts.Models; @@ -20,7 +22,7 @@ using Volo.Abp.Domain.Repositories; namespace JiShe.CollectBus.Subscribers { - public class SubscriberAppService : CollectBusAppService, ISubscriberAppService, ICapSubscribe + public class SubscriberAppService : CollectBusAppService, ISubscriberAppService, IKafkaSubscribe { private readonly ILogger _logger; private readonly ITcpService _tcpService; @@ -63,9 +65,10 @@ namespace JiShe.CollectBus.Subscribers _dbProvider = dbProvider; } - [CapSubscribe(ProtocolConst.SubscriberLoginIssuedEventName)] - public async Task LoginIssuedEvent(IssuedEventMessage issuedEventMessage) + [KafkaSubscribe(ProtocolConst.SubscriberLoginIssuedEventName)] + public async Task LoginIssuedEvent(IssuedEventMessage issuedEventMessage) { + bool isAck = false; switch (issuedEventMessage.Type) { case IssuedEventType.Heartbeat: @@ -76,6 +79,7 @@ namespace JiShe.CollectBus.Subscribers loginEntity.AckTime = Clock.Now; loginEntity.IsAck = true; await _messageReceivedLoginEventRepository.UpdateAsync(loginEntity); + isAck = true; break; case IssuedEventType.Data: break; @@ -90,11 +94,13 @@ namespace JiShe.CollectBus.Subscribers //} await _tcpService.SendAsync(issuedEventMessage.ClientId, issuedEventMessage.Message); + return isAck? SubscribeAck.Success(): SubscribeAck.Fail(); } - [CapSubscribe(ProtocolConst.SubscriberHeartbeatIssuedEventName)] - public async Task HeartbeatIssuedEvent(IssuedEventMessage issuedEventMessage) + [KafkaSubscribe(ProtocolConst.SubscriberHeartbeatIssuedEventName)] + public async Task HeartbeatIssuedEvent(IssuedEventMessage issuedEventMessage) { + bool isAck = false; switch (issuedEventMessage.Type) { case IssuedEventType.Heartbeat: @@ -103,6 +109,7 @@ namespace JiShe.CollectBus.Subscribers heartbeatEntity.AckTime = Clock.Now; heartbeatEntity.IsAck = true; await _messageReceivedHeartbeatEventRepository.UpdateAsync(heartbeatEntity); + isAck = true; break; case IssuedEventType.Data: break; @@ -117,10 +124,11 @@ namespace JiShe.CollectBus.Subscribers //} await _tcpService.SendAsync(issuedEventMessage.ClientId, issuedEventMessage.Message); + return isAck ? SubscribeAck.Success() : SubscribeAck.Fail(); } - [CapSubscribe(ProtocolConst.SubscriberReceivedEventName)] - public async Task ReceivedEvent(MessageReceived receivedMessage) + [KafkaSubscribe(ProtocolConst.SubscriberReceivedEventName)] + public async Task ReceivedEvent(MessageReceived receivedMessage) { var currentTime = Clock.Now; @@ -137,13 +145,13 @@ namespace JiShe.CollectBus.Subscribers if(fN == null) { Logger.LogError($"数据解析失败：{receivedMessage.Serialize()}"); - return; + return SubscribeAck.Success(); } var tb3761FN = fN.FnList.FirstOrDefault(); if (tb3761FN == null) { Logger.LogError($"数据解析失败：{receivedMessage.Serialize()}"); - return; + return SubscribeAck.Success(); } //报文入库 @@ -169,11 +177,14 @@ namespace JiShe.CollectBus.Subscribers //todo 查找是否有下发任务 //await _messageReceivedEventRepository.InsertAsync(receivedMessage); + + } + return SubscribeAck.Success(); } - [CapSubscribe(ProtocolConst.SubscriberHeartbeatReceivedEventName)] - public async Task ReceivedHeartbeatEvent(MessageReceivedHeartbeat receivedHeartbeatMessage) + [KafkaSubscribe(ProtocolConst.SubscriberHeartbeatReceivedEventName)] + public async Task ReceivedHeartbeatEvent(MessageReceivedHeartbeat receivedHeartbeatMessage) { var protocolPlugin = _serviceProvider.GetKeyedService("StandardProtocolPlugin"); if (protocolPlugin == null) @@ -185,10 +196,11 @@ namespace JiShe.CollectBus.Subscribers await protocolPlugin.HeartbeatAsync(receivedHeartbeatMessage); await _messageReceivedHeartbeatEventRepository.InsertAsync(receivedHeartbeatMessage); } + return SubscribeAck.Success(); } - [CapSubscribe(ProtocolConst.SubscriberLoginReceivedEventName)] - public async Task ReceivedLoginEvent(MessageReceivedLogin receivedLoginMessage) + [KafkaSubscribe(ProtocolConst.SubscriberLoginReceivedEventName)] + public async Task ReceivedLoginEvent(MessageReceivedLogin receivedLoginMessage) { var protocolPlugin = _serviceProvider.GetKeyedService("StandardProtocolPlugin"); if (protocolPlugin == null) @@ -200,6 +212,7 @@ namespace JiShe.CollectBus.Subscribers await protocolPlugin.LoginAsync(receivedLoginMessage); await _messageReceivedLoginEventRepository.InsertAsync(receivedLoginMessage); } + return SubscribeAck.Success(); } } } diff --git a/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs b/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs index 3caba20..f3c485c 100644 --- a/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs @@ -8,6 +8,8 @@ using JiShe.CollectBus.IotSystems.Devices; using JiShe.CollectBus.IotSystems.MessageIssueds; using JiShe.CollectBus.IotSystems.MessageReceiveds; using JiShe.CollectBus.IotSystems.MeterReadingRecords; +using JiShe.CollectBus.Kafka; +using JiShe.CollectBus.Kafka.Attributes; using JiShe.CollectBus.Protocol.Contracts; using JiShe.CollectBus.Protocol.Contracts.Interfaces; using JiShe.CollectBus.Repository.MeterReadingRecord; @@ -24,7 +26,7 @@ namespace JiShe.CollectBus.Subscribers /// 定时抄读任务消息消费订阅 /// [Route($"/worker/app/subscriber")] - public class WorkerSubscriberAppService : CollectBusAppService, IWorkerSubscriberAppService,ICapSubscribe + public class WorkerSubscriberAppService : CollectBusAppService, IWorkerSubscriberAppService, IKafkaSubscribe { private readonly ILogger _logger; private readonly ITcpService _tcpService; @@ -63,8 +65,8 @@ namespace JiShe.CollectBus.Subscribers /// [HttpPost] [Route("ammeter/oneminute/issued-event")] - [CapSubscribe(ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName)] - public async Task AmmeterScheduledMeterOneMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) + [KafkaSubscribe(ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName)] + public async Task AmmeterScheduledMeterOneMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) { _logger.LogInformation("1分钟采集电表数据下行消息消费队列开始处理"); var protocolPlugin = _serviceProvider.GetKeyedService("StandardProtocolPlugin"); @@ -81,6 +83,7 @@ namespace JiShe.CollectBus.Subscribers } } + return SubscribeAck.Success(); } /// @@ -90,8 +93,8 @@ namespace JiShe.CollectBus.Subscribers /// [HttpPost] [Route("ammeter/fiveminute/issued-event")] - [CapSubscribe(ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName)] - public async Task AmmeterScheduledMeterFiveMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) + [KafkaSubscribe(ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName)] + public async Task AmmeterScheduledMeterFiveMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) { _logger.LogInformation("5分钟采集电表数据下行消息消费队列开始处理"); var protocolPlugin = _serviceProvider.GetKeyedService("StandardProtocolPlugin"); @@ -108,6 +111,7 @@ namespace JiShe.CollectBus.Subscribers } } + return SubscribeAck.Success(); } /// @@ -117,8 +121,8 @@ namespace JiShe.CollectBus.Subscribers /// [HttpPost] [Route("ammeter/fifteenminute/issued-event")] - [CapSubscribe(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName)] - public async Task AmmeterScheduledMeterFifteenMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) + [KafkaSubscribe(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName)] + public async Task AmmeterScheduledMeterFifteenMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) { _logger.LogInformation("15分钟采集电表数据下行消息消费队列开始处理"); try @@ -137,6 +141,7 @@ namespace JiShe.CollectBus.Subscribers } } + return SubscribeAck.Success(); } catch (Exception ex) { @@ -155,8 +160,8 @@ namespace JiShe.CollectBus.Subscribers /// [HttpPost] [Route("watermeter/fifteenminute/issued-event")] - [CapSubscribe(ProtocolConst.WatermeterSubscriberWorkerAutoReadingIssuedEventName)] - public async Task WatermeterSubscriberWorkerAutoReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) + [KafkaSubscribe(ProtocolConst.WatermeterSubscriberWorkerAutoReadingIssuedEventName)] + public async Task WatermeterSubscriberWorkerAutoReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) { _logger.LogInformation("15分钟采集水表数据下行消息消费队列开始处理"); var protocolPlugin = _serviceProvider.GetKeyedService("StandardProtocolPlugin"); @@ -172,6 +177,7 @@ namespace JiShe.CollectBus.Subscribers await _tcpService.SendAsync(device.ClientId, Convert.FromHexString(receivedMessage.MessageHexString)); } } + return SubscribeAck.Success(); } #endregion } diff --git a/src/JiShe.CollectBus.Host/CollectBusHostModule.cs b/src/JiShe.CollectBus.Host/CollectBusHostModule.cs index aabc2ba..377453e 100644 --- a/src/JiShe.CollectBus.Host/CollectBusHostModule.cs +++ b/src/JiShe.CollectBus.Host/CollectBusHostModule.cs @@ -43,7 +43,7 @@ namespace JiShe.CollectBus.Host ConfigureNetwork(context, configuration); ConfigureJwtAuthentication(context, configuration); ConfigureHangfire(context); - ConfigureCap(context, configuration); + //ConfigureCap(context, configuration); //ConfigureMassTransit(context, configuration); //ConfigureKafkaTopic(context, configuration); ConfigureAuditLog(context); diff --git a/src/JiShe.CollectBus.Host/Pages/Monitor.cshtml b/src/JiShe.CollectBus.Host/Pages/Monitor.cshtml index b438e18..aaadf3f 100644 --- a/src/JiShe.CollectBus.Host/Pages/Monitor.cshtml +++ b/src/JiShe.CollectBus.Host/Pages/Monitor.cshtml @@ -16,7 +16,6 @@ 后端服务 - diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs index 08b8cb1..f59385f 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs @@ -146,14 +146,14 @@ namespace JiShe.CollectBus.Kafka.Consumer /// public async Task SubscribeAsync(string[] topics, Func> messageHandler, string? groupId) where TValue : class { - var consumerKey = typeof((Null, TValue)); + var consumerKey = typeof((Ignore, TValue)); var cts = new CancellationTokenSource(); var consumer = _consumerStore.GetOrAdd(consumerKey, _=> ( - CreateConsumer(groupId), + CreateConsumer(groupId), cts - )).Consumer as IConsumer; + )).Consumer as IConsumer; consumer!.Subscribe(topics); diff --git a/src/JiShe.CollectBus.KafkaProducer/Producer/IProducerService.cs b/src/JiShe.CollectBus.KafkaProducer/Producer/IProducerService.cs index becea90..b00f5cf 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Producer/IProducerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Producer/IProducerService.cs @@ -15,6 +15,6 @@ namespace JiShe.CollectBus.Kafka.Producer Task ProduceAsync(string topic, TKey key, TValue value, int? partition, Action>? deliveryHandler = null) where TKey : notnull where TValue : class; - Task ProduceAsync(string topic, TValue value, int? partition = null, Action>? deliveryHandler = null) where TValue : class; + Task ProduceAsync(string topic, TValue value, int? partition = null, Action>? deliveryHandler = null) where TValue : class; } } diff --git a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs index ca45f8c..c322294 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs @@ -114,8 +114,8 @@ namespace JiShe.CollectBus.Kafka.Producer /// public async Task ProduceAsync(string topic, TValue value) where TValue : class { - var producer = GetProducer(); - await producer.ProduceAsync(topic, new Message { Value = value }); + var producer = GetProducer(); + await producer.ProduceAsync(topic, new Message { Value = value }); } /// @@ -160,13 +160,13 @@ namespace JiShe.CollectBus.Kafka.Producer /// /// /// - public async Task ProduceAsync(string topic, TValue value, int? partition=null, Action>? deliveryHandler = null) where TValue : class + public async Task ProduceAsync(string topic, TValue value, int? partition=null, Action>? deliveryHandler = null) where TValue : class { - var message = new Message + var message = new Message { Value = value }; - var producer = GetProducer(); + var producer = GetProducer(); if (partition.HasValue) { var topicPartition = new TopicPartition(topic, partition.Value); diff --git a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs index 762a2ca..23d0917 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs +++ b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs @@ -12,12 +12,14 @@ using Microsoft.Extensions.DependencyInjection; using JiShe.CollectBus.IotSystems.MessageReceiveds; using JiShe.CollectBus.IotSystems.Protocols; using MassTransit; +using JiShe.CollectBus.Kafka.Producer; +using JiShe.CollectBus.Common.Helpers; namespace JiShe.CollectBus.Protocol.Contracts.Abstracts { public abstract class BaseProtocolPlugin : IProtocolPlugin { - private readonly ICapPublisher _producerBus; + private readonly IProducerService _producerService; private readonly ILogger _logger; private readonly IRepository _protocolInfoRepository; @@ -37,7 +39,7 @@ namespace JiShe.CollectBus.Protocol.Contracts.Abstracts _logger = serviceProvider.GetRequiredService>(); _protocolInfoRepository = serviceProvider.GetRequiredService>(); - _producerBus = serviceProvider.GetRequiredService(); + _producerService = serviceProvider.GetRequiredService(); } public abstract ProtocolInfo Info { get; } @@ -87,7 +89,7 @@ namespace JiShe.CollectBus.Protocol.Contracts.Abstracts }; var bytes = Build3761SendData.BuildSendCommandBytes(reqParam); - await _producerBus.PublishAsync(ProtocolConst.SubscriberLoginIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }); + await _producerService.ProduceAsync(ProtocolConst.SubscriberLoginIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }.Serialize()); //await _producerBus.Publish(new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }); } @@ -126,7 +128,7 @@ namespace JiShe.CollectBus.Protocol.Contracts.Abstracts Fn = 1 }; var bytes = Build3761SendData.BuildSendCommandBytes(reqParam); - await _producerBus.PublishAsync(ProtocolConst.SubscriberHeartbeatIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }); + await _producerService.ProduceAsync(ProtocolConst.SubscriberHeartbeatIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }.Serialize()); //await _producerBus.Publish(new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }); } diff --git a/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj b/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj index aa5bd73..248ee30 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj +++ b/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj @@ -7,7 +7,7 @@ - + @@ -17,6 +17,7 @@ + From e1106b32417285a363809b1208603117739dc74f Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Tue, 15 Apr 2025 19:09:16 +0800 Subject: [PATCH 06/27] =?UTF-8?q?=E7=A7=BB=E9=99=A4CAP=E7=BB=84=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EnergySystem/EnergySystemAppService.cs | 77 ++++++++++--------- .../Plugins/TcpMonitor.cs | 1 - .../BasicScheduledMeterReadingService.cs | 3 +- ...nergySystemScheduledMeterReadingService.cs | 1 - .../Subscribers/SubscriberAppService.cs | 3 +- .../Subscribers/WorkerSubscriberAppService.cs | 1 - .../Abstracts/BaseProtocolPlugin.cs | 3 +- 7 files changed, 42 insertions(+), 47 deletions(-) diff --git a/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs b/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs index 0542ede..a210d28 100644 --- a/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs +++ b/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs @@ -5,15 +5,16 @@ using System.Net; using System.Text; using System.Threading.Tasks; using DeviceDetectorNET.Class.Device; -using DotNetCore.CAP; using JiShe.CollectBus.Common.BuildSendDatas; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; +using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.EnergySystem.Dto; using JiShe.CollectBus.FreeSql; using JiShe.CollectBus.IotSystems.PrepayModel; using JiShe.CollectBus.IotSystems.Records; +using JiShe.CollectBus.Kafka.Producer; using JiShe.CollectBus.Protocol.Contracts; using MassTransit; using Microsoft.AspNetCore.Mvc; @@ -28,15 +29,15 @@ namespace JiShe.CollectBus.EnergySystem private readonly IRepository _focusRecordRepository; private readonly IRepository _csqRecordRepository; private readonly IRepository _conrOnlineRecordRepository; - private readonly ICapPublisher _capBus; + private readonly IProducerService _producerService; public EnergySystemAppService(IRepository focusRecordRepository, IRepository csqRecordRepository, - IRepository conrOnlineRecordRepository, ICapPublisher capBus) + IRepository conrOnlineRecordRepository, IProducerService producerService) { _focusRecordRepository = focusRecordRepository; _csqRecordRepository = csqRecordRepository; _conrOnlineRecordRepository = conrOnlineRecordRepository; - _capBus = capBus; + _producerService = producerService; } /// @@ -71,14 +72,14 @@ namespace JiShe.CollectBus.EnergySystem return result; - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); result.Status = true; result.Msg = "操作成功"; result.Data.ValidData = true; @@ -108,14 +109,14 @@ namespace JiShe.CollectBus.EnergySystem foreach (var bytes in bytesList) { - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); } return result; @@ -149,14 +150,14 @@ namespace JiShe.CollectBus.EnergySystem }).ToList(); var bytes = Build3761SendData.BuildAmmeterParameterSetSendCmd(address, meterParameters); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); result.Status = true; result.Msg = "操作成功"; result.Data.ValidData = true; @@ -178,14 +179,14 @@ namespace JiShe.CollectBus.EnergySystem { var dataUnit = Build645SendData.BuildReadMeterAddressSendDataUnit(detail.MeterAddress); var bytes =Build3761SendData.BuildTransparentForwardingSendCmd(address, detail.Port, detail.BaudRate.ToString(), dataUnit, StopBit.Stop1, Parity.None); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); } result.Status = true; @@ -261,14 +262,14 @@ namespace JiShe.CollectBus.EnergySystem if (bytes != null) { - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); if (isManual) { @@ -320,14 +321,14 @@ namespace JiShe.CollectBus.EnergySystem var bytes = Build3761SendData.BuildCommunicationParametersSetSendCmd(address, masterIP, materPort, backupIP, backupPort, input.Data.APN); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); result.Status = true; result.Msg = "操作成功"; @@ -347,14 +348,14 @@ namespace JiShe.CollectBus.EnergySystem var address = $"{input.AreaCode}{input.Address}"; var bytes = Build3761SendData.BuildTerminalCalendarClockSendCmd(address); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); result.Status = true; result.Msg = "操作成功"; @@ -375,14 +376,14 @@ namespace JiShe.CollectBus.EnergySystem bool isManual = !input.AreaCode.Equals("5110");//低功耗集中器不是长连接，在连接的那一刻再发送 var bytes = Build3761SendData.BuildConrCheckTimeSendCmd(address,DateTime.Now, isManual); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); result.Status = true; result.Msg = "操作成功"; @@ -402,14 +403,14 @@ namespace JiShe.CollectBus.EnergySystem var address = $"{input.AreaCode}{input.Address}"; var bytes = Build3761SendData.BuildConrRebootSendCmd(address); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); result.Status = true; result.Msg = "操作成功"; @@ -430,14 +431,14 @@ namespace JiShe.CollectBus.EnergySystem var address = $"{input.AreaCode}{input.Address}"; var pnList = input.Data.Split(',').Select(it => int.Parse(it)).ToList(); var bytes = Build3761SendData.BuildAmmeterParameterReadingSendCmd(address, pnList); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); result.Status = true; result.Msg = "操作成功"; return result; @@ -479,14 +480,14 @@ namespace JiShe.CollectBus.EnergySystem foreach (var bytes in bytesList) { - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); } result.Status = true; @@ -548,14 +549,14 @@ namespace JiShe.CollectBus.EnergySystem foreach (var bytes in bytesList) { - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); } result.Status = true; result.Msg = "操作成功"; @@ -577,14 +578,14 @@ namespace JiShe.CollectBus.EnergySystem var address = $"{code.AreaCode}{code.Address}"; var bytes = Build3761SendData.BuildAmmeterReportCollectionItemsSetSendCmd(address,input.Detail.Pn, input.Detail.Unit,input.Detail.Cycle,input.Detail.BaseTime, input.Detail.CurveRatio,input.Detail.Details.Select(it => new PnFn(it.Pn,it.Fn)).ToList()); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); } result.Status = true; result.Msg = "操作成功"; @@ -605,14 +606,14 @@ namespace JiShe.CollectBus.EnergySystem { var address = $"{code.AreaCode}{code.Address}"; var bytes = Build3761SendData.BuildAmmeterAutoUpSwitchSetSendCmd(address, input.Detail.Pn,input.Detail.IsOpen); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); } result.Status = true; result.Msg = "操作成功"; @@ -631,14 +632,14 @@ namespace JiShe.CollectBus.EnergySystem var result = new BaseResultDto(); var address = $"{input.AreaCode}{input.Address}"; var bytes = Build3761SendData.BuildAmmeterReadAutoUpSwitchSendCmd(address, input.Detail.Pn); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); result.Status = true; result.Msg = "操作成功"; return result; @@ -658,14 +659,14 @@ namespace JiShe.CollectBus.EnergySystem { var address = $"{data.AreaCode}{data.Address}"; var bytes = Build3761SendData.BuildTerminalVersionInfoReadingSendCmd(address); - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); } result.Status = true; result.Msg = "操作成功"; @@ -713,14 +714,14 @@ namespace JiShe.CollectBus.EnergySystem foreach (var bytes in bytesList) { - await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, DeviceNo = address, Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }); + }.Serialize()); } result.Status = true; diff --git a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs index 29427fc..7af99bc 100644 --- a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs +++ b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs @@ -3,7 +3,6 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Threading.Tasks; using DeviceDetectorNET.Parser.Device; -using DotNetCore.CAP; using JiShe.CollectBus.Ammeters; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index f2a08b1..0f601d9 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -1,5 +1,4 @@ -using DotNetCore.CAP; -using JiShe.CollectBus.Ammeters; +using JiShe.CollectBus.Ammeters; using JiShe.CollectBus.Common.BuildSendDatas; using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.DeviceBalanceControl; diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs index 8593dfd..c1e45bd 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Threading.Tasks; using Confluent.Kafka; -using DotNetCore.CAP; using JiShe.CollectBus.Ammeters; using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.DeviceBalanceControl; diff --git a/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs b/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs index 9bf297f..fb469ac 100644 --- a/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs @@ -1,5 +1,4 @@ -using DotNetCore.CAP; -using JiShe.CollectBus.Common.Enums; +using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.IoTDBProvider; diff --git a/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs b/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs index f3c485c..95b563c 100644 --- a/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Threading.Tasks; using DeviceDetectorNET.Parser.Device; -using DotNetCore.CAP; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.IotSystems.Devices; using JiShe.CollectBus.IotSystems.MessageIssueds; diff --git a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs index 23d0917..d066a28 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs +++ b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs @@ -1,5 +1,4 @@ -using DotNetCore.CAP; -using JiShe.CollectBus.Common.Enums; +using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.Protocol.Contracts.Interfaces; From 57123d653c62bb8817f129435f1cefc5b774c11a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E7=9B=8A?= Date: Tue, 15 Apr 2025 23:20:46 +0800 Subject: [PATCH 07/27] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BasicScheduledMeterReadingService.cs | 1 - .../Models/BusCacheGlobalPagedResult.cs | 7 +- .../Models/BusPagedResult.cs | 15 + .../Models/DeviceCacheBasicModel.cs | 4 +- .../FreeRedisProvider.cs | 544 +++++++++++++----- .../IFreeRedisProvider.cs | 60 ++ 6 files changed, 491 insertions(+), 140 deletions(-) diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index 33e70f9..2fd9781 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -21,7 +21,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; -using static FreeSql.Internal.GlobalFilter; namespace JiShe.CollectBus.ScheduledMeterReading { diff --git a/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs b/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs index ed42529..303b2e4 100644 --- a/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs +++ b/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs @@ -17,6 +17,11 @@ namespace JiShe.CollectBus.Common.Models /// public List Items { get; set; } + /// + /// 总条数 + /// + public long TotalCount { get; set; } + /// /// 是否有下一页 /// @@ -25,7 +30,7 @@ namespace JiShe.CollectBus.Common.Models /// /// 下一页的分页索引 /// - public long? NextScore { get; set; } + public decimal? NextScore { get; set; } /// /// 下一页的分页索引 diff --git a/src/JiShe.CollectBus.Common/Models/BusPagedResult.cs b/src/JiShe.CollectBus.Common/Models/BusPagedResult.cs index 2e68b14..f2bd1a3 100644 --- a/src/JiShe.CollectBus.Common/Models/BusPagedResult.cs +++ b/src/JiShe.CollectBus.Common/Models/BusPagedResult.cs @@ -31,5 +31,20 @@ namespace JiShe.CollectBus.Common.Models /// 数据集合 /// public IEnumerable Items { get; set; } + + /// + /// 是否有下一页 + /// + public bool HasNext { get; set; } + + /// + /// 下一页的分页索引 + /// + public decimal? NextScore { get; set; } + + /// + /// 下一页的分页索引 + /// + public string NextMember { get; set; } } } diff --git a/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs b/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs index 06a427b..59011de 100644 --- a/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs +++ b/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs @@ -22,8 +22,8 @@ namespace JiShe.CollectBus.Common.Models public int MeterId { get; set; } /// - /// 唯一标识 + /// 唯一标识，是redis ZSet和Set memberid /// - public virtual string UniqueId => $"{FocusId}:{MeterId}"; + public virtual string MemberID => $"{FocusId}:{MeterId}"; } } diff --git a/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs b/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs index 1e07e20..cebc9c4 100644 --- a/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs +++ b/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs @@ -2,12 +2,13 @@ using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.Common.Extensions; -using JiShe.CollectBus.FreeRedisProvider.Options; +using JiShe.CollectBus.FreeRedisProvider.Options; using Microsoft.Extensions.Options; using System.Diagnostics; using System.Text.Json; using Volo.Abp.DependencyInjection; using static System.Runtime.InteropServices.JavaScript.JSType; +using System.Collections.Concurrent; namespace JiShe.CollectBus.FreeRedisProvider { @@ -70,12 +71,12 @@ namespace JiShe.CollectBus.FreeRedisProvider { throw new ArgumentException($"{nameof(AddMeterCacheData)} 参数异常，-101"); } - + // 计算组合score（分类ID + 时间戳） var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow; long scoreValue = ((long)data.FocusId << 32) | (uint)actualTimestamp.Ticks; - + //全局索引写入 long globalScore = actualTimestamp.ToUnixTimeMilliseconds(); @@ -83,16 +84,16 @@ namespace JiShe.CollectBus.FreeRedisProvider using (var trans = Instance.Multi()) { // 主数据存储Hash - trans.HSet(redisCacheKey, data.UniqueId, data.Serialize()); - + trans.HSet(redisCacheKey, data.MemberID, data.Serialize()); + // 分类索引 - trans.SAdd(redisCacheFocusIndexKey, data.UniqueId); + trans.SAdd(redisCacheFocusIndexKey, data.MemberID); // 排序索引使用ZSET - trans.ZAdd(redisCacheScoresIndexKey, scoreValue, data.UniqueId); + trans.ZAdd(redisCacheScoresIndexKey, scoreValue, data.MemberID); //全局索引 - trans.ZAdd(redisCacheGlobalIndexKey, globalScore, data.UniqueId); + trans.ZAdd(redisCacheGlobalIndexKey, globalScore, data.MemberID); var results = trans.Exec(); @@ -122,6 +123,16 @@ namespace JiShe.CollectBus.FreeRedisProvider IEnumerable items, DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel { + if (items == null + || items.Count() <=0 + || string.IsNullOrWhiteSpace(redisCacheKey) + || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey) + || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) + || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) + { + throw new ArgumentException($"{nameof(BatchAddMeterData)} 参数异常，-101"); + } + const int BATCH_SIZE = 1000; // 每批1000条 var semaphore = new SemaphoreSlim(Environment.ProcessorCount * 2); @@ -144,16 +155,16 @@ namespace JiShe.CollectBus.FreeRedisProvider long globalScore = actualTimestamp.ToUnixTimeMilliseconds(); // 主数据存储Hash - pipe.HSet(redisCacheKey, item.UniqueId, item.Serialize()); + pipe.HSet(redisCacheKey, item.MemberID, item.Serialize()); // 分类索引 - pipe.SAdd(redisCacheFocusIndexKey, item.UniqueId); + pipe.SAdd(redisCacheFocusIndexKey, item.MemberID); // 排序索引使用ZSET - pipe.ZAdd(redisCacheScoresIndexKey, scoreValue, item.UniqueId); + pipe.ZAdd(redisCacheScoresIndexKey, scoreValue, item.MemberID); //全局索引 - pipe.ZAdd(redisCacheGlobalIndexKey, globalScore, item.UniqueId); + pipe.ZAdd(redisCacheGlobalIndexKey, globalScore, item.MemberID); } pipe.EndPipe(); } @@ -164,29 +175,108 @@ namespace JiShe.CollectBus.FreeRedisProvider await Task.CompletedTask; } - public async Task UpdateMeterData( + /// + /// 删除指定redis缓存key的缓存数据 + /// + /// + /// 主数据存储Hash缓存Key + /// 集中器索引Set缓存Key + /// 集中器排序索引ZSET缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 表计信息 + /// + public async Task RemoveMeterData( string redisCacheKey, - string oldCategoryIndexKey, - string newCategoryIndexKey, - string memberId, // 唯一标识（格式："分类ID:GUID"） - T newData, - int? newCategoryId = null, - DateTimeOffset? newTimestamp = null) + string redisCacheFocusIndexKey, + string redisCacheScoresIndexKey, + string redisCacheGlobalIndexKey, + T data) where T : DeviceCacheBasicModel { - // 参数校验 - if (string.IsNullOrWhiteSpace(memberId)) - throw new ArgumentException("Invalid member ID"); + + if (data == null + || string.IsNullOrWhiteSpace(redisCacheKey) + || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey) + || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) + || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) + { + throw new ArgumentException($"{nameof(RemoveMeterData)} 参数异常，-101"); + } + + const string luaScript = @" + local mainKey = KEYS[1] + local focusIndexKey = KEYS[2] + local scoresIndexKey = KEYS[3] + local globalIndexKey = KEYS[4] + local member = ARGV[1] + + local deleted = 0 + if redis.call('HDEL', mainKey, member) > 0 then + deleted = 1 + end + + redis.call('SREM', focusIndexKey, member) + redis.call('ZREM', scoresIndexKey, member) + redis.call('ZREM', globalIndexKey, member) + return deleted + "; + + var keys = new[] + { + redisCacheKey, + redisCacheFocusIndexKey, + redisCacheScoresIndexKey, + redisCacheGlobalIndexKey + }; + + var result = await Instance.EvalAsync(luaScript, keys, new[] { data.MemberID }); + + if ((int)result == 0) + throw new KeyNotFoundException("指定数据不存在"); + } + + /// + /// 修改表计缓存信息 + /// + /// + /// 主数据存储Hash缓存Key + /// 旧集中器索引Set缓存Key + /// 新集中器索引Set缓存Key + /// 集中器排序索引ZSET缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 表计信息 + /// 可选时间戳 + /// + public async Task UpdateMeterData( + string redisCacheKey, + string oldRedisCacheFocusIndexKey, + string newRedisCacheFocusIndexKey, + string redisCacheScoresIndexKey, + string redisCacheGlobalIndexKey, + T newData, + DateTimeOffset? newTimestamp = null) where T : DeviceCacheBasicModel + { + if (newData == null + || string.IsNullOrWhiteSpace(redisCacheKey) + || string.IsNullOrWhiteSpace(oldRedisCacheFocusIndexKey) + || string.IsNullOrWhiteSpace(newRedisCacheFocusIndexKey) + || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) + || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) + { + throw new ArgumentException($"{nameof(UpdateMeterData)} 参数异常，-101"); + } var luaScript = @" local mainKey = KEYS[1] - local scoreKey = KEYS[2] - local oldIndex = KEYS[3] - local newIndex = KEYS[4] + local oldFocusIndexKey = KEYS[2] + local newFocusIndexKey = KEYS[3] + local scoresIndexKey = KEYS[4] + local globalIndexKey = KEYS[5] local member = ARGV[1] local newData = ARGV[2] local newScore = ARGV[3] + local newGlobalScore = ARGV[4] - -- 校验旧数据是否存在 + -- 校验存在性 if redis.call('HEXISTS', mainKey, member) == 0 then return 0 end @@ -194,70 +284,67 @@ namespace JiShe.CollectBus.FreeRedisProvider -- 更新主数据 redis.call('HSET', mainKey, member, newData) - -- 处理分类变更 + -- 处理变更 if newScore ~= '' then -- 删除旧索引 - redis.call('SREM', oldIndex, member) - -- 更新排序分数 - redis.call('ZADD', scoreKey, newScore, member) + redis.call('SREM', oldFocusIndexKey, member) + redis.call('ZREM', scoresIndexKey, member) + -- 添加新索引 - redis.call('SADD', newIndex, member) + redis.call('SADD', newFocusIndexKey, member) + redis.call('ZADD', scoresIndexKey, newScore, member) + end + + -- 更新全局索引 + if newGlobalScore ~= '' then + -- 删除旧索引 + redis.call('ZREM', globalIndexKey, member) + + -- 添加新索引 + redis.call('ZADD', globalIndexKey, newGlobalScore, member) end return 1 "; - // 计算新score（当分类或时间变化时） - long? newScoreValue = null; - if (newCategoryId.HasValue || newTimestamp.HasValue) - { - var parts = memberId.Split(':'); - var oldCategoryId = int.Parse(parts[0]); - - var actualCategoryId = newCategoryId ?? oldCategoryId; - var actualTimestamp = newTimestamp ?? DateTimeOffset.UtcNow; - - newScoreValue = ((long)actualCategoryId << 32) | (uint)actualTimestamp.Ticks; - } + var actualTimestamp = newTimestamp ?? DateTimeOffset.UtcNow; + var newGlobalScore = actualTimestamp.ToUnixTimeMilliseconds(); + var newScoreValue = ((long)newData.FocusId << 32) | (uint)actualTimestamp.Ticks; var result = await Instance.EvalAsync(luaScript, new[] { redisCacheKey, - $"{redisCacheKey}_scores", - oldCategoryIndexKey, - newCategoryIndexKey + oldRedisCacheFocusIndexKey, + newRedisCacheFocusIndexKey, + redisCacheScoresIndexKey, + redisCacheGlobalIndexKey }, - new[] + new object[] { - memberId, + newData.MemberID, newData.Serialize(), - newScoreValue?.ToString() ?? "" + newScoreValue.ToString() ?? "", + newGlobalScore.ToString() ?? "" }); - // 如果时间戳变化则更新全局索引 - if (newTimestamp.HasValue) - { - long newGlobalScore = newTimestamp.Value.ToUnixTimeMilliseconds(); - await Instance.ZAddAsync("global_data_all", newGlobalScore, memberId); - } - if ((int)result == 0) - throw new KeyNotFoundException("指定数据不存在"); + { + throw new KeyNotFoundException($"{nameof(UpdateMeterData)}指定Key{redisCacheKey}的数据不存在"); + } } - - - public async Task> GetMeterZSetPagedData( + + public async Task> SingleGetMeterPagedData( string redisCacheKey, - string redisCacheIndexKey, - int categoryId, + string redisCacheScoresIndexKey, + int focusId, int pageSize = 10, int pageIndex = 1, bool descending = true) { // 计算score范围 - long minScore = (long)categoryId << 32; - long maxScore = ((long)categoryId + 1) << 32; + long minScore = (long)focusId << 32; + long maxScore = ((long)focusId + 1) << 32; // 分页参数计算 int start = (pageIndex - 1) * pageSize; @@ -265,13 +352,13 @@ namespace JiShe.CollectBus.FreeRedisProvider // 获取排序后的member列表 var members = descending ? await Instance.ZRevRangeByScoreAsync( - $"{redisCacheKey}_scores", + redisCacheScoresIndexKey, maxScore, minScore, start, pageSize) : await Instance.ZRangeByScoreAsync( - $"{redisCacheKey}_scores", + redisCacheScoresIndexKey, minScore, maxScore, start, @@ -284,7 +371,7 @@ namespace JiShe.CollectBus.FreeRedisProvider // 总数统计优化 var total = await Instance.ZCountAsync( - $"{redisCacheKey}_scores", + redisCacheScoresIndexKey, minScore, maxScore); @@ -298,105 +385,290 @@ namespace JiShe.CollectBus.FreeRedisProvider } - public async Task RemoveMeterZSetData( - string redisCacheKey, - string redisCacheIndexKey, - string uniqueId) // 改为基于唯一标识删除 - { - // 原子操作 - var luaScript = @" - local mainKey = KEYS[1] - local scoreKey = KEYS[2] - local indexKey = KEYS[3] - local member = ARGV[1] - - redis.call('HDEL', mainKey, member) - redis.call('ZREM', scoreKey, member) - redis.call('SREM', indexKey, member) - return 1 - "; - - var keys = new[] - { - redisCacheKey, - $"{redisCacheKey}_scores", - redisCacheIndexKey - }; - - var result = await Instance.EvalAsync(luaScript, - keys, - new[] { uniqueId }); - - if ((int)result != 1) - throw new Exception("删除操作失败"); - } - - public async Task> GetGlobalPagedData( + public async Task> GetFocusPagedData( string redisCacheKey, + string redisCacheScoresIndexKey, + int focusId, int pageSize = 10, long? lastScore = null, string lastMember = null, - bool descending = true) + bool descending = true) where T : DeviceCacheBasicModel { - const string zsetKey = "global_data_all"; - - // 分页参数处理 - var (startScore, excludeMember) = descending - ? (lastScore ?? long.MaxValue, lastMember) - : (lastScore ?? 0, lastMember); + // 计算分数范围 + long minScore = (long)focusId << 32; + long maxScore = ((long)focusId + 1) << 32; // 获取成员列表 - string[] members; - if (descending) + var members = await GetSortedMembers( + redisCacheScoresIndexKey, + minScore, + maxScore, + pageSize, + lastScore, + lastMember, + descending); + + // 批量获取数据 + var dataDict = await Instance.HMGetAsync(redisCacheKey, members.CurrentItems); + + return new BusPagedResult { - members = await Instance.ZRevRangeByScoreAsync( + Items = dataDict, + TotalCount = await GetTotalCount(redisCacheScoresIndexKey, minScore, maxScore), + HasNext = members.HasNext, + NextScore = members.NextScore, + NextMember = members.NextMember + }; + } + + private async Task<(string[] CurrentItems, bool HasNext, decimal? NextScore, string NextMember)> + GetSortedMembers( + string zsetKey, + long minScore, + long maxScore, + int pageSize, + long? lastScore, + string lastMember, + bool descending) + { + var querySize = pageSize + 1; + var (startScore, exclude) = descending + ? (lastScore ?? maxScore, lastMember) + : (lastScore ?? minScore, lastMember); + + var members = descending + ? await Instance.ZRevRangeByScoreAsync( zsetKey, max: startScore, - min: 0, + min: minScore, offset: 0, - count: pageSize + 1); - } - else - { - members = await Instance.ZRangeByScoreAsync( + count: querySize) + : await Instance.ZRangeByScoreAsync( zsetKey, min: startScore, - max: long.MaxValue, + max: maxScore, offset: 0, - count: pageSize + 1); + count: querySize); + + var hasNext = members.Length > pageSize; + var currentItems = members.Take(pageSize).ToArray(); + + var nextCursor = currentItems.Any() + ? await GetNextCursor(zsetKey, currentItems.Last(), descending) + : (null, null); + + return (currentItems, hasNext, nextCursor.score, nextCursor.member); + } + + private async Task GetTotalCount(string zsetKey, long min, long max) + { + // 缓存计数优化 + var cacheKey = $"{zsetKey}_count_{min}_{max}"; + var cached = await Instance.GetAsync(cacheKey); + + if (cached.HasValue) + return cached.Value; + + var count = await Instance.ZCountAsync(zsetKey, min, max); + await Instance.SetExAsync(cacheKey, 60, count); // 缓存60秒 + return count; + } + + + public async Task>> BatchGetMeterPagedData( + string redisCacheKey, + string redisCacheScoresIndexKey, + IEnumerable focusIds, + int pageSizePerFocus = 10) where T : DeviceCacheBasicModel + { + var results = new ConcurrentDictionary>(); + var parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount * 2 + }; + + await Parallel.ForEachAsync(focusIds, parallelOptions, async (focusId, _) => + { + var data = await SingleGetMeterPagedData( + redisCacheKey, + redisCacheScoresIndexKey, + focusId, + pageSizePerFocus); + + results.TryAdd(focusId, data); + }); + + return new Dictionary>(results); + } + + /// + /// 通过全局索引分页查询表计缓存数据 + /// + /// + /// 主数据存储Hash缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 分页尺寸 + /// 最后一个索引 + /// 最后一个唯一标识 + /// 排序方式 + /// + public async Task> GetGlobalPagedData( + string redisCacheKey, + string redisCacheGlobalIndexKey, + int pageSize = 10, + decimal? lastScore = null, + string lastMember = null, + bool descending = true) + where T : DeviceCacheBasicModel + { + // 参数校验增强 + if (string.IsNullOrWhiteSpace(redisCacheKey) || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) + { + throw new ArgumentException($"{nameof(GetGlobalPagedData)} 参数异常，-101"); } - // 处理分页结果 - bool hasNext = members.Length > pageSize; - var actualMembers = members.Take(pageSize).ToArray(); + if (pageSize < 1 || pageSize > 1000) + { + throw new ArgumentException($"{nameof(GetGlobalPagedData)} 分页大小应在1-1000之间，-102"); + } - // 批量获取数据（优化版本） - var dataTasks = actualMembers - .Select(m => Instance.HGetAsync(redisCacheKey, m)) - .ToArray(); - await Task.WhenAll(dataTasks); + // 分页参数解析 + var (startScore, excludeMember) = descending + ? (lastScore ?? decimal.MaxValue, lastMember) + : (lastScore ?? 0, lastMember); + + // 游标分页查询 + var (members, hasNext) = await GetPagedMembers( + redisCacheGlobalIndexKey, + pageSize, + startScore, + excludeMember, + descending); + + // 批量获取数据（优化内存分配） + var dataDict = await BatchGetData(redisCacheKey, members); // 获取下一页游标 - (long? nextScore, string nextMember) = actualMembers.Any() - ? await GetNextCursor(zsetKey, actualMembers.Last(), descending) + var nextCursor = members.Any() + ? await GetNextCursor(redisCacheGlobalIndexKey, members.Last(), descending) : (null, null); return new BusCacheGlobalPagedResult { - Items = dataTasks.Select(t => t.Result).ToList(), + Items = members.Select(m => dataDict.TryGetValue(m, out var v) ? v : default) + .Where(x => x != null).ToList(), HasNext = hasNext, - NextScore = nextScore, - NextMember = nextMember + NextScore = nextCursor.score, + NextMember = nextCursor.member }; } - private async Task<(long? score, string member)> GetNextCursor( - string zsetKey, + /// + /// 游标分页查询 + /// + /// + /// 分页数量 + /// 开始索引 + /// 开始唯一标识 + /// 排序方式 + /// + private async Task<(List Members, bool HasNext)> GetPagedMembers( + string redisCacheGlobalIndexKey, + int pageSize, + decimal? startScore, + string excludeMember, + bool descending) + { + const int bufferSize = 50; // 预读缓冲区大小 + + // 使用流式分页（避免OFFSET性能问题） + var members = new List(pageSize + 1); + decimal? currentScore = startScore; + string lastMember = excludeMember; + + while (members.Count < pageSize + 1 && currentScore.HasValue) + { + var querySize = Math.Min(bufferSize, pageSize + 1 - members.Count); + + var batch = descending + ? await Instance.ZRevRangeByScoreAsync( + redisCacheGlobalIndexKey, + max: currentScore.Value, + min: 0, + offset: 0, + count: querySize + ) + : await Instance.ZRangeByScoreAsync( + redisCacheGlobalIndexKey, + min: currentScore.Value, + max: long.MaxValue, + offset: 0, + count: querySize); + + if (!batch.Any()) break; + + members.AddRange(batch); + lastMember = batch.LastOrDefault(); + currentScore = await Instance.ZScoreAsync(redisCacheGlobalIndexKey, lastMember); + } + + return ( + members.Take(pageSize).ToList(), + members.Count > pageSize + ); + } + + /// + /// 批量获取指定分页的数据 + /// + /// + /// + /// + /// + private async Task> BatchGetData( + string hashKey, + IEnumerable members) + where T : DeviceCacheBasicModel + { + const int batchSize = 100; + var result = new Dictionary(); + + foreach (var batch in members.Batch(batchSize)) + { + var batchArray = batch.ToArray(); + var values = await Instance.HMGetAsync(hashKey, batchArray); + + for (int i = 0; i < batchArray.Length; i++) + { + if (EqualityComparer.Default.Equals(values[i], default)) continue; + result[batchArray[i]] = values[i]; + } + } + + return result; + } + + /// + /// 获取下一页游标 + /// + /// 全局索引Key + /// 最后一个唯一标识 + /// 排序方式 + /// + private async Task<(decimal? score, string member)> GetNextCursor( + string redisCacheGlobalIndexKey, string lastMember, bool descending) { - var score = await Instance.ZScoreAsync(zsetKey, lastMember); - return (score.HasValue ? (long)score.Value : null, lastMember); + if (string.IsNullOrWhiteSpace(lastMember)) + { + return (null, null); + } + + var score = await Instance.ZScoreAsync(redisCacheGlobalIndexKey, lastMember); + return score.HasValue + ? (Convert.ToInt64(score.Value), lastMember) + : (null, null); } } } \ No newline at end of file diff --git a/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs b/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs index 36f9a81..dc0aaa3 100644 --- a/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs +++ b/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs @@ -48,6 +48,66 @@ namespace JiShe.CollectBus.FreeRedisProvider string redisCacheGlobalIndexKey, IEnumerable items, DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel; + + /// + /// 删除指定redis缓存key的缓存数据 + /// + /// + /// 主数据存储Hash缓存Key + /// 集中器索引Set缓存Key + /// 集中器排序索引ZSET缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 表计信息 + /// + Task RemoveMeterData( + string redisCacheKey, + string redisCacheFocusIndexKey, + string redisCacheScoresIndexKey, + string redisCacheGlobalIndexKey, + T data) where T : DeviceCacheBasicModel; + + /// + /// 修改表计缓存信息 + /// + /// + /// 主数据存储Hash缓存Key + /// 旧集中器索引Set缓存Key + /// 新集中器索引Set缓存Key + /// 集中器排序索引ZSET缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 表计信息 + /// 可选时间戳 + /// + Task UpdateMeterData( + string redisCacheKey, + string oldRedisCacheFocusIndexKey, + string newRedisCacheFocusIndexKey, + string redisCacheScoresIndexKey, + string redisCacheGlobalIndexKey, + T newData, + DateTimeOffset? newTimestamp = null) where T : DeviceCacheBasicModel; + + + + /// + /// 通过全局索引分页查询表计缓存数据 + /// + /// + /// 主数据存储Hash缓存Key + /// 集中器采集频率分组全局索引ZSet缓存Key + /// 分页尺寸 + /// 最后一个索引 + /// 最后一个唯一标识 + /// 排序方式 + /// + Task> GetGlobalPagedData( + string redisCacheKey, + string redisCacheGlobalIndexKey, + int pageSize = 10, + decimal? lastScore = null, + string lastMember = null, + bool descending = true) + where T : DeviceCacheBasicModel; } } From 0d4c7807273bb64d6ecc0ee8e4a5372c222c0fa8 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Wed, 16 Apr 2025 09:54:21 +0800 Subject: [PATCH 08/27] =?UTF-8?q?=E4=BF=9D=E7=95=99CAP=E7=BB=84=E4=BB=B6?= =?UTF-8?q?=EF=BC=8C=E5=90=8E=E7=BB=AD=E5=86=B3=E5=AE=9A=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E7=A7=BB=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- JiShe.CollectBus.sln | 7 + .../EnergySystem/EnergySystemAppService.cs | 152 +++++++++++++++++- .../JiShe.CollectBus.Application.csproj | 1 + .../Plugins/TcpMonitor.cs | 23 ++- .../Samples/SampleAppService.cs | 2 +- .../BasicScheduledMeterReadingService.cs | 14 +- ...nergySystemScheduledMeterReadingService.cs | 3 +- .../Subscribers/SubscriberAppService.cs | 10 +- .../Subscribers/WorkerSubscriberAppService.cs | 7 +- .../CollectBusHostModule.cs | 2 +- .../AdminClient/AdminClientService.cs | 13 ++ .../AdminClient/IAdminClientService.cs | 7 + .../Attributes/KafkaSubscribeAttribute.cs | 22 +-- .../CollectBusKafkaModule.cs | 4 +- .../Consumer/ConsumerService.cs | 24 ++- .../KafkaSubcribesExtensions.cs | 30 +++- .../Producer/IProducerService.cs | 2 +- .../Producer/ProducerService.cs | 12 +- .../Abstracts/BaseProtocolPlugin.cs | 8 +- ...JiShe.CollectBus.Protocol.Contracts.csproj | 1 + 20 files changed, 299 insertions(+), 45 deletions(-) diff --git a/JiShe.CollectBus.sln b/JiShe.CollectBus.sln index 9c67aae..ea5e278 100644 --- a/JiShe.CollectBus.sln +++ b/JiShe.CollectBus.sln @@ -39,6 +39,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Cassandra", "src\JiShe.CollectBus.Cassandra\JiShe.CollectBus.Cassandra.csproj", "{443B4549-0AC0-4493-8F3E-49C83225DD76}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Kafka.Test", "src\JiShe.CollectBus.Kafka.Test\JiShe.CollectBus.Kafka.Test.csproj", "{FA762E8F-659A-DECF-83D6-5F364144450E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -113,6 +115,10 @@ Global {443B4549-0AC0-4493-8F3E-49C83225DD76}.Debug|Any CPU.Build.0 = Debug|Any CPU {443B4549-0AC0-4493-8F3E-49C83225DD76}.Release|Any CPU.ActiveCfg = Release|Any CPU {443B4549-0AC0-4493-8F3E-49C83225DD76}.Release|Any CPU.Build.0 = Release|Any CPU + {FA762E8F-659A-DECF-83D6-5F364144450E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FA762E8F-659A-DECF-83D6-5F364144450E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FA762E8F-659A-DECF-83D6-5F364144450E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FA762E8F-659A-DECF-83D6-5F364144450E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -135,6 +141,7 @@ Global {A3F3C092-0A25-450B-BF6A-5983163CBEF5} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545} {A377955E-7EA1-6F29-8CF7-774569E93925} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545} {443B4549-0AC0-4493-8F3E-49C83225DD76} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545} + {FA762E8F-659A-DECF-83D6-5F364144450E} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4324B3B4-B60B-4E3C-91D8-59576B4E26DD} diff --git a/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs b/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs index a210d28..2a60803 100644 --- a/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs +++ b/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs @@ -5,6 +5,7 @@ using System.Net; using System.Text; using System.Threading.Tasks; using DeviceDetectorNET.Class.Device; +using DotNetCore.CAP; using JiShe.CollectBus.Common.BuildSendDatas; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; @@ -30,14 +31,16 @@ namespace JiShe.CollectBus.EnergySystem private readonly IRepository _csqRecordRepository; private readonly IRepository _conrOnlineRecordRepository; private readonly IProducerService _producerService; + private readonly ICapPublisher _capBus; public EnergySystemAppService(IRepository focusRecordRepository, IRepository csqRecordRepository, - IRepository conrOnlineRecordRepository, IProducerService producerService) + IRepository conrOnlineRecordRepository, IProducerService producerService, ICapPublisher capBus) { _focusRecordRepository = focusRecordRepository; _csqRecordRepository = csqRecordRepository; _conrOnlineRecordRepository = conrOnlineRecordRepository; _producerService = producerService; + _capBus = capBus; } /// @@ -71,7 +74,15 @@ namespace JiShe.CollectBus.EnergySystem if (bytes == null) return result; - + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -109,6 +120,15 @@ namespace JiShe.CollectBus.EnergySystem foreach (var bytes in bytesList) { + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -117,6 +137,7 @@ namespace JiShe.CollectBus.EnergySystem Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() }.Serialize()); + } return result; @@ -150,6 +171,14 @@ namespace JiShe.CollectBus.EnergySystem }).ToList(); var bytes = Build3761SendData.BuildAmmeterParameterSetSendCmd(address, meterParameters); + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -179,6 +208,15 @@ namespace JiShe.CollectBus.EnergySystem { var dataUnit = Build645SendData.BuildReadMeterAddressSendDataUnit(detail.MeterAddress); var bytes =Build3761SendData.BuildTransparentForwardingSendCmd(address, detail.Port, detail.BaudRate.ToString(), dataUnit, StopBit.Stop1, Parity.None); + + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -262,6 +300,15 @@ namespace JiShe.CollectBus.EnergySystem if (bytes != null) { + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -321,6 +368,15 @@ namespace JiShe.CollectBus.EnergySystem var bytes = Build3761SendData.BuildCommunicationParametersSetSendCmd(address, masterIP, materPort, backupIP, backupPort, input.Data.APN); + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -348,6 +404,15 @@ namespace JiShe.CollectBus.EnergySystem var address = $"{input.AreaCode}{input.Address}"; var bytes = Build3761SendData.BuildTerminalCalendarClockSendCmd(address); + + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -376,6 +441,14 @@ namespace JiShe.CollectBus.EnergySystem bool isManual = !input.AreaCode.Equals("5110");//低功耗集中器不是长连接，在连接的那一刻再发送 var bytes = Build3761SendData.BuildConrCheckTimeSendCmd(address,DateTime.Now, isManual); + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -403,6 +476,14 @@ namespace JiShe.CollectBus.EnergySystem var address = $"{input.AreaCode}{input.Address}"; var bytes = Build3761SendData.BuildConrRebootSendCmd(address); + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -431,6 +512,14 @@ namespace JiShe.CollectBus.EnergySystem var address = $"{input.AreaCode}{input.Address}"; var pnList = input.Data.Split(',').Select(it => int.Parse(it)).ToList(); var bytes = Build3761SendData.BuildAmmeterParameterReadingSendCmd(address, pnList); + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -480,6 +569,15 @@ namespace JiShe.CollectBus.EnergySystem foreach (var bytes in bytesList) { + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); + await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -488,7 +586,7 @@ namespace JiShe.CollectBus.EnergySystem Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() }.Serialize()); - + } result.Status = true; result.Msg = "操作成功"; @@ -549,6 +647,14 @@ namespace JiShe.CollectBus.EnergySystem foreach (var bytes in bytesList) { + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -578,6 +684,14 @@ namespace JiShe.CollectBus.EnergySystem var address = $"{code.AreaCode}{code.Address}"; var bytes = Build3761SendData.BuildAmmeterReportCollectionItemsSetSendCmd(address,input.Detail.Pn, input.Detail.Unit,input.Detail.Cycle,input.Detail.BaseTime, input.Detail.CurveRatio,input.Detail.Details.Select(it => new PnFn(it.Pn,it.Fn)).ToList()); + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -606,6 +720,14 @@ namespace JiShe.CollectBus.EnergySystem { var address = $"{code.AreaCode}{code.Address}"; var bytes = Build3761SendData.BuildAmmeterAutoUpSwitchSetSendCmd(address, input.Detail.Pn,input.Detail.IsOpen); + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -632,6 +754,14 @@ namespace JiShe.CollectBus.EnergySystem var result = new BaseResultDto(); var address = $"{input.AreaCode}{input.Address}"; var bytes = Build3761SendData.BuildAmmeterReadAutoUpSwitchSendCmd(address, input.Detail.Pn); + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -659,6 +789,14 @@ namespace JiShe.CollectBus.EnergySystem { var address = $"{data.AreaCode}{data.Address}"; var bytes = Build3761SendData.BuildTerminalVersionInfoReadingSendCmd(address); + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, @@ -714,6 +852,14 @@ namespace JiShe.CollectBus.EnergySystem foreach (var bytes in bytesList) { + //await _capBus.PublishAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage + //{ + // //ClientId = messageReceived.ClientId, + // DeviceNo = address, + // Message = bytes, + // Type = IssuedEventType.Data, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerManualValveReadingIssuedEventName, new IssuedEventMessage { //ClientId = messageReceived.ClientId, diff --git a/src/JiShe.CollectBus.Application/JiShe.CollectBus.Application.csproj b/src/JiShe.CollectBus.Application/JiShe.CollectBus.Application.csproj index 24db5a5..3fa551c 100644 --- a/src/JiShe.CollectBus.Application/JiShe.CollectBus.Application.csproj +++ b/src/JiShe.CollectBus.Application/JiShe.CollectBus.Application.csproj @@ -15,6 +15,7 @@ + diff --git a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs index 7af99bc..50b41b0 100644 --- a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs +++ b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Threading.Tasks; using DeviceDetectorNET.Parser.Device; +using DotNetCore.CAP; using JiShe.CollectBus.Ammeters; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; @@ -27,6 +28,7 @@ namespace JiShe.CollectBus.Plugins { public partial class TcpMonitor : PluginBase, ITransientDependency, ITcpReceivedPlugin, ITcpConnectingPlugin, ITcpConnectedPlugin, ITcpClosedPlugin { + private readonly ICapPublisher _producerBus; private readonly IProducerService _producerService; private readonly ILogger _logger; private readonly IRepository _deviceRepository; @@ -39,11 +41,12 @@ namespace JiShe.CollectBus.Plugins /// /// /// - public TcpMonitor(IProducerService producerService, + public TcpMonitor(ICapPublisher producerBus, IProducerService producerService, ILogger logger, IRepository deviceRepository, IDistributedCache ammeterInfoCache) { + _producerBus = producerBus; _producerService = producerService; _logger = logger; _deviceRepository = deviceRepository; @@ -171,6 +174,10 @@ namespace JiShe.CollectBus.Plugins DeviceNo = deviceNo, MessageId = NewId.NextGuid().ToString() }; + + //await _producerBus.PublishAsync(ProtocolConst.SubscriberLoginReceivedEventName, messageReceivedLoginEvent); + + await _producerService.ProduceAsync(ProtocolConst.SubscriberLoginReceivedEventName, messageReceivedLoginEvent.Serialize()); //await _producerBus.Publish( messageReceivedLoginEvent); @@ -218,6 +225,8 @@ namespace JiShe.CollectBus.Plugins DeviceNo = deviceNo, MessageId = NewId.NextGuid().ToString() }; + //await _producerBus.PublishAsync(ProtocolConst.SubscriberHeartbeatReceivedEventName, messageReceivedHeartbeatEvent); + await _producerService.ProduceAsync(ProtocolConst.SubscriberHeartbeatReceivedEventName, messageReceivedHeartbeatEvent.Serialize()); //await _producerBus.Publish(messageReceivedHeartbeatEvent); } @@ -242,10 +251,18 @@ namespace JiShe.CollectBus.Plugins // MessageId = NewId.NextGuid().ToString() //}); - + //string topicName = string.Format(ProtocolConst.AFNTopicNameFormat, aFn); //todo 如何确定时标？目前集中器的采集频率，都是固定，数据上报的时候，根据当前时间，往后推测出应当采集的时间点作为时标。但是如果由于网络问题，数据一直没上报的情况改怎么计算？ - + //await _producerBus.PublishAsync(ProtocolConst.SubscriberReceivedEventName, new MessageReceived + //{ + // ClientId = client.Id, + // ClientIp = client.IP, + // ClientPort = client.Port, + // MessageHexString = messageHexString, + // DeviceNo = deviceNo, + // MessageId = NewId.NextGuid().ToString() + //}); await _producerService.ProduceAsync(ProtocolConst.SubscriberReceivedEventName, new MessageReceived { ClientId = client.Id, diff --git a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs index 78b472d..d4c9a59 100644 --- a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs +++ b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs @@ -215,7 +215,7 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS return aa == null; } - [KafkaSubscribe(["test-topic"])] + [KafkaSubscribe("test-topic1")] public async Task KafkaSubscribeAsync(object obj) { diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index 0f601d9..8fa4f5e 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -1,4 +1,5 @@ -using JiShe.CollectBus.Ammeters; +using DotNetCore.CAP; +using JiShe.CollectBus.Ammeters; using JiShe.CollectBus.Common.BuildSendDatas; using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.DeviceBalanceControl; @@ -31,14 +32,16 @@ namespace JiShe.CollectBus.ScheduledMeterReading private readonly IIoTDBProvider _dbProvider; private readonly IMeterReadingRecordRepository _meterReadingRecordRepository; private readonly IProducerService _producerService; - + private readonly ICapPublisher _producerBus; public BasicScheduledMeterReadingService( ILogger logger, + ICapPublisher producerBus, IMeterReadingRecordRepository meterReadingRecordRepository, IProducerService producerService, IIoTDBProvider dbProvider) { + _producerBus = producerBus; _logger = logger; _dbProvider = dbProvider; _meterReadingRecordRepository = meterReadingRecordRepository; @@ -377,6 +380,8 @@ namespace JiShe.CollectBus.ScheduledMeterReading FocusAddress = ammerterItem.Value.FocusAddress, TimeDensity = timeDensity.ToString(), }; + //_ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName, tempMsg); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName, tempMsg.Serialize()); //_= _producerBus.Publish(tempMsg); @@ -441,6 +446,8 @@ namespace JiShe.CollectBus.ScheduledMeterReading FocusAddress = ammerterItem.Value.FocusAddress, TimeDensity = timeDensity.ToString(), }; + //_ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName, tempMsg); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName, tempMsg.Serialize()); //_ = _producerBus.Publish(tempMsg); @@ -505,6 +512,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading FocusAddress = ammerterItem.Value.FocusAddress, TimeDensity = timeDensity.ToString(), }; + //_ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg); _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg.Serialize()); @@ -841,6 +849,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading FocusAddress = ammerterItem.Value.FocusAddress, TimeDensity = timeDensity.ToString(), }; + //_ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg); _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg.Serialize()); @@ -1149,6 +1158,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading FocusAddress = ammerterItem.Value.FocusAddress, TimeDensity = timeDensity.ToString(), }; + //await _producerBus.PublishAsync(ProtocolConst.WatermeterSubscriberWorkerAutoReadingIssuedEventName, tempMsg); //_ = _producerBus.Publish(tempMsg); diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs index c1e45bd..3b0da49 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Confluent.Kafka; +using DotNetCore.CAP; using JiShe.CollectBus.Ammeters; using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.DeviceBalanceControl; @@ -35,7 +36,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading { string serverTagName = string.Empty; public EnergySystemScheduledMeterReadingService(ILogger logger, - IIoTDBProvider dbProvider, IMeterReadingRecordRepository meterReadingRecordRepository,IConfiguration configuration, IProducerService producerService) : base(logger, meterReadingRecordRepository, producerService,dbProvider) + ICapPublisher producerBus, IIoTDBProvider dbProvider, IMeterReadingRecordRepository meterReadingRecordRepository,IConfiguration configuration, IProducerService producerService) : base(logger, producerBus, meterReadingRecordRepository, producerService,dbProvider) { serverTagName = configuration.GetValue(CommonConst.ServerTagName)!; } diff --git a/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs b/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs index fb469ac..3ec6936 100644 --- a/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs @@ -1,4 +1,5 @@ -using JiShe.CollectBus.Common.Enums; +using DotNetCore.CAP; +using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.IoTDBProvider; @@ -21,7 +22,7 @@ using Volo.Abp.Domain.Repositories; namespace JiShe.CollectBus.Subscribers { - public class SubscriberAppService : CollectBusAppService, ISubscriberAppService, IKafkaSubscribe + public class SubscriberAppService : CollectBusAppService, ISubscriberAppService, ICapSubscribe, IKafkaSubscribe { private readonly ILogger _logger; private readonly ITcpService _tcpService; @@ -65,6 +66,7 @@ namespace JiShe.CollectBus.Subscribers } [KafkaSubscribe(ProtocolConst.SubscriberLoginIssuedEventName)] + //[CapSubscribe(ProtocolConst.SubscriberLoginIssuedEventName)] public async Task LoginIssuedEvent(IssuedEventMessage issuedEventMessage) { bool isAck = false; @@ -97,6 +99,7 @@ namespace JiShe.CollectBus.Subscribers } [KafkaSubscribe(ProtocolConst.SubscriberHeartbeatIssuedEventName)] + //[CapSubscribe(ProtocolConst.SubscriberHeartbeatIssuedEventName)] public async Task HeartbeatIssuedEvent(IssuedEventMessage issuedEventMessage) { bool isAck = false; @@ -127,6 +130,7 @@ namespace JiShe.CollectBus.Subscribers } [KafkaSubscribe(ProtocolConst.SubscriberReceivedEventName)] + //[CapSubscribe(ProtocolConst.SubscriberReceivedEventName)] public async Task ReceivedEvent(MessageReceived receivedMessage) { var currentTime = Clock.Now; @@ -183,6 +187,7 @@ namespace JiShe.CollectBus.Subscribers } [KafkaSubscribe(ProtocolConst.SubscriberHeartbeatReceivedEventName)] + //[CapSubscribe(ProtocolConst.SubscriberHeartbeatReceivedEventName)] public async Task ReceivedHeartbeatEvent(MessageReceivedHeartbeat receivedHeartbeatMessage) { var protocolPlugin = _serviceProvider.GetKeyedService("StandardProtocolPlugin"); @@ -199,6 +204,7 @@ namespace JiShe.CollectBus.Subscribers } [KafkaSubscribe(ProtocolConst.SubscriberLoginReceivedEventName)] + //[CapSubscribe(ProtocolConst.SubscriberLoginReceivedEventName)] public async Task ReceivedLoginEvent(MessageReceivedLogin receivedLoginMessage) { var protocolPlugin = _serviceProvider.GetKeyedService("StandardProtocolPlugin"); diff --git a/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs b/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs index 95b563c..47cff47 100644 --- a/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using DeviceDetectorNET.Parser.Device; +using DotNetCore.CAP; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.IotSystems.Devices; using JiShe.CollectBus.IotSystems.MessageIssueds; @@ -25,7 +26,7 @@ namespace JiShe.CollectBus.Subscribers /// 定时抄读任务消息消费订阅 /// [Route($"/worker/app/subscriber")] - public class WorkerSubscriberAppService : CollectBusAppService, IWorkerSubscriberAppService, IKafkaSubscribe + public class WorkerSubscriberAppService : CollectBusAppService, IWorkerSubscriberAppService, ICapSubscribe, IKafkaSubscribe { private readonly ILogger _logger; private readonly ITcpService _tcpService; @@ -65,6 +66,7 @@ namespace JiShe.CollectBus.Subscribers [HttpPost] [Route("ammeter/oneminute/issued-event")] [KafkaSubscribe(ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName)] + //[CapSubscribe(ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName)] public async Task AmmeterScheduledMeterOneMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) { _logger.LogInformation("1分钟采集电表数据下行消息消费队列开始处理"); @@ -93,6 +95,7 @@ namespace JiShe.CollectBus.Subscribers [HttpPost] [Route("ammeter/fiveminute/issued-event")] [KafkaSubscribe(ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName)] + //[CapSubscribe(ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName)] public async Task AmmeterScheduledMeterFiveMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) { _logger.LogInformation("5分钟采集电表数据下行消息消费队列开始处理"); @@ -121,6 +124,7 @@ namespace JiShe.CollectBus.Subscribers [HttpPost] [Route("ammeter/fifteenminute/issued-event")] [KafkaSubscribe(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName)] + //[CapSubscribe(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName)] public async Task AmmeterScheduledMeterFifteenMinuteReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) { _logger.LogInformation("15分钟采集电表数据下行消息消费队列开始处理"); @@ -160,6 +164,7 @@ namespace JiShe.CollectBus.Subscribers [HttpPost] [Route("watermeter/fifteenminute/issued-event")] [KafkaSubscribe(ProtocolConst.WatermeterSubscriberWorkerAutoReadingIssuedEventName)] + //[CapSubscribe(ProtocolConst.WatermeterSubscriberWorkerAutoReadingIssuedEventName)] public async Task WatermeterSubscriberWorkerAutoReadingIssuedEvent(ScheduledMeterReadingIssuedEventMessage receivedMessage) { _logger.LogInformation("15分钟采集水表数据下行消息消费队列开始处理"); diff --git a/src/JiShe.CollectBus.Host/CollectBusHostModule.cs b/src/JiShe.CollectBus.Host/CollectBusHostModule.cs index 377453e..aabc2ba 100644 --- a/src/JiShe.CollectBus.Host/CollectBusHostModule.cs +++ b/src/JiShe.CollectBus.Host/CollectBusHostModule.cs @@ -43,7 +43,7 @@ namespace JiShe.CollectBus.Host ConfigureNetwork(context, configuration); ConfigureJwtAuthentication(context, configuration); ConfigureHangfire(context); - //ConfigureCap(context, configuration); + ConfigureCap(context, configuration); //ConfigureMassTransit(context, configuration); //ConfigureKafkaTopic(context, configuration); ConfigureAuditLog(context); diff --git a/src/JiShe.CollectBus.KafkaProducer/AdminClient/AdminClientService.cs b/src/JiShe.CollectBus.KafkaProducer/AdminClient/AdminClientService.cs index 26d028e..59e34fa 100644 --- a/src/JiShe.CollectBus.KafkaProducer/AdminClient/AdminClientService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/AdminClient/AdminClientService.cs @@ -183,6 +183,19 @@ namespace JiShe.CollectBus.Kafka.AdminClient return partitions.Any(p => p.PartitionId == targetPartition); } + /// + /// 获取主题的分区数量 + /// + /// + /// + public int GetTopicPartitionsNum(string topic) + { + var metadata = Instance.GetMetadata(topic, TimeSpan.FromSeconds(10)); + if (metadata.Topics.Count == 0) + return 0; + return metadata.Topics[0].Partitions.Count; + } + public void Dispose() { Instance?.Dispose(); diff --git a/src/JiShe.CollectBus.KafkaProducer/AdminClient/IAdminClientService.cs b/src/JiShe.CollectBus.KafkaProducer/AdminClient/IAdminClientService.cs index 238a130..92121c5 100644 --- a/src/JiShe.CollectBus.KafkaProducer/AdminClient/IAdminClientService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/AdminClient/IAdminClientService.cs @@ -52,5 +52,12 @@ namespace JiShe.CollectBus.Kafka.AdminClient /// /// bool CheckPartitionsExist(string topic, int targetPartition); + + /// + /// 获取主题的分区数量 + /// + /// + /// + int GetTopicPartitionsNum(string topic); } } diff --git a/src/JiShe.CollectBus.KafkaProducer/Attributes/KafkaSubscribeAttribute.cs b/src/JiShe.CollectBus.KafkaProducer/Attributes/KafkaSubscribeAttribute.cs index 7a059e0..32f652e 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Attributes/KafkaSubscribeAttribute.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Attributes/KafkaSubscribeAttribute.cs @@ -12,7 +12,7 @@ namespace JiShe.CollectBus.Kafka.Attributes /// /// 订阅的主题 /// - public string[] Topics { get; set; } + public string Topic { get; set; } /// /// 分区 @@ -24,28 +24,20 @@ namespace JiShe.CollectBus.Kafka.Attributes /// public string GroupId { get; set; } - public KafkaSubscribeAttribute(string[] topics, string groupId = "default") - { - this.Topics = topics; - this.GroupId = groupId; - } + /// + /// 任务数(默认是多少个分区多少个任务) + /// + public int TaskCount { get; set; } = -1; public KafkaSubscribeAttribute(string topic, string groupId = "default") { - this.Topics = new string[] { topic }; + this.Topic = topic; this.GroupId = groupId; } - public KafkaSubscribeAttribute(string[] topics, int partition, string groupId = "default") - { - this.Topics = topics; - this.Partition = partition; - this.GroupId = groupId; - } - public KafkaSubscribeAttribute(string topic, int partition, string groupId = "default") { - this.Topics = new string[] { topic }; + this.Topic = topic ; this.Partition = partition; this.GroupId = groupId; } diff --git a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs index 153e5ef..fe0e866 100644 --- a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs +++ b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs @@ -17,9 +17,9 @@ namespace JiShe.CollectBus.Kafka public override void ConfigureServices(ServiceConfigurationContext context) { // 注册Producer - context.Services.AddTransient(); + context.Services.AddSingleton(); // 注册Consumer - context.Services.AddTransient(); + context.Services.AddSingleton(); } public override void OnApplicationInitialization(ApplicationInitializationContext context) diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs index f59385f..9a2142e 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs @@ -9,6 +9,7 @@ using static Confluent.Kafka.ConfigPropertyNames; using System.Collections.Concurrent; using System.Text.RegularExpressions; using NUglify.Html; +using Serilog; namespace JiShe.CollectBus.Kafka.Consumer { @@ -37,6 +38,7 @@ namespace JiShe.CollectBus.Kafka.Consumer { var config = BuildConsumerConfig(groupId); return new ConsumerBuilder(config) + .SetLogHandler((_, log) => _logger.LogInformation($"消费者Log: {log.Message}")) .SetErrorHandler((_, e) => _logger.LogError($"消费者错误: {e.Reason}")) .Build(); } @@ -50,7 +52,9 @@ namespace JiShe.CollectBus.Kafka.Consumer BootstrapServers = _configuration["Kafka:BootstrapServers"], GroupId = groupId ?? "default", AutoOffsetReset = AutoOffsetReset.Earliest, - EnableAutoCommit = false // 禁止AutoCommit + EnableAutoCommit = false, // 禁止AutoCommit + EnablePartitionEof = true, // 启用分区末尾标记 + AllowAutoCreateTopics= true // 启用自动创建 }; if (enableAuth) @@ -119,6 +123,15 @@ namespace JiShe.CollectBus.Kafka.Consumer try { var result = consumer.Consume(cts.Token); + if (result == null) continue; + if (result.Message.Value == null) continue; + if (result.IsPartitionEOF) + { + _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition); + await Task.Delay(TimeSpan.FromSeconds(1)); + continue; + } + bool sucess= await messageHandler(result.Message.Key, result.Message.Value); if (sucess) { @@ -164,6 +177,15 @@ namespace JiShe.CollectBus.Kafka.Consumer try { var result = consumer.Consume(cts.Token); + if (result == null) continue; + if (result.Message == null) continue; + if (result.IsPartitionEOF) + { + _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition); + await Task.Delay(TimeSpan.FromSeconds(1)); + continue; + } + bool sucess = await messageHandler(result.Message.Value); if (sucess) consumer.Commit(result); // 手动提交 diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index d3ce904..cfe5eee 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -2,6 +2,7 @@ using DeviceDetectorNET; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Helpers; +using JiShe.CollectBus.Kafka.AdminClient; using JiShe.CollectBus.Kafka.Attributes; using JiShe.CollectBus.Kafka.Consumer; using Microsoft.AspNetCore.Builder; @@ -41,6 +42,9 @@ namespace JiShe.CollectBus.Kafka lifetime.ApplicationStarted.Register(() => { + var logger = provider.GetRequiredService>(); + int threadCount = 0; + int topicCount = 0; foreach (var subscribeType in subscribeTypes) { var subscribes = provider.GetServices(subscribeType).ToList(); @@ -48,10 +52,13 @@ namespace JiShe.CollectBus.Kafka if(subscribe is IKafkaSubscribe) { - BuildKafkaSubscriber(subscribe, provider); + Tuple tuple= BuildKafkaSubscriber(subscribe, provider, logger); + threadCount+= tuple.Item1; + topicCount+= tuple.Item2; } }); } + logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程"); }); } @@ -60,16 +67,27 @@ namespace JiShe.CollectBus.Kafka /// /// /// - private static void BuildKafkaSubscriber(object subscribe, IServiceProvider provider) + private static Tuple BuildKafkaSubscriber(object subscribe, IServiceProvider provider,ILogger logger) { var subscribedMethods = subscribe.GetType().GetMethods() .Select(m => new { Method = m, Attribute = m.GetCustomAttribute() }) .Where(x => x.Attribute != null) .ToArray(); + + int threadCount = 0; foreach (var sub in subscribedMethods) { - Task.Run(() => StartConsumerAsync(provider, sub.Attribute!, sub.Method, subscribe)); + var adminClientService = provider.GetRequiredService(); + int partitionCount = sub.Attribute!.TaskCount==-1?adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic) : sub.Attribute!.TaskCount; + if (partitionCount <= 0) + partitionCount = 1; + for (int i = 0; i < partitionCount; i++) + { + Task.Run(() => StartConsumerAsync(provider, sub.Attribute!, sub.Method, subscribe, logger)); + threadCount++; + } } + return Tuple.Create(threadCount, subscribedMethods.Length); } /// @@ -80,11 +98,11 @@ namespace JiShe.CollectBus.Kafka /// /// /// - private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr,MethodInfo method, object subscribe) + private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr,MethodInfo method, object subscribe, ILogger logger) { var consumerService = provider.GetRequiredService(); - var logger = provider.GetRequiredService>(); - await consumerService.SubscribeAsync(attr.Topics, async (message) => + + await consumerService.SubscribeAsync(attr.Topic, async (message) => { try { diff --git a/src/JiShe.CollectBus.KafkaProducer/Producer/IProducerService.cs b/src/JiShe.CollectBus.KafkaProducer/Producer/IProducerService.cs index b00f5cf..a401775 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Producer/IProducerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Producer/IProducerService.cs @@ -15,6 +15,6 @@ namespace JiShe.CollectBus.Kafka.Producer Task ProduceAsync(string topic, TKey key, TValue value, int? partition, Action>? deliveryHandler = null) where TKey : notnull where TValue : class; - Task ProduceAsync(string topic, TValue value, int? partition = null, Action>? deliveryHandler = null) where TValue : class; + Task ProduceAsync(string topic, TValue value, int? partition = null, Action>? deliveryHandler = null) where TValue : class; } } diff --git a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs index c322294..8c069a9 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs @@ -9,6 +9,7 @@ using JiShe.CollectBus.Kafka.Consumer; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Volo.Abp.DependencyInjection; +using YamlDotNet.Serialization; namespace JiShe.CollectBus.Kafka.Producer { @@ -62,6 +63,7 @@ namespace JiShe.CollectBus.Kafka.Producer LingerMs = 20, // 修改等待时间为20ms Acks = Acks.All, // 表明只有所有副本Broker都收到消息才算提交成功, 可以 Acks.Leader MessageSendMaxRetries = 50, // 消息发送失败最大重试50次 + MessageTimeoutMs = 120000, // 消息发送超时时间为2分钟,设置值MessageTimeoutMs > LingerMs }; if (enableAuth) @@ -114,8 +116,8 @@ namespace JiShe.CollectBus.Kafka.Producer /// public async Task ProduceAsync(string topic, TValue value) where TValue : class { - var producer = GetProducer(); - await producer.ProduceAsync(topic, new Message { Value = value }); + var producer = GetProducer(); + await producer.ProduceAsync(topic, new Message { Value = value }); } /// @@ -160,13 +162,13 @@ namespace JiShe.CollectBus.Kafka.Producer /// /// /// - public async Task ProduceAsync(string topic, TValue value, int? partition=null, Action>? deliveryHandler = null) where TValue : class + public async Task ProduceAsync(string topic, TValue value, int? partition=null, Action>? deliveryHandler = null) where TValue : class { - var message = new Message + var message = new Message { Value = value }; - var producer = GetProducer(); + var producer = GetProducer(); if (partition.HasValue) { var topicPartition = new TopicPartition(topic, partition.Value); diff --git a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs index d066a28..963d89b 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs +++ b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs @@ -13,11 +13,13 @@ using JiShe.CollectBus.IotSystems.Protocols; using MassTransit; using JiShe.CollectBus.Kafka.Producer; using JiShe.CollectBus.Common.Helpers; +using DotNetCore.CAP; namespace JiShe.CollectBus.Protocol.Contracts.Abstracts { public abstract class BaseProtocolPlugin : IProtocolPlugin { + private readonly ICapPublisher _producerBus; private readonly IProducerService _producerService; private readonly ILogger _logger; private readonly IRepository _protocolInfoRepository; @@ -38,7 +40,8 @@ namespace JiShe.CollectBus.Protocol.Contracts.Abstracts _logger = serviceProvider.GetRequiredService>(); _protocolInfoRepository = serviceProvider.GetRequiredService>(); - _producerService = serviceProvider.GetRequiredService(); + _producerService = serviceProvider.GetRequiredService(); + _producerBus = serviceProvider.GetRequiredService(); } public abstract ProtocolInfo Info { get; } @@ -87,6 +90,7 @@ namespace JiShe.CollectBus.Protocol.Contracts.Abstracts Fn = 1 }; var bytes = Build3761SendData.BuildSendCommandBytes(reqParam); + //await _producerBus.PublishAsync(ProtocolConst.SubscriberLoginIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }); await _producerService.ProduceAsync(ProtocolConst.SubscriberLoginIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }.Serialize()); //await _producerBus.Publish(new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }); @@ -127,6 +131,8 @@ namespace JiShe.CollectBus.Protocol.Contracts.Abstracts Fn = 1 }; var bytes = Build3761SendData.BuildSendCommandBytes(reqParam); + //await _producerBus.PublishAsync(ProtocolConst.SubscriberHeartbeatIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }); + await _producerService.ProduceAsync(ProtocolConst.SubscriberHeartbeatIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }.Serialize()); //await _producerBus.Publish(new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }); diff --git a/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj b/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj index 248ee30..2721676 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj +++ b/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj @@ -8,6 +8,7 @@ + From 1a275ec9c3ca87effa7ad64d0c640e5428f396eb Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Wed, 16 Apr 2025 17:36:46 +0800 Subject: [PATCH 09/27] =?UTF-8?q?=E6=89=B9=E9=87=8F=E5=86=99=E5=85=A5?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E8=B0=83=E4=BC=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RedisDataCache/IRedisDataCacheService.cs | 176 ++++ .../CollectBusAppService.cs | 4 +- .../RedisDataCache/RedisDataCacheService.cs | 745 ++++++++++++++ .../Samples/SampleAppService.cs | 57 +- .../BasicScheduledMeterReadingService.cs | 90 +- ...nergySystemScheduledMeterReadingService.cs | 16 +- .../Consts/RedisConst.cs | 29 +- .../Models/BusCacheGlobalPagedResult.cs | 16 + .../Models/DeviceCacheBasicModel.cs | 7 +- .../FreeRedisProvider.cs | 957 ++++++++---------- .../IFreeRedisProvider.cs | 100 +- .../Options/FreeRedisOptions.cs | 5 + src/JiShe.CollectBus.Host/appsettings.json | 3 +- 13 files changed, 1491 insertions(+), 714 deletions(-) create mode 100644 src/JiShe.CollectBus.Application.Contracts/RedisDataCache/IRedisDataCacheService.cs create mode 100644 src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs diff --git a/src/JiShe.CollectBus.Application.Contracts/RedisDataCache/IRedisDataCacheService.cs b/src/JiShe.CollectBus.Application.Contracts/RedisDataCache/IRedisDataCacheService.cs new file mode 100644 index 0000000..c6f3613 --- /dev/null +++ b/src/JiShe.CollectBus.Application.Contracts/RedisDataCache/IRedisDataCacheService.cs @@ -0,0 +1,176 @@ +using JiShe.CollectBus.Common.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JiShe.CollectBus.Application.Contracts +{ + /// + /// 数据缓存服务接口 + /// + public interface IRedisDataCacheService + { + /// + /// 单个添加数据 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// ZSET索引缓存Key + /// 待缓存数据 + /// + Task InsertDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string redisZSetScoresIndexCacheKey, + T data) where T : DeviceCacheBasicModel; + + /// + /// 批量添加数据 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// ZSET索引缓存Key + /// 待缓存数据集合 + /// + Task BatchInsertDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string redisZSetScoresIndexCacheKey, + IEnumerable items) where T : DeviceCacheBasicModel; + + /// + /// 删除缓存信息 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// ZSET索引缓存Key + /// 已缓存数据 + /// + Task RemoveCacheDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string redisZSetScoresIndexCacheKey, + T data) where T : DeviceCacheBasicModel; + + /// + /// 修改缓存信息,映射关系未发生改变 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// ZSET索引缓存Key + /// 待修改缓存数据 + /// + Task ModifyDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string redisZSetScoresIndexCacheKey, + T newData) where T : DeviceCacheBasicModel; + + + /// + /// 修改缓存信息,映射关系已改变 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// 旧的映射关系 + /// ZSET索引缓存Key + /// 待修改缓存数据 + /// + Task ModifyDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string oldMemberId, + string redisZSetScoresIndexCacheKey, + T newData) where T : DeviceCacheBasicModel; + + ///// + ///// 通过集中器与表计信息排序索引获取数据 + ///// + ///// + ///// 主数据存储Hash缓存Key + ///// ZSET索引缓存Key + ///// 分页尺寸 + ///// 最后一个索引 + ///// 最后一个唯一标识 + ///// 排序方式 + ///// + //Task> GetPagedData( + //string redisHashCacheKey, + //string redisZSetScoresIndexCacheKey, + //IEnumerable focusIds, + //int pageSize = 10, + //decimal? lastScore = null, + //string lastMember = null, + //bool descending = true) + //where T : DeviceCacheBasicModel; + + + /// + /// 通过ZSET索引获取数据 + /// + /// + /// 主数据存储Hash缓存Key + /// ZSET索引缓存Key + /// 分页尺寸 + /// 最后一个索引 + /// 最后一个唯一标识 + /// 排序方式 + /// + Task> GetAllPagedData( + string redisHashCacheKey, + string redisZSetScoresIndexCacheKey, + int pageSize = 1000, + decimal? lastScore = null, + string lastMember = null, + bool descending = true) + where T : DeviceCacheBasicModel; + + + ///// + ///// 游标分页查询 + ///// + ///// 排序索引ZSET缓存Key + ///// 分页数量 + ///// 开始索引 + ///// 开始唯一标识 + ///// 排序方式 + ///// + //Task<(List Members, bool HasNext)> GetPagedMembers( + // string redisZSetScoresIndexCacheKey, + // int pageSize, + // decimal? startScore, + // string excludeMember, + // bool descending); + + ///// + ///// 批量获取指定分页的数据 + ///// + ///// + ///// Hash表缓存key + ///// Hash表字段集合 + ///// + //Task> BatchGetData( + // string redisHashCacheKey, + // IEnumerable members) + // where T : DeviceCacheBasicModel; + + ///// + ///// 获取下一页游标 + ///// + ///// 排序索引ZSET缓存Key + ///// 最后一个唯一标识 + ///// 排序方式 + ///// + //Task GetNextScore( + // string redisZSetScoresIndexCacheKey, + // string lastMember, + // bool descending); + } +} diff --git a/src/JiShe.CollectBus.Application/CollectBusAppService.cs b/src/JiShe.CollectBus.Application/CollectBusAppService.cs index 6ecc509..ad5b585 100644 --- a/src/JiShe.CollectBus.Application/CollectBusAppService.cs +++ b/src/JiShe.CollectBus.Application/CollectBusAppService.cs @@ -79,7 +79,7 @@ public abstract class CollectBusAppService : ApplicationService { string key = (string)item[0]; object[] fieldsAndValues = (object[])item[1]; - var redisCacheKey = $"{string.Format(RedisConst.CacheMeterInfoKey, systemType, serverTagName, meterType, timeDensity)}"; + var redisCacheKey = $"{string.Format(RedisConst.CacheMeterInfoHashKey, systemType, serverTagName, meterType, timeDensity)}"; string focusAddress = key.Replace(redisCacheKey, ""); var meterHashs = new Dictionary(); @@ -182,7 +182,7 @@ public abstract class CollectBusAppService : ApplicationService string key = (string)item[0]; object[] fieldsAndValues = (object[])item[1]; var redisCacheKey = string.Format( - RedisConst.CacheMeterInfoKey, + RedisConst.CacheMeterInfoHashKey, systemType, serverTagName, meterType, diff --git a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs new file mode 100644 index 0000000..4d78706 --- /dev/null +++ b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs @@ -0,0 +1,745 @@ +using Confluent.Kafka; +using FreeRedis; +using JiShe.CollectBus.Application.Contracts; +using JiShe.CollectBus.Common.Extensions; +using JiShe.CollectBus.Common.Helpers; +using JiShe.CollectBus.Common.Models; +using JiShe.CollectBus.FreeRedisProvider; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Volo.Abp.DependencyInjection; + +namespace JiShe.CollectBus.RedisDataCache +{ + /// + /// 数据缓存服务接口 + /// + public class RedisDataCacheService : IRedisDataCacheService, ITransientDependency + { + private readonly IFreeRedisProvider _freeRedisProvider; + private readonly ILogger _logger; + private RedisClient Instance { get; set; } + + /// + /// 数据缓存服务接口 + /// + /// + /// + public RedisDataCacheService(IFreeRedisProvider freeRedisProvider, + ILogger logger) + { + this._freeRedisProvider = freeRedisProvider; + this._logger = logger; + + Instance = _freeRedisProvider.Instance; + } + + /// + /// 单个添加数据 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// ZSET索引缓存Key + /// 待缓存数据 + /// + public async Task InsertDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string redisZSetScoresIndexCacheKey, + T data) where T : DeviceCacheBasicModel + { + // 参数校验增强 + if (data == null || string.IsNullOrWhiteSpace(redisHashCacheKey) + || string.IsNullOrWhiteSpace(redisSetIndexCacheKey) + || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + { + _logger.LogError($"{nameof(InsertDataAsync)} 参数异常，-101"); + return; + } + + // 使用事务保证原子性 + using (var trans = Instance.Multi()) + { + // 主数据存储Hash + trans.HSet(redisHashCacheKey, data.MemberID, data.Serialize()); + + // 集中器号分组索引Set缓存 + trans.SAdd(redisSetIndexCacheKey, data.MemberID); + + // 集中器与表计信息排序索引ZSET缓存Key + trans.ZAdd(redisZSetScoresIndexCacheKey, data.ScoreValue, data.MemberID); + + var results = trans.Exec(); + + if (results == null || results.Length <= 0) + { + _logger.LogError($"{nameof(InsertDataAsync)} 添加事务提交失败，-102"); + } + } + + await Task.CompletedTask; + } + + /// + /// 批量添加数据 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// ZSET索引缓存Key + /// 待缓存数据集合 + /// + public async Task BatchInsertDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string redisZSetScoresIndexCacheKey, + IEnumerable items) where T : DeviceCacheBasicModel + { + if (items == null + || items.Count() <= 0 + || string.IsNullOrWhiteSpace(redisHashCacheKey) + || string.IsNullOrWhiteSpace(redisSetIndexCacheKey) + || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + { + _logger.LogError($"{nameof(BatchInsertDataAsync)} 参数异常，-101"); + return; + } + + const int BATCH_SIZE = 1000; // 每批1000条 + var semaphore = new SemaphoreSlim(Environment.ProcessorCount * 2); + + foreach (var batch in items.Batch(BATCH_SIZE)) + { + await semaphore.WaitAsync(); + + _ = Task.Run(() => + { + using (var pipe = Instance.StartPipe()) + { + foreach (var item in batch) + { + // 主数据存储Hash + pipe.HSet(redisHashCacheKey, item.MemberID, item.Serialize()); + + // Set索引缓存 + pipe.SAdd(redisSetIndexCacheKey, item.MemberID); + + // ZSET索引缓存Key + pipe.ZAdd(redisZSetScoresIndexCacheKey, item.ScoreValue, item.MemberID); + } + pipe.EndPipe(); + } + semaphore.Release(); + }); + } + + await Task.CompletedTask; + } + + /// + /// 删除缓存信息 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// ZSET索引缓存Key + /// 已缓存数据 + /// + public async Task RemoveCacheDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string redisZSetScoresIndexCacheKey, + T data) where T : DeviceCacheBasicModel + { + if (data == null + || string.IsNullOrWhiteSpace(redisHashCacheKey) + || string.IsNullOrWhiteSpace(redisSetIndexCacheKey) + || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + { + _logger.LogError($"{nameof(RemoveCacheDataAsync)} 参数异常，-101"); + return; + } + + const string luaScript = @" + local hashCacheKey = KEYS[1] + local setIndexCacheKey = KEYS[2] + local zsetScoresIndexCacheKey = KEYS[3] + local member = ARGV[1] + + local deleted = 0 + if redis.call('HDEL', hashCacheKey, member) > 0 then + deleted = 1 + end + + redis.call('SREM', setIndexCacheKey, member) + redis.call('ZREM', zsetScoresIndexCacheKey, member) + return deleted + "; + + var keys = new[] + { + redisHashCacheKey, + redisSetIndexCacheKey, + redisZSetScoresIndexCacheKey + }; + + var result = await Instance.EvalAsync(luaScript, keys, new[] { data.MemberID }); + + if ((int)result == 0) + { + _logger.LogError($"{nameof(RemoveCacheDataAsync)} 删除指定Key{redisHashCacheKey}的{data.MemberID}数据失败,-102"); + } + } + + /// + /// 修改缓存信息,映射关系未发生改变 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// ZSET索引缓存Key + /// 待修改缓存数据 + /// + public async Task ModifyDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string redisZSetScoresIndexCacheKey, + T newData) where T : DeviceCacheBasicModel + { + if (newData == null + || string.IsNullOrWhiteSpace(redisHashCacheKey) + || string.IsNullOrWhiteSpace(redisSetIndexCacheKey) + || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + { + _logger.LogError($"{nameof(ModifyDataAsync)} 参数异常，-101"); + return; + } + + var luaScript = @" + local hashCacheKey = KEYS[1] + local member = ARGV[1] + local newData = ARGV[2] + + -- 校验存在性 + if redis.call('HEXISTS', hashCacheKey, member) == 0 then + return 0 + end + + -- 更新主数据 + redis.call('HSET', hashCacheKey, member, newData) + + return 1 + "; + + + var result = await Instance.EvalAsync(luaScript, + new[] + { + redisHashCacheKey + }, + new object[] + { + newData.MemberID, + newData.Serialize() + }); + + if ((int)result == 0) + { + _logger.LogError($"{nameof(ModifyDataAsync)} 更新指定Key{redisHashCacheKey}的{newData.MemberID}数据失败,-102"); + } + } + + /// + /// 修改缓存信息,映射关系已经改变 + /// + /// + /// 主数据存储Hash缓存Key + /// Set索引缓存Key + /// 旧的映射关系 + /// ZSET索引缓存Key + /// 待修改缓存数据 + /// + public async Task ModifyDataAsync( + string redisHashCacheKey, + string redisSetIndexCacheKey, + string oldMemberId, + string redisZSetScoresIndexCacheKey, + T newData) where T : DeviceCacheBasicModel + { + if (newData == null + || string.IsNullOrWhiteSpace(redisHashCacheKey) + || string.IsNullOrWhiteSpace(redisSetIndexCacheKey) + || string.IsNullOrWhiteSpace(oldMemberId) + || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + { + _logger.LogError($"{nameof(ModifyDataAsync)} 参数异常，-101"); + return; + } + + var luaScript = @" + local hashCacheKey = KEYS[1] + local setIndexCacheKey = KEYS[2] + local zsetScoresIndexCacheKey = KEYS[3] + local member = ARGV[1] + local oldMember = ARGV[2] + local newData = ARGV[3] + local newScore = ARGV[4] + + -- 校验存在性 + if redis.call('HEXISTS', hashCacheKey, oldMember) == 0 then + return 0 + end + + -- 删除旧数据 + redis.call('HDEL', hashCacheKey, oldMember) + + -- 插入新主数据 + redis.call('HSET', hashCacheKey, member, newData) + + -- 处理变更 + if newScore ~= '' then + -- 删除旧索引 + redis.call('SREM', setIndexCacheKey, oldMember) + redis.call('ZREM', zsetScoresIndexCacheKey, oldMember) + + -- 添加新索引 + redis.call('SADD', setIndexCacheKey, member) + redis.call('ZADD', zsetScoresIndexCacheKey, newScore, member) + end + + return 1 + "; + + var result = await Instance.EvalAsync(luaScript, + new[] + { + redisHashCacheKey, + redisSetIndexCacheKey, + redisZSetScoresIndexCacheKey + }, + new object[] + { + newData.MemberID, + oldMemberId, + newData.Serialize(), + newData.ScoreValue.ToString() ?? "", + }); + + if ((int)result == 0) + { + _logger.LogError($"{nameof(ModifyDataAsync)} 更新指定Key{redisHashCacheKey}的{newData.MemberID}数据失败,-102"); + } + } + + + /// + /// 通过集中器与表计信息排序索引获取指定集中器号集合数据 + /// + /// + /// 主数据存储Hash缓存Key + /// 集中器与表计信息排序索引ZSET缓存Key + /// 集中器Id + /// 分页尺寸 + /// 最后一个索引 + /// 最后一个唯一标识 + /// 排序方式 + /// + public async Task> GetPagedData( + string redisCacheKey, + string redisCacheFocusScoresIndexKey, + IEnumerable focusIds, + int pageSize = 10, + decimal? lastScore = null, + string lastMember = null, + bool descending = true) + where T : DeviceCacheBasicModel + { + throw new Exception(); + } + + /// + /// 通过ZSET索引获取数据 + /// + /// + /// 主数据存储Hash缓存Key + /// ZSET索引缓存Key + /// 分页尺寸 + /// 最后一个索引 + /// 最后一个唯一标识 + /// 排序方式 + /// + public async Task> GetAllPagedData( + string redisHashCacheKey, + string redisZSetScoresIndexCacheKey, + int pageSize = 1000, + decimal? lastScore = null, + string lastMember = null, + bool descending = true) + where T : DeviceCacheBasicModel + { + // 参数校验（保持不变） + if (string.IsNullOrWhiteSpace(redisHashCacheKey) || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + { + _logger.LogError($"{nameof(GetAllPagedData)} 参数异常，-101"); + return null; + } + + if (pageSize < 1 || pageSize > 10000) + { + _logger.LogError($"{nameof(GetAllPagedData)} 分页大小应在1-10000之间，-102"); + return null; + } + + var luaScript = @" + local command = ARGV[1] + local range_start = ARGV[2] + local range_end = ARGV[3] + local limit = tonumber(ARGV[4]) + local last_score = ARGV[5] + local last_member = ARGV[6] + + -- 处理相同分数下的字典序分页 + if last_score ~= '' and last_member ~= '' then + if command == 'ZRANGEBYSCORE' then + range_start = '(' .. last_score + range_end = '(' .. last_score .. ' ' .. last_member + else + range_start = '(' .. last_score .. ' ' .. last_member + range_end = '(' .. last_score + end + end + + -- 执行范围查询 + local members + if command == 'ZRANGEBYSCORE' then + members = redis.call(command, KEYS[1], range_start, range_end, + 'WITHSCORES', 'LIMIT', 0, limit) + else + members = redis.call(command, KEYS[1], range_end, range_start, + 'WITHSCORES', 'LIMIT', 0, limit) + end + + -- 提取成员和分数 + local result_members = {} + local result_scores = {} + for i = 1, #members, 2 do + table.insert(result_members, members[i]) + table.insert(result_scores, members[i+1]) + end + + -- 获取Hash数据 + local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members)) + + return { + #result_members, + result_members, + result_scores, + hash_data + }"; + + //正确设置范围参数 + string rangeStart, rangeEnd; + if (descending) + { + rangeStart = lastScore.HasValue ? $"({lastScore}" : "+inf"; + rangeEnd = "-inf"; // 降序时固定为最小值 + } + else + { + rangeStart = lastScore.HasValue ? $"({lastScore}" : "-inf"; + rangeEnd = "+inf"; // 升序时固定为最大值 + } + + var result = (object[])await Instance.EvalAsync( + luaScript, + new[] { redisZSetScoresIndexCacheKey, redisHashCacheKey }, + new object[] + { + descending ? "ZREVRANGEBYSCORE" : "ZRANGEBYSCORE", + rangeStart, + rangeEnd, + (pageSize + 1).ToString(), // 多取1条用于判断hasNext + lastScore?.ToString() ?? "", + lastMember ?? "" + }); + + if ((long)result[0] == 0) + return new BusCacheGlobalPagedResult { Items = new List() }; + + // 处理结果集 + var members = ((object[])result[1]).Cast().ToList(); + var scores = ((object[])result[2]).Cast().Select(decimal.Parse).ToList(); + var hashData = ((object[])result[3]).Cast().ToList(); + + //合并有效数据并处理游标 + var validItems = members.Zip(hashData, (m, h) => + !string.IsNullOrWhiteSpace(h) ? BusJsonSerializer.Deserialize(h) : null) + .Where(x => x != null) + .Take(pageSize + 1) + .ToList(); + + var hasNext = validItems.Count > pageSize; + var actualItems = hasNext ? validItems.Take(pageSize) : validItems; + + // 计算下一页起始点 + string nextMember = null; + decimal? nextScore = null; + if (hasNext) + { + // 获取实际返回的最后一条有效数据 + var lastValidIndex = actualItems.Count() - 1; + nextMember = members[lastValidIndex]; + nextScore = scores[lastValidIndex]; + } + + return new BusCacheGlobalPagedResult + { + Items = actualItems.ToList(), + HasNext = hasNext, + NextScore = nextScore, + NextMember = nextMember, + TotalCount = await GetTotalCount(redisZSetScoresIndexCacheKey), + PageSize = pageSize, + }; + } + + ///// + ///// 通过集中器与表计信息排序索引获取数据 + ///// + ///// + ///// 主数据存储Hash缓存Key + ///// ZSET索引缓存Key + ///// 分页尺寸 + ///// 最后一个索引 + ///// 最后一个唯一标识 + ///// 排序方式 + ///// + //public async Task> GetAllPagedData( + //string redisHashCacheKey, + //string redisZSetScoresIndexCacheKey, + //int pageSize = 1000, + //decimal? lastScore = null, + //string lastMember = null, + //bool descending = true) + //where T : DeviceCacheBasicModel + //{ + // // 参数校验增强 + // if (string.IsNullOrWhiteSpace(redisHashCacheKey) || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + // { + // _logger.LogError($"{nameof(GetAllPagedData)} 参数异常，-101"); + // return null; + // } + + // if (pageSize < 1 || pageSize > 10000) + // { + // _logger.LogError($"{nameof(GetAllPagedData)} 分页大小应在1-10000之间，-102"); + // return null; + // } + + // //// 分页参数解析 + // //var (startScore, excludeMember) = descending + // // ? (lastScore ?? decimal.MaxValue, lastMember) + // // : (lastScore ?? 0, lastMember); + + // //执行分页查询（整合游标处理） + // var pageResult = await GetPagedMembers( + // redisZSetScoresIndexCacheKey, + // pageSize, + // lastScore, + // lastMember, + // descending); + + // // 批量获取数据（优化内存分配） + // var dataDict = await BatchGetData(redisHashCacheKey, pageResult.Members); + + // return new BusCacheGlobalPagedResult + // { + // Items = pageResult.Members.Select(m => dataDict.TryGetValue(m, out var v) ? v : default) + // .Where(x => x != null).ToList(), + // HasNext = pageResult.HasNext, + // NextScore = pageResult.NextScore, + // NextMember = pageResult.NextMember, + // TotalCount = await GetTotalCount(redisZSetScoresIndexCacheKey), + // PageSize = pageSize, + // }; + //} + + /// + /// 游标分页查询 + /// + /// 排序索引ZSET缓存Key + /// 分页数量 + /// 上一个索引 + /// 上一个标识 + /// 排序方式 + /// + private async Task<(List Members, bool HasNext, decimal? NextScore, string NextMember)> GetPagedMembers( + string redisZSetScoresIndexCacheKey, + int pageSize, + decimal? lastScore, + string lastMember, + bool descending) + { + // 根据排序方向初始化参数 + long initialScore = descending ? long.MaxValue : 0; + decimal? currentScore = lastScore ?? initialScore; + string currentMember = lastMember; + var members = new List(pageSize + 1); + + // 使用游标分页查询 + while (members.Count < pageSize + 1 && currentScore.HasValue) + { + var (batch, hasMore) = await GetNextBatch( + redisZSetScoresIndexCacheKey, + pageSize + 1 - members.Count, + currentScore.Value, + currentMember, + descending); + + if (!batch.Any()) break; + + members.AddRange(batch); + + // 更新游标 + currentMember = batch.LastOrDefault(); + currentScore = await GetNextScore(redisZSetScoresIndexCacheKey, currentMember, descending); + } + + // 处理分页结果 + bool hasNext = members.Count > pageSize; + var resultMembers = members.Take(pageSize).ToList(); + + return ( + resultMembers, + hasNext, + currentScore, + currentMember + ); + } + + /// + /// 批量获取指定分页的数据 + /// + /// + /// Hash表缓存key + /// Hash表字段集合 + /// + private async Task> BatchGetData( + string redisHashCacheKey, + IEnumerable members) + where T : DeviceCacheBasicModel + { + using var pipe = Instance.StartPipe(); + + foreach (var member in members) + { + pipe.HGet(redisHashCacheKey, member); + } + + var results = pipe.EndPipe(); + return await Task.FromResult(members.Zip(results, (k, v) => new { k, v }) + .ToDictionary(x => x.k, x => (T)x.v)); + } + + /// + /// 处理下一个分页数据 + /// + /// + /// + /// + /// + /// + /// + private async Task<(string[] Batch, bool HasMore)> GetNextBatch( + string zsetKey, + int limit, + decimal score, + string excludeMember, + bool descending) + { + var query = descending + ? await Instance.ZRevRangeByScoreAsync( + zsetKey, + max: score, + min: 0, + offset: 0, + count: limit) + : await Instance.ZRangeByScoreAsync( + zsetKey, + min: score, + max: long.MaxValue, + offset: 0, + count: limit); + + return (query, query.Length >= limit); + } + + /// + /// 获取下一页游标 + /// + /// 排序索引ZSET缓存Key + /// 最后一个唯一标识 + /// 排序方式 + /// + private async Task GetNextScore( + string redisZSetScoresIndexCacheKey, + string lastMember, + bool descending) + { + if (string.IsNullOrEmpty(lastMember)) return null; + + var score = await Instance.ZScoreAsync(redisZSetScoresIndexCacheKey, lastMember); + if (!score.HasValue) return null; + + // 根据排序方向调整score + return descending + ? score.Value - 1 // 降序时下页查询小于当前score + : score.Value + 1; // 升序时下页查询大于当前score + } + + /// + /// 获取指定ZSET区间内的总数量 + /// + /// + /// + /// + /// + public async Task GetCount(string zsetKey, long min, long max) + { + // 缓存计数优化 + var cacheKey = $"{zsetKey}_count_{min}_{max}"; + var cached = await Instance.GetAsync(cacheKey); + + if (cached.HasValue) + return cached.Value; + + var count = await Instance.ZCountAsync(zsetKey, min, max); + await Instance.SetExAsync(cacheKey, 60, count); // 缓存60秒 + return count; + } + + /// + /// 获取指定ZSET的总数量 + /// + /// + /// + private async Task GetTotalCount(string redisZSetScoresIndexCacheKey) + { + // 缓存计数优化 + var cacheKey = $"{redisZSetScoresIndexCacheKey}_total_count"; + var cached = await Instance.GetAsync(cacheKey); + + if (cached.HasValue) + return cached.Value; + + var count = await Instance.ZCountAsync(redisZSetScoresIndexCacheKey, 0, decimal.MaxValue); + await Instance.SetExAsync(cacheKey, 30, count); // 缓存30秒 + return count; + } + } +} diff --git a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs index e4c078d..3af4b6a 100644 --- a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs +++ b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs @@ -23,6 +23,9 @@ using JiShe.CollectBus.Common.DeviceBalanceControl; using JiShe.CollectBus.Kafka.Attributes; using System.Text.Json; using JiShe.CollectBus.Kafka; +using JiShe.CollectBus.Application.Contracts; +using JiShe.CollectBus.Common.Models; +using System.Diagnostics; namespace JiShe.CollectBus.Samples; @@ -32,17 +35,23 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS private readonly IIoTDBProvider _iotDBProvider; private readonly IoTDBRuntimeContext _dbContext; private readonly IoTDBOptions _options; + private readonly IRedisDataCacheService _redisDataCacheService; public SampleAppService(IIoTDBProvider iotDBProvider, IOptions options, - IoTDBRuntimeContext dbContext, ILogger logger) + IoTDBRuntimeContext dbContext, ILogger logger, IRedisDataCacheService redisDataCacheService) { _iotDBProvider = iotDBProvider; _options = options.Value; _dbContext = dbContext; _logger = logger; + _redisDataCacheService = redisDataCacheService; } - + /// + /// 测试 UseSessionPool + /// + /// + /// [HttpGet] public async Task UseSessionPool(long timestamps) { @@ -72,7 +81,10 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS await _iotDBProvider.InsertAsync(meter); } - + /// + /// 测试Session切换 + /// + /// [HttpGet] public async Task UseTableSessionPool() { @@ -125,7 +137,7 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS var timeDensity = "15"; //获取缓存中的电表信息 - var redisKeyList = $"{string.Format(RedisConst.CacheMeterInfoKey, "Energy", "JiSheCollectBus", MeterTypeEnum.Ammeter.ToString(), timeDensity)}*"; + var redisKeyList = $"{string.Format(RedisConst.CacheMeterInfoHashKey, "Energy", "JiSheCollectBus", MeterTypeEnum.Ammeter.ToString(), timeDensity)}*"; var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); var meterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, "Energy", "JiSheCollectBus", timeDensity, MeterTypeEnum.Ammeter); @@ -178,6 +190,43 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS await _iotDBProvider.InsertAsync(meter); } + /// + /// 测试单个测点数据项 + /// + /// + [HttpGet] + public async Task TestRedisCacheGetAllPagedData() + { + var timeDensity = "15"; + string SystemType = ""; + string ServerTagName = "JiSheCollectBus2"; + var redisCacheMeterInfoHashKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}"; + var redisCacheMeterInfoSetIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}"; + var redisCacheMeterInfoZSetScoresIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}"; + var timer = Stopwatch.StartNew(); + + decimal? cursor = null; + string member = null; + bool hasNext; + List meterInfos = new List(); + do + { + var page = await _redisDataCacheService + .GetAllPagedData( + redisCacheMeterInfoHashKeyTemp, + redisCacheMeterInfoZSetScoresIndexKeyTemp); + + meterInfos.AddRange(page.Items); + cursor = page.NextScore; + member = page.NextMember; + hasNext = page.HasNext; + } while (hasNext); + + timer.Stop(); + + _logger.LogInformation($"{nameof(TestRedisCacheGetAllPagedData)} 获取电表缓存数据完成，耗时{timer.ElapsedMilliseconds}毫秒"); + } + public Task GetAsync() { diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index 2fd9781..ce4635b 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -1,11 +1,13 @@ using DotNetCore.CAP; using JiShe.CollectBus.Ammeters; +using JiShe.CollectBus.Application.Contracts; using JiShe.CollectBus.Common.BuildSendDatas; using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.DeviceBalanceControl; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; using JiShe.CollectBus.Common.Helpers; +using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.GatherItem; using JiShe.CollectBus.IoTDBProvider; using JiShe.CollectBus.IotSystems.MessageIssueds; @@ -34,13 +36,14 @@ namespace JiShe.CollectBus.ScheduledMeterReading private readonly IIoTDBProvider _dbProvider; private readonly IMeterReadingRecordRepository _meterReadingRecordRepository; private readonly IProducerService _producerService; - + private readonly IRedisDataCacheService _redisDataCacheService; public BasicScheduledMeterReadingService( ILogger logger, ICapPublisher producerBus, IMeterReadingRecordRepository meterReadingRecordRepository, IProducerService producerService, + IRedisDataCacheService redisDataCacheService, IIoTDBProvider dbProvider) { _producerBus = producerBus; @@ -48,6 +51,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading _dbProvider = dbProvider; _meterReadingRecordRepository = meterReadingRecordRepository; _producerService = producerService; + _redisDataCacheService = redisDataCacheService; } /// @@ -121,7 +125,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading //获取缓存中的表信息 - var redisKeyList = $"{string.Format(RedisConst.CacheMeterInfoKey, SystemType, ServerTagName, meteryType, timeDensity)}*"; + var redisKeyList = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, ServerTagName, meteryType, timeDensity)}*"; var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); if (oneMinutekeyList == null || oneMinutekeyList.Length <= 0) { @@ -209,25 +213,51 @@ namespace JiShe.CollectBus.ScheduledMeterReading public virtual async Task InitAmmeterCacheData(string gatherCode = "") { #if DEBUG - var timeDensity = "15"; - string tempCacheMeterInfoKey = $"CollectBus:{"{0}:{1}"}:MeterInfo:{"{2}"}:{"{3}"}"; - //获取缓存中的电表信息 - var redisKeyList = $"{string.Format(tempCacheMeterInfoKey, SystemType, "JiSheCollectBus", MeterTypeEnum.Ammeter, timeDensity)}*"; + //var timeDensity = "15"; + //string tempCacheMeterInfoKey = $"CollectBus:{"{0}:{1}"}:MeterInfo:{"{2}"}:{"{3}"}"; + ////获取缓存中的电表信息 + //var redisKeyList = $"{string.Format(tempCacheMeterInfoKey, SystemType, "JiSheCollectBus", MeterTypeEnum.Ammeter, timeDensity)}*"; - var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); - var tempMeterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); - //List focusAddressDataLista = new List(); - List meterInfos = new List(); - foreach (var item in tempMeterInfos) - { - var tempData = item.Adapt(); - tempData.FocusId = item.FocusID; - tempData.MeterId = item.Id; - meterInfos.Add(tempData); - //focusAddressDataLista.Add(item.FocusAddress); - } + //var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); + //var tempMeterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); + ////List focusAddressDataLista = new List(); + //List meterInfos = new List(); + //foreach (var item in tempMeterInfos) + //{ + // var tempData = item.Adapt(); + // tempData.FocusId = item.FocusID; + // tempData.MeterId = item.Id; + // meterInfos.Add(tempData); + // //focusAddressDataLista.Add(item.FocusAddress); + //} //DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); + + var timeDensity = "15"; + var redisCacheMeterInfoHashKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + var redisCacheMeterInfoSetIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + var redisCacheMeterInfoZSetScoresIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + + + decimal? cursor = null; + string member = null; + bool hasNext; + List meterInfos = new List(); + do + { + var page = await _redisDataCacheService.GetAllPagedData( + redisCacheMeterInfoHashKeyTemp, + redisCacheMeterInfoZSetScoresIndexKeyTemp, + pageSize: 1000, + lastScore: cursor, + lastMember: member); + + meterInfos.AddRange(page.Items); + cursor = page.HasNext ? page.NextScore : null; + member = page.HasNext ? page.NextMember : null; + hasNext = page.HasNext; + } while (hasNext); + #else var meterInfos = await GetAmmeterInfoList(gatherCode); #endif @@ -251,10 +281,9 @@ namespace JiShe.CollectBus.ScheduledMeterReading var meterInfoGroupByTimeDensity = meterInfos.GroupBy(d => d.TimeDensity); foreach (var itemTimeDensity in meterInfoGroupByTimeDensity) { - var redisCacheKey = $"{string.Format(RedisConst.CacheMeterInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; - var redisCacheFocusIndexKey = $"{string.Format(RedisConst.CacheMeterInfoFocusIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; - var redisCacheScoresIndexKey = $"{string.Format(RedisConst.CacheMeterInfoScoresIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; - var redisCacheGlobalIndexKey = $"{string.Format(RedisConst.CacheMeterInfoGlobalIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; + var redisCacheMeterInfoHashKey = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; + var redisCacheMeterInfoSetIndexKey = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; + var redisCacheMeterInfoZSetScoresIndexKey = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, itemTimeDensity.Key)}"; List ammeterInfos = new List(); //将表计信息根据集中器分组，获得集中器号 @@ -329,11 +358,10 @@ namespace JiShe.CollectBus.ScheduledMeterReading //await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs); } - await FreeRedisProvider.BatchAddMeterData( - redisCacheKey, - redisCacheFocusIndexKey, - redisCacheScoresIndexKey, - redisCacheGlobalIndexKey, ammeterInfos); + await _redisDataCacheService.BatchInsertDataAsync( + redisCacheMeterInfoHashKey, + redisCacheMeterInfoSetIndexKey, + redisCacheMeterInfoZSetScoresIndexKey,ammeterInfos); //在缓存表信息数据的时候，新增下一个时间的自动处理任务，1分钟后执行所有的采集频率任务 TasksToBeIssueModel nextTask = new TasksToBeIssueModel() @@ -635,7 +663,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading var currentTime = DateTime.Now; var pendingCopyReadTime = currentTime.AddMinutes(timeDensity); //构建缓存任务key，依然 表计类型+采集频率+集中器地址，存hash类型 - var redisCacheKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}{ammeterInfo.FocusAddress}"; + var redisCacheKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoHashKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}{ammeterInfo.FocusAddress}"; if (string.IsNullOrWhiteSpace(ammeterInfo.ItemCodes)) { @@ -897,7 +925,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading foreach (var focusInfo in focusGroup) { //构建缓存任务key，依然 表计类型+采集频率+集中器地址，存hash类型 - var redisCacheKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}{focusInfo.Key}"; + var redisCacheKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoHashKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}{focusInfo.Key}"; foreach (var ammeterInfo in focusInfo.Value) { @@ -1113,7 +1141,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading continue; } - var redisCacheKey = $"{string.Format(RedisConst.CacheMeterInfoKey, SystemType, ServerTagName, MeterTypeEnum.WaterMeter, itemTimeDensity.Key)}{item.Key}"; + var redisCacheKey = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, ServerTagName, MeterTypeEnum.WaterMeter, itemTimeDensity.Key)}{item.Key}"; Dictionary keyValuePairs = new Dictionary(); foreach (var subItem in item) { @@ -1260,7 +1288,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading /// private string GetTelemetryPacketCacheKeyPrefix(int timeDensity, MeterTypeEnum meterType) { - return $"{string.Format(RedisConst.CacheTelemetryPacketInfoKey, SystemType, ServerTagName, meterType, timeDensity)}*"; + return $"{string.Format(RedisConst.CacheTelemetryPacketInfoHashKey, SystemType, ServerTagName, meterType, timeDensity)}*"; } #endregion diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs index d44fe56..25c5476 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Confluent.Kafka; using DotNetCore.CAP; using JiShe.CollectBus.Ammeters; +using JiShe.CollectBus.Application.Contracts; using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.DeviceBalanceControl; using JiShe.CollectBus.Common.Helpers; @@ -35,8 +36,19 @@ namespace JiShe.CollectBus.ScheduledMeterReading public class EnergySystemScheduledMeterReadingService : BasicScheduledMeterReadingService { string serverTagName = string.Empty; - public EnergySystemScheduledMeterReadingService(ILogger logger, - ICapPublisher producerBus, IIoTDBProvider dbProvider, IMeterReadingRecordRepository meterReadingRecordRepository,IConfiguration configuration, IProducerService producerService) : base(logger, producerBus, meterReadingRecordRepository, producerService,dbProvider) + public EnergySystemScheduledMeterReadingService( + ILogger logger, + ICapPublisher producerBus, IIoTDBProvider dbProvider, + IMeterReadingRecordRepository meterReadingRecordRepository, + IConfiguration configuration, + IProducerService producerService, + IRedisDataCacheService redisDataCacheService) + : base(logger, + producerBus, + meterReadingRecordRepository, + producerService, + redisDataCacheService, + dbProvider) { serverTagName = configuration.GetValue(CommonConst.ServerTagName)!; } diff --git a/src/JiShe.CollectBus.Common/Consts/RedisConst.cs b/src/JiShe.CollectBus.Common/Consts/RedisConst.cs index 9377056..dce5307 100644 --- a/src/JiShe.CollectBus.Common/Consts/RedisConst.cs +++ b/src/JiShe.CollectBus.Common/Consts/RedisConst.cs @@ -32,23 +32,18 @@ namespace JiShe.CollectBus.Common.Consts /// /// 缓存表计信息,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 /// - public const string CacheMeterInfoKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:{"{3}"}"; + public const string CacheMeterInfoHashKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:{"{3}"}"; /// - /// 缓存表计信息集中器索引Set缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// 缓存表计信息索引Set缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 /// - public const string CacheMeterInfoFocusIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:FocusIndex:{"{3}"}"; + public const string CacheMeterInfoSetIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:SetIndex:{"{3}"}"; /// - /// 缓存表计信息集中器排序索引ZSET缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// 缓存表计信息排序索引ZSET缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 /// - public const string CacheMeterInfoScoresIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:ScoresIndex:{"{3}"}"; - - /// - /// 缓存表计信息集中器采集频率分组全局索引ZSet缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 - /// - public const string CacheMeterInfoGlobalIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:GlobalIndex:{"{3}"}"; - + public const string CacheMeterInfoZSetScoresIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{MeterInfo}:{"{2}"}:ZSetScoresIndex:{"{3}"}"; + public const string TaskInfo = "TaskInfo"; /// @@ -60,7 +55,17 @@ namespace JiShe.CollectBus.Common.Consts /// /// 缓存表计下发指令数据集,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 /// - public const string CacheTelemetryPacketInfoKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:{"{3}"}"; + public const string CacheTelemetryPacketInfoHashKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:{"{3}"}"; + + /// + /// 缓存表计下发指令数据集索引Set缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// + public const string CacheTelemetryPacketInfoSetIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:SetIndex:{"{3}"}"; + + /// + /// 缓存表计下发指令数据集排序索引ZSET缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// + public const string CacheTelemetryPacketInfoZSetScoresIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:ZSetScoresIndex:{"{3}"}"; ///// ///// 缓存设备平衡关系映射结果,{0}=>系统类型，{1}=>应用服务部署标记 diff --git a/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs b/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs index 303b2e4..465bd15 100644 --- a/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs +++ b/src/JiShe.CollectBus.Common/Models/BusCacheGlobalPagedResult.cs @@ -22,6 +22,22 @@ namespace JiShe.CollectBus.Common.Models /// public long TotalCount { get; set; } + /// + /// 每页条数 + /// + public int PageSize { get; set; } + + /// + /// 总页数 + /// + public int PageCount + { + get + { + return (int)Math.Ceiling((double)TotalCount / PageSize); + } + } + /// /// 是否有下一页 /// diff --git a/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs b/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs index 59011de..335c17c 100644 --- a/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs +++ b/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs @@ -22,8 +22,13 @@ namespace JiShe.CollectBus.Common.Models public int MeterId { get; set; } /// - /// 唯一标识，是redis ZSet和Set memberid + /// 关系映射标识，用于ZSet的Member字段和Set的Value字段，具体值可以根据不同业务场景进行定义 /// public virtual string MemberID => $"{FocusId}:{MeterId}"; + + /// + /// ZSet排序索引分数值，具体值可以根据不同业务场景进行定义，例如时间戳 + /// + public virtual long ScoreValue=> ((long)FocusId << 32) | (uint)MeterId; } } diff --git a/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs b/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs index cebc9c4..d3a9bff 100644 --- a/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs +++ b/src/JiShe.CollectBus.FreeRedisProvider/FreeRedisProvider.cs @@ -36,639 +36,472 @@ namespace JiShe.CollectBus.FreeRedisProvider public IRedisClient GetInstance() { - var connectionString = $"{_option.Configuration},defaultdatabase={_option.DefaultDB}"; + var connectionString = $"{_option.Configuration},defaultdatabase={_option.DefaultDB},MaxPoolSize={_option.MaxPoolSize}"; Instance = new RedisClient(connectionString); Instance.Serialize = obj => BusJsonSerializer.Serialize(obj); Instance.Deserialize = (json, type) => BusJsonSerializer.Deserialize(json, type); - Instance.Notice += (s, e) => Trace.WriteLine(e.Log); + Instance.Notice += (s, e) => Trace.WriteLine(e.Log); return Instance; } - /// - /// 单个添加数据 - /// - /// - /// 主数据存储Hash缓存Key - /// 集中器索引Set缓存Key - /// 集中器排序索引ZSET缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 表计信息 - /// 可选时间戳 - /// - public async Task AddMeterCacheData( - string redisCacheKey, - string redisCacheFocusIndexKey, - string redisCacheScoresIndexKey, - string redisCacheGlobalIndexKey, - T data, - DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel - { - // 参数校验增强 - if (data == null || string.IsNullOrWhiteSpace(redisCacheKey) - || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey) - || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) - || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) - { - throw new ArgumentException($"{nameof(AddMeterCacheData)} 参数异常，-101"); - } + ///// + ///// 单个添加数据 + ///// + ///// + ///// 主数据存储Hash缓存Key + ///// 集中器索引Set缓存Key + ///// 集中器排序索引ZSET缓存Key + ///// 集中器采集频率分组全局索引ZSet缓存Key + ///// 表计信息 + ///// 可选时间戳 + ///// + //public async Task AddMeterCacheData( + //string redisCacheKey, + //string redisCacheFocusIndexKey, + //string redisCacheScoresIndexKey, + //string redisCacheGlobalIndexKey, + //T data, + //DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel + //{ + // // 参数校验增强 + // if (data == null || string.IsNullOrWhiteSpace(redisCacheKey) + // || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey) + // || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) + // || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) + // { + // throw new ArgumentException($"{nameof(AddMeterCacheData)} 参数异常，-101"); + // } - // 计算组合score（分类ID + 时间戳） - var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow; + // // 计算组合score（分类ID + 时间戳） + // var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow; - long scoreValue = ((long)data.FocusId << 32) | (uint)actualTimestamp.Ticks; + // long scoreValue = ((long)data.FocusId << 32) | (uint)actualTimestamp.Ticks; - //全局索引写入 - long globalScore = actualTimestamp.ToUnixTimeMilliseconds(); + // //全局索引写入 + // long globalScore = actualTimestamp.ToUnixTimeMilliseconds(); - // 使用事务保证原子性 - using (var trans = Instance.Multi()) - { - // 主数据存储Hash - trans.HSet(redisCacheKey, data.MemberID, data.Serialize()); + // // 使用事务保证原子性 + // using (var trans = Instance.Multi()) + // { + // // 主数据存储Hash + // trans.HSet(redisCacheKey, data.MemberID, data.Serialize()); - // 分类索引 - trans.SAdd(redisCacheFocusIndexKey, data.MemberID); + // // 分类索引 + // trans.SAdd(redisCacheFocusIndexKey, data.MemberID); - // 排序索引使用ZSET - trans.ZAdd(redisCacheScoresIndexKey, scoreValue, data.MemberID); + // // 排序索引使用ZSET + // trans.ZAdd(redisCacheScoresIndexKey, scoreValue, data.MemberID); - //全局索引 - trans.ZAdd(redisCacheGlobalIndexKey, globalScore, data.MemberID); + // //全局索引 + // trans.ZAdd(redisCacheGlobalIndexKey, globalScore, data.MemberID); - var results = trans.Exec(); + // var results = trans.Exec(); - if (results == null || results.Length <= 0) - throw new Exception($"{nameof(AddMeterCacheData)} 事务提交失败，-102"); - } + // if (results == null || results.Length <= 0) + // throw new Exception($"{nameof(AddMeterCacheData)} 事务提交失败，-102"); + // } - await Task.CompletedTask; - } + // await Task.CompletedTask; + //} - /// - /// 批量添加数据 - /// - /// - /// 主数据存储Hash缓存Key - /// 集中器索引Set缓存Key - /// 集中器排序索引ZSET缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 数据集合 - /// 可选时间戳 - /// - public async Task BatchAddMeterData( - string redisCacheKey, - string redisCacheFocusIndexKey, - string redisCacheScoresIndexKey, - string redisCacheGlobalIndexKey, - IEnumerable items, - DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel - { - if (items == null - || items.Count() <=0 - || string.IsNullOrWhiteSpace(redisCacheKey) - || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey) - || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) - || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) - { - throw new ArgumentException($"{nameof(BatchAddMeterData)} 参数异常，-101"); - } + ///// + ///// 批量添加数据 + ///// + ///// + ///// 主数据存储Hash缓存Key + ///// 集中器索引Set缓存Key + ///// 集中器排序索引ZSET缓存Key + ///// 集中器采集频率分组全局索引ZSet缓存Key + ///// 数据集合 + ///// 可选时间戳 + ///// + //public async Task BatchAddMeterData( + //string redisCacheKey, + //string redisCacheFocusIndexKey, + //string redisCacheScoresIndexKey, + //string redisCacheGlobalIndexKey, + //IEnumerable items, + //DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel + //{ + // if (items == null + // || items.Count() <=0 + // || string.IsNullOrWhiteSpace(redisCacheKey) + // || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey) + // || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) + // || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) + // { + // throw new ArgumentException($"{nameof(BatchAddMeterData)} 参数异常，-101"); + // } - const int BATCH_SIZE = 1000; // 每批1000条 - var semaphore = new SemaphoreSlim(Environment.ProcessorCount * 2); + // const int BATCH_SIZE = 1000; // 每批1000条 + // var semaphore = new SemaphoreSlim(Environment.ProcessorCount * 2); - foreach (var batch in items.Batch(BATCH_SIZE)) - { - await semaphore.WaitAsync(); + // foreach (var batch in items.Batch(BATCH_SIZE)) + // { + // await semaphore.WaitAsync(); - _ = Task.Run(() => - { - using (var pipe = Instance.StartPipe()) - { - foreach (var item in batch) - { - // 计算组合score（分类ID + 时间戳） - var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow; + // _ = Task.Run(() => + // { + // using (var pipe = Instance.StartPipe()) + // { + // foreach (var item in batch) + // { + // // 计算组合score（分类ID + 时间戳） + // var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow; - long scoreValue = ((long)item.FocusId << 32) | (uint)actualTimestamp.Ticks; + // long scoreValue = ((long)item.FocusId << 32) | (uint)actualTimestamp.Ticks; - //全局索引写入 - long globalScore = actualTimestamp.ToUnixTimeMilliseconds(); + // //全局索引写入 + // long globalScore = actualTimestamp.ToUnixTimeMilliseconds(); - // 主数据存储Hash - pipe.HSet(redisCacheKey, item.MemberID, item.Serialize()); + // // 主数据存储Hash + // pipe.HSet(redisCacheKey, item.MemberID, item.Serialize()); - // 分类索引 - pipe.SAdd(redisCacheFocusIndexKey, item.MemberID); + // // 分类索引Set + // pipe.SAdd(redisCacheFocusIndexKey, item.MemberID); - // 排序索引使用ZSET - pipe.ZAdd(redisCacheScoresIndexKey, scoreValue, item.MemberID); + // // 排序索引使用ZSET + // pipe.ZAdd(redisCacheScoresIndexKey, scoreValue, item.MemberID); - //全局索引 - pipe.ZAdd(redisCacheGlobalIndexKey, globalScore, item.MemberID); - } - pipe.EndPipe(); - } - semaphore.Release(); - }); - } + // //全局索引 + // pipe.ZAdd(redisCacheGlobalIndexKey, globalScore, item.MemberID); + // } + // pipe.EndPipe(); + // } + // semaphore.Release(); + // }); + // } - await Task.CompletedTask; - } + // await Task.CompletedTask; + //} - /// - /// 删除指定redis缓存key的缓存数据 - /// - /// - /// 主数据存储Hash缓存Key - /// 集中器索引Set缓存Key - /// 集中器排序索引ZSET缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 表计信息 - /// - public async Task RemoveMeterData( - string redisCacheKey, - string redisCacheFocusIndexKey, - string redisCacheScoresIndexKey, - string redisCacheGlobalIndexKey, - T data) where T : DeviceCacheBasicModel - { + ///// + ///// 删除指定redis缓存key的缓存数据 + ///// + ///// + ///// 主数据存储Hash缓存Key + ///// 集中器索引Set缓存Key + ///// 集中器排序索引ZSET缓存Key + ///// 集中器采集频率分组全局索引ZSet缓存Key + ///// 表计信息 + ///// + //public async Task RemoveMeterData( + //string redisCacheKey, + //string redisCacheFocusIndexKey, + //string redisCacheScoresIndexKey, + //string redisCacheGlobalIndexKey, + //T data) where T : DeviceCacheBasicModel + //{ - if (data == null - || string.IsNullOrWhiteSpace(redisCacheKey) - || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey) - || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) - || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) - { - throw new ArgumentException($"{nameof(RemoveMeterData)} 参数异常，-101"); - } + // if (data == null + // || string.IsNullOrWhiteSpace(redisCacheKey) + // || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey) + // || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) + // || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) + // { + // throw new ArgumentException($"{nameof(RemoveMeterData)} 参数异常，-101"); + // } - const string luaScript = @" - local mainKey = KEYS[1] - local focusIndexKey = KEYS[2] - local scoresIndexKey = KEYS[3] - local globalIndexKey = KEYS[4] - local member = ARGV[1] + // const string luaScript = @" + // local mainKey = KEYS[1] + // local focusIndexKey = KEYS[2] + // local scoresIndexKey = KEYS[3] + // local globalIndexKey = KEYS[4] + // local member = ARGV[1] - local deleted = 0 - if redis.call('HDEL', mainKey, member) > 0 then - deleted = 1 - end + // local deleted = 0 + // if redis.call('HDEL', mainKey, member) > 0 then + // deleted = 1 + // end - redis.call('SREM', focusIndexKey, member) - redis.call('ZREM', scoresIndexKey, member) - redis.call('ZREM', globalIndexKey, member) - return deleted - "; + // redis.call('SREM', focusIndexKey, member) + // redis.call('ZREM', scoresIndexKey, member) + // redis.call('ZREM', globalIndexKey, member) + // return deleted + // "; - var keys = new[] - { - redisCacheKey, - redisCacheFocusIndexKey, - redisCacheScoresIndexKey, - redisCacheGlobalIndexKey - }; + // var keys = new[] + // { + // redisCacheKey, + // redisCacheFocusIndexKey, + // redisCacheScoresIndexKey, + // redisCacheGlobalIndexKey + // }; - var result = await Instance.EvalAsync(luaScript, keys, new[] { data.MemberID }); + // var result = await Instance.EvalAsync(luaScript, keys, new[] { data.MemberID }); - if ((int)result == 0) - throw new KeyNotFoundException("指定数据不存在"); - } + // if ((int)result == 0) + // throw new KeyNotFoundException("指定数据不存在"); + //} - /// - /// 修改表计缓存信息 - /// - /// - /// 主数据存储Hash缓存Key - /// 旧集中器索引Set缓存Key - /// 新集中器索引Set缓存Key - /// 集中器排序索引ZSET缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 表计信息 - /// 可选时间戳 - /// - public async Task UpdateMeterData( - string redisCacheKey, - string oldRedisCacheFocusIndexKey, - string newRedisCacheFocusIndexKey, - string redisCacheScoresIndexKey, - string redisCacheGlobalIndexKey, - T newData, - DateTimeOffset? newTimestamp = null) where T : DeviceCacheBasicModel - { - if (newData == null - || string.IsNullOrWhiteSpace(redisCacheKey) - || string.IsNullOrWhiteSpace(oldRedisCacheFocusIndexKey) - || string.IsNullOrWhiteSpace(newRedisCacheFocusIndexKey) - || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) - || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) - { - throw new ArgumentException($"{nameof(UpdateMeterData)} 参数异常，-101"); - } + ///// + ///// 修改表计缓存信息 + ///// + ///// + ///// 主数据存储Hash缓存Key + ///// 旧集中器索引Set缓存Key + ///// 新集中器索引Set缓存Key + ///// 集中器排序索引ZSET缓存Key + ///// 集中器采集频率分组全局索引ZSet缓存Key + ///// 表计信息 + ///// 可选时间戳 + ///// + //public async Task UpdateMeterData( + //string redisCacheKey, + //string oldRedisCacheFocusIndexKey, + //string newRedisCacheFocusIndexKey, + //string redisCacheScoresIndexKey, + //string redisCacheGlobalIndexKey, + //T newData, + //DateTimeOffset? newTimestamp = null) where T : DeviceCacheBasicModel + //{ + // if (newData == null + // || string.IsNullOrWhiteSpace(redisCacheKey) + // || string.IsNullOrWhiteSpace(oldRedisCacheFocusIndexKey) + // || string.IsNullOrWhiteSpace(newRedisCacheFocusIndexKey) + // || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey) + // || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) + // { + // throw new ArgumentException($"{nameof(UpdateMeterData)} 参数异常，-101"); + // } - var luaScript = @" - local mainKey = KEYS[1] - local oldFocusIndexKey = KEYS[2] - local newFocusIndexKey = KEYS[3] - local scoresIndexKey = KEYS[4] - local globalIndexKey = KEYS[5] - local member = ARGV[1] - local newData = ARGV[2] - local newScore = ARGV[3] - local newGlobalScore = ARGV[4] + // var luaScript = @" + // local mainKey = KEYS[1] + // local oldFocusIndexKey = KEYS[2] + // local newFocusIndexKey = KEYS[3] + // local scoresIndexKey = KEYS[4] + // local globalIndexKey = KEYS[5] + // local member = ARGV[1] + // local newData = ARGV[2] + // local newScore = ARGV[3] + // local newGlobalScore = ARGV[4] - -- 校验存在性 - if redis.call('HEXISTS', mainKey, member) == 0 then - return 0 - end + // -- 校验存在性 + // if redis.call('HEXISTS', mainKey, member) == 0 then + // return 0 + // end - -- 更新主数据 - redis.call('HSET', mainKey, member, newData) + // -- 更新主数据 + // redis.call('HSET', mainKey, member, newData) - -- 处理变更 - if newScore ~= '' then - -- 删除旧索引 - redis.call('SREM', oldFocusIndexKey, member) - redis.call('ZREM', scoresIndexKey, member) + // -- 处理变更 + // if newScore ~= '' then + // -- 删除旧索引 + // redis.call('SREM', oldFocusIndexKey, member) + // redis.call('ZREM', scoresIndexKey, member) - -- 添加新索引 - redis.call('SADD', newFocusIndexKey, member) - redis.call('ZADD', scoresIndexKey, newScore, member) - end + // -- 添加新索引 + // redis.call('SADD', newFocusIndexKey, member) + // redis.call('ZADD', scoresIndexKey, newScore, member) + // end - -- 更新全局索引 - if newGlobalScore ~= '' then - -- 删除旧索引 - redis.call('ZREM', globalIndexKey, member) + // -- 更新全局索引 + // if newGlobalScore ~= '' then + // -- 删除旧索引 + // redis.call('ZREM', globalIndexKey, member) - -- 添加新索引 - redis.call('ZADD', globalIndexKey, newGlobalScore, member) - end + // -- 添加新索引 + // redis.call('ZADD', globalIndexKey, newGlobalScore, member) + // end - return 1 - "; + // return 1 + // "; - var actualTimestamp = newTimestamp ?? DateTimeOffset.UtcNow; - var newGlobalScore = actualTimestamp.ToUnixTimeMilliseconds(); - var newScoreValue = ((long)newData.FocusId << 32) | (uint)actualTimestamp.Ticks; + // var actualTimestamp = newTimestamp ?? DateTimeOffset.UtcNow; + // var newGlobalScore = actualTimestamp.ToUnixTimeMilliseconds(); + // var newScoreValue = ((long)newData.FocusId << 32) | (uint)actualTimestamp.Ticks; - var result = await Instance.EvalAsync(luaScript, - new[] - { - redisCacheKey, - oldRedisCacheFocusIndexKey, - newRedisCacheFocusIndexKey, - redisCacheScoresIndexKey, - redisCacheGlobalIndexKey - }, - new object[] - { - newData.MemberID, - newData.Serialize(), - newScoreValue.ToString() ?? "", - newGlobalScore.ToString() ?? "" - }); + // var result = await Instance.EvalAsync(luaScript, + // new[] + // { + // redisCacheKey, + // oldRedisCacheFocusIndexKey, + // newRedisCacheFocusIndexKey, + // redisCacheScoresIndexKey, + // redisCacheGlobalIndexKey + // }, + // new object[] + // { + // newData.MemberID, + // newData.Serialize(), + // newScoreValue.ToString() ?? "", + // newGlobalScore.ToString() ?? "" + // }); - if ((int)result == 0) - { - throw new KeyNotFoundException($"{nameof(UpdateMeterData)}指定Key{redisCacheKey}的数据不存在"); - } - } + // if ((int)result == 0) + // { + // throw new KeyNotFoundException($"{nameof(UpdateMeterData)}指定Key{redisCacheKey}的数据不存在"); + // } + //} - public async Task> SingleGetMeterPagedData( - string redisCacheKey, - string redisCacheScoresIndexKey, - int focusId, - int pageSize = 10, - int pageIndex = 1, - bool descending = true) - { - // 计算score范围 - long minScore = (long)focusId << 32; - long maxScore = ((long)focusId + 1) << 32; + //public async Task> SingleGetMeterPagedData( + //string redisCacheKey, + //string redisCacheScoresIndexKey, + //int focusId, + //int pageSize = 10, + //int pageIndex = 1, + //bool descending = true) + //{ + // // 计算score范围 + // long minScore = (long)focusId << 32; + // long maxScore = ((long)focusId + 1) << 32; - // 分页参数计算 - int start = (pageIndex - 1) * pageSize; + // // 分页参数计算 + // int start = (pageIndex - 1) * pageSize; - // 获取排序后的member列表 - var members = descending - ? await Instance.ZRevRangeByScoreAsync( - redisCacheScoresIndexKey, - maxScore, - minScore, - start, - pageSize) - : await Instance.ZRangeByScoreAsync( - redisCacheScoresIndexKey, - minScore, - maxScore, - start, - pageSize); + // // 获取排序后的member列表 + // var members = descending + // ? await Instance.ZRevRangeByScoreAsync( + // redisCacheScoresIndexKey, + // maxScore, + // minScore, + // start, + // pageSize) + // : await Instance.ZRangeByScoreAsync( + // redisCacheScoresIndexKey, + // minScore, + // maxScore, + // start, + // pageSize); - // 批量获取实际数据 - var dataTasks = members.Select(m => - Instance.HGetAsync(redisCacheKey, m)).ToArray(); - await Task.WhenAll(dataTasks); + // // 批量获取实际数据 + // var dataTasks = members.Select(m => + // Instance.HGetAsync(redisCacheKey, m)).ToArray(); + // await Task.WhenAll(dataTasks); - // 总数统计优化 - var total = await Instance.ZCountAsync( - redisCacheScoresIndexKey, - minScore, - maxScore); + // // 总数统计优化 + // var total = await Instance.ZCountAsync( + // redisCacheScoresIndexKey, + // minScore, + // maxScore); - return new BusPagedResult - { - Items = dataTasks.Select(t => t.Result).ToList(), - TotalCount = total, - PageIndex = pageIndex, - PageSize = pageSize - }; - } + // return new BusPagedResult + // { + // Items = dataTasks.Select(t => t.Result).ToList(), + // TotalCount = total, + // PageIndex = pageIndex, + // PageSize = pageSize + // }; + //} - public async Task> GetFocusPagedData( - string redisCacheKey, - string redisCacheScoresIndexKey, - int focusId, - int pageSize = 10, - long? lastScore = null, - string lastMember = null, - bool descending = true) where T : DeviceCacheBasicModel - { - // 计算分数范围 - long minScore = (long)focusId << 32; - long maxScore = ((long)focusId + 1) << 32; + //public async Task> GetFocusPagedData( + //string redisCacheKey, + //string redisCacheScoresIndexKey, + //int focusId, + //int pageSize = 10, + //long? lastScore = null, + //string lastMember = null, + //bool descending = true) where T : DeviceCacheBasicModel + //{ + // // 计算分数范围 + // long minScore = (long)focusId << 32; + // long maxScore = ((long)focusId + 1) << 32; - // 获取成员列表 - var members = await GetSortedMembers( - redisCacheScoresIndexKey, - minScore, - maxScore, - pageSize, - lastScore, - lastMember, - descending); + // // 获取成员列表 + // var members = await GetSortedMembers( + // redisCacheScoresIndexKey, + // minScore, + // maxScore, + // pageSize, + // lastScore, + // lastMember, + // descending); - // 批量获取数据 - var dataDict = await Instance.HMGetAsync(redisCacheKey, members.CurrentItems); + // // 批量获取数据 + // var dataDict = await Instance.HMGetAsync(redisCacheKey, members.CurrentItems); - return new BusPagedResult - { - Items = dataDict, - TotalCount = await GetTotalCount(redisCacheScoresIndexKey, minScore, maxScore), - HasNext = members.HasNext, - NextScore = members.NextScore, - NextMember = members.NextMember - }; - } + // return new BusPagedResult + // { + // Items = dataDict, + // TotalCount = await GetTotalCount(redisCacheScoresIndexKey, minScore, maxScore), + // HasNext = members.HasNext, + // NextScore = members.NextScore, + // NextMember = members.NextMember + // }; + //} - private async Task<(string[] CurrentItems, bool HasNext, decimal? NextScore, string NextMember)> - GetSortedMembers( - string zsetKey, - long minScore, - long maxScore, - int pageSize, - long? lastScore, - string lastMember, - bool descending) - { - var querySize = pageSize + 1; - var (startScore, exclude) = descending - ? (lastScore ?? maxScore, lastMember) - : (lastScore ?? minScore, lastMember); + //private async Task<(string[] CurrentItems, bool HasNext, decimal? NextScore, string NextMember)> + // GetSortedMembers( + // string zsetKey, + // long minScore, + // long maxScore, + // int pageSize, + // long? lastScore, + // string lastMember, + // bool descending) + //{ + // var querySize = pageSize + 1; + // var (startScore, exclude) = descending + // ? (lastScore ?? maxScore, lastMember) + // : (lastScore ?? minScore, lastMember); - var members = descending - ? await Instance.ZRevRangeByScoreAsync( - zsetKey, - max: startScore, - min: minScore, - offset: 0, - count: querySize) - : await Instance.ZRangeByScoreAsync( - zsetKey, - min: startScore, - max: maxScore, - offset: 0, - count: querySize); + // var members = descending + // ? await Instance.ZRevRangeByScoreAsync( + // zsetKey, + // max: startScore, + // min: minScore, + // offset: 0, + // count: querySize) + // : await Instance.ZRangeByScoreAsync( + // zsetKey, + // min: startScore, + // max: maxScore, + // offset: 0, + // count: querySize); - var hasNext = members.Length > pageSize; - var currentItems = members.Take(pageSize).ToArray(); + // var hasNext = members.Length > pageSize; + // var currentItems = members.Take(pageSize).ToArray(); - var nextCursor = currentItems.Any() - ? await GetNextCursor(zsetKey, currentItems.Last(), descending) - : (null, null); + // var nextCursor = currentItems.Any() + // ? await GetNextCursor(zsetKey, currentItems.Last(), descending) + // : (null, null); - return (currentItems, hasNext, nextCursor.score, nextCursor.member); - } + // return (currentItems, hasNext, nextCursor.score, nextCursor.member); + //} - private async Task GetTotalCount(string zsetKey, long min, long max) - { - // 缓存计数优化 - var cacheKey = $"{zsetKey}_count_{min}_{max}"; - var cached = await Instance.GetAsync(cacheKey); + //private async Task GetTotalCount(string zsetKey, long min, long max) + //{ + // // 缓存计数优化 + // var cacheKey = $"{zsetKey}_count_{min}_{max}"; + // var cached = await Instance.GetAsync(cacheKey); - if (cached.HasValue) - return cached.Value; + // if (cached.HasValue) + // return cached.Value; - var count = await Instance.ZCountAsync(zsetKey, min, max); - await Instance.SetExAsync(cacheKey, 60, count); // 缓存60秒 - return count; - } + // var count = await Instance.ZCountAsync(zsetKey, min, max); + // await Instance.SetExAsync(cacheKey, 60, count); // 缓存60秒 + // return count; + //} - public async Task>> BatchGetMeterPagedData( - string redisCacheKey, - string redisCacheScoresIndexKey, - IEnumerable focusIds, - int pageSizePerFocus = 10) where T : DeviceCacheBasicModel - { - var results = new ConcurrentDictionary>(); - var parallelOptions = new ParallelOptions - { - MaxDegreeOfParallelism = Environment.ProcessorCount * 2 - }; + //public async Task>> BatchGetMeterPagedData( + //string redisCacheKey, + //string redisCacheScoresIndexKey, + //IEnumerable focusIds, + //int pageSizePerFocus = 10) where T : DeviceCacheBasicModel + //{ + // var results = new ConcurrentDictionary>(); + // var parallelOptions = new ParallelOptions + // { + // MaxDegreeOfParallelism = Environment.ProcessorCount * 2 + // }; - await Parallel.ForEachAsync(focusIds, parallelOptions, async (focusId, _) => - { - var data = await SingleGetMeterPagedData( - redisCacheKey, - redisCacheScoresIndexKey, - focusId, - pageSizePerFocus); + // await Parallel.ForEachAsync(focusIds, parallelOptions, async (focusId, _) => + // { + // var data = await SingleGetMeterPagedData( + // redisCacheKey, + // redisCacheScoresIndexKey, + // focusId, + // pageSizePerFocus); - results.TryAdd(focusId, data); - }); + // results.TryAdd(focusId, data); + // }); + + // return new Dictionary>(results); + //} + - return new Dictionary>(results); - } - /// - /// 通过全局索引分页查询表计缓存数据 - /// - /// - /// 主数据存储Hash缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 分页尺寸 - /// 最后一个索引 - /// 最后一个唯一标识 - /// 排序方式 - /// - public async Task> GetGlobalPagedData( - string redisCacheKey, - string redisCacheGlobalIndexKey, - int pageSize = 10, - decimal? lastScore = null, - string lastMember = null, - bool descending = true) - where T : DeviceCacheBasicModel - { - // 参数校验增强 - if (string.IsNullOrWhiteSpace(redisCacheKey) || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey)) - { - throw new ArgumentException($"{nameof(GetGlobalPagedData)} 参数异常，-101"); - } - - if (pageSize < 1 || pageSize > 1000) - { - throw new ArgumentException($"{nameof(GetGlobalPagedData)} 分页大小应在1-1000之间，-102"); - } - - // 分页参数解析 - var (startScore, excludeMember) = descending - ? (lastScore ?? decimal.MaxValue, lastMember) - : (lastScore ?? 0, lastMember); - - // 游标分页查询 - var (members, hasNext) = await GetPagedMembers( - redisCacheGlobalIndexKey, - pageSize, - startScore, - excludeMember, - descending); - - // 批量获取数据（优化内存分配） - var dataDict = await BatchGetData(redisCacheKey, members); - - // 获取下一页游标 - var nextCursor = members.Any() - ? await GetNextCursor(redisCacheGlobalIndexKey, members.Last(), descending) - : (null, null); - - return new BusCacheGlobalPagedResult - { - Items = members.Select(m => dataDict.TryGetValue(m, out var v) ? v : default) - .Where(x => x != null).ToList(), - HasNext = hasNext, - NextScore = nextCursor.score, - NextMember = nextCursor.member - }; - } - - /// - /// 游标分页查询 - /// - /// - /// 分页数量 - /// 开始索引 - /// 开始唯一标识 - /// 排序方式 - /// - private async Task<(List Members, bool HasNext)> GetPagedMembers( - string redisCacheGlobalIndexKey, - int pageSize, - decimal? startScore, - string excludeMember, - bool descending) - { - const int bufferSize = 50; // 预读缓冲区大小 - - // 使用流式分页（避免OFFSET性能问题） - var members = new List(pageSize + 1); - decimal? currentScore = startScore; - string lastMember = excludeMember; - - while (members.Count < pageSize + 1 && currentScore.HasValue) - { - var querySize = Math.Min(bufferSize, pageSize + 1 - members.Count); - - var batch = descending - ? await Instance.ZRevRangeByScoreAsync( - redisCacheGlobalIndexKey, - max: currentScore.Value, - min: 0, - offset: 0, - count: querySize - ) - : await Instance.ZRangeByScoreAsync( - redisCacheGlobalIndexKey, - min: currentScore.Value, - max: long.MaxValue, - offset: 0, - count: querySize); - - if (!batch.Any()) break; - - members.AddRange(batch); - lastMember = batch.LastOrDefault(); - currentScore = await Instance.ZScoreAsync(redisCacheGlobalIndexKey, lastMember); - } - - return ( - members.Take(pageSize).ToList(), - members.Count > pageSize - ); - } - - /// - /// 批量获取指定分页的数据 - /// - /// - /// - /// - /// - private async Task> BatchGetData( - string hashKey, - IEnumerable members) - where T : DeviceCacheBasicModel - { - const int batchSize = 100; - var result = new Dictionary(); - - foreach (var batch in members.Batch(batchSize)) - { - var batchArray = batch.ToArray(); - var values = await Instance.HMGetAsync(hashKey, batchArray); - - for (int i = 0; i < batchArray.Length; i++) - { - if (EqualityComparer.Default.Equals(values[i], default)) continue; - result[batchArray[i]] = values[i]; - } - } - - return result; - } - - /// - /// 获取下一页游标 - /// - /// 全局索引Key - /// 最后一个唯一标识 - /// 排序方式 - /// - private async Task<(decimal? score, string member)> GetNextCursor( - string redisCacheGlobalIndexKey, - string lastMember, - bool descending) - { - if (string.IsNullOrWhiteSpace(lastMember)) - { - return (null, null); - } - - var score = await Instance.ZScoreAsync(redisCacheGlobalIndexKey, lastMember); - return score.HasValue - ? (Convert.ToInt64(score.Value), lastMember) - : (null, null); - } } } \ No newline at end of file diff --git a/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs b/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs index dc0aaa3..8fc6861 100644 --- a/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs +++ b/src/JiShe.CollectBus.FreeRedisProvider/IFreeRedisProvider.cs @@ -9,105 +9,7 @@ namespace JiShe.CollectBus.FreeRedisProvider /// 获取客户端 /// /// - RedisClient Instance { get; set; } - - /// - /// 单个添加数据 - /// - /// - /// 主数据存储Hash缓存Key - /// 集中器索引Set缓存Key - /// 集中器排序索引ZSET缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 表计信息 - /// 可选时间戳 - /// - Task AddMeterCacheData( - string redisCacheKey, - string redisCacheFocusIndexKey, - string redisCacheScoresIndexKey, - string redisCacheGlobalIndexKey, - T data, - DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel; - - /// - /// 批量添加数据 - /// - /// - /// 主数据存储Hash缓存Key - /// 集中器索引Set缓存Key - /// 集中器排序索引ZSET缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 数据集合 - /// 可选时间戳 - /// - Task BatchAddMeterData( - string redisCacheKey, - string redisCacheFocusIndexKey, - string redisCacheScoresIndexKey, - string redisCacheGlobalIndexKey, - IEnumerable items, - DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel; - - /// - /// 删除指定redis缓存key的缓存数据 - /// - /// - /// 主数据存储Hash缓存Key - /// 集中器索引Set缓存Key - /// 集中器排序索引ZSET缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 表计信息 - /// - Task RemoveMeterData( - string redisCacheKey, - string redisCacheFocusIndexKey, - string redisCacheScoresIndexKey, - string redisCacheGlobalIndexKey, - T data) where T : DeviceCacheBasicModel; - - /// - /// 修改表计缓存信息 - /// - /// - /// 主数据存储Hash缓存Key - /// 旧集中器索引Set缓存Key - /// 新集中器索引Set缓存Key - /// 集中器排序索引ZSET缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 表计信息 - /// 可选时间戳 - /// - Task UpdateMeterData( - string redisCacheKey, - string oldRedisCacheFocusIndexKey, - string newRedisCacheFocusIndexKey, - string redisCacheScoresIndexKey, - string redisCacheGlobalIndexKey, - T newData, - DateTimeOffset? newTimestamp = null) where T : DeviceCacheBasicModel; - - - - /// - /// 通过全局索引分页查询表计缓存数据 - /// - /// - /// 主数据存储Hash缓存Key - /// 集中器采集频率分组全局索引ZSet缓存Key - /// 分页尺寸 - /// 最后一个索引 - /// 最后一个唯一标识 - /// 排序方式 - /// - Task> GetGlobalPagedData( - string redisCacheKey, - string redisCacheGlobalIndexKey, - int pageSize = 10, - decimal? lastScore = null, - string lastMember = null, - bool descending = true) - where T : DeviceCacheBasicModel; + RedisClient Instance { get; set; } } } diff --git a/src/JiShe.CollectBus.FreeRedisProvider/Options/FreeRedisOptions.cs b/src/JiShe.CollectBus.FreeRedisProvider/Options/FreeRedisOptions.cs index 75d92a2..2ed7e49 100644 --- a/src/JiShe.CollectBus.FreeRedisProvider/Options/FreeRedisOptions.cs +++ b/src/JiShe.CollectBus.FreeRedisProvider/Options/FreeRedisOptions.cs @@ -13,6 +13,11 @@ namespace JiShe.CollectBus.FreeRedisProvider.Options /// public string? Configuration { get; set; } + /// + /// 最大连接数 + /// + public string? MaxPoolSize { get; set; } + /// /// 默认数据库 /// diff --git a/src/JiShe.CollectBus.Host/appsettings.json b/src/JiShe.CollectBus.Host/appsettings.json index c922f8d..bb09204 100644 --- a/src/JiShe.CollectBus.Host/appsettings.json +++ b/src/JiShe.CollectBus.Host/appsettings.json @@ -41,6 +41,7 @@ }, "Redis": { "Configuration": "192.168.1.9:6380,password=1q2w3e!@#,syncTimeout=30000,abortConnect=false,connectTimeout=30000,allowAdmin=true", + "MaxPoolSize": "50", "DefaultDB": "14", "HangfireDB": "15" }, @@ -128,7 +129,7 @@ "OpenDebugMode": true, "UseTableSessionPoolByDefault": false }, - "ServerTagName": "JiSheCollectBus2", + "ServerTagName": "JiSheCollectBus3", "KafkaReplicationFactor": 3, "NumPartitions": 30 } \ No newline at end of file From eed68d0fe007b7a2d7558e76fb7df227f77742b2 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Wed, 16 Apr 2025 18:26:25 +0800 Subject: [PATCH 10/27] =?UTF-8?q?kafka=E5=A2=9E=E5=8A=A0=E6=89=B9=E9=87=8F?= =?UTF-8?q?=E6=B6=88=E8=B4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EnergySystem/EnergySystemAppService.cs | 34 +- .../Plugins/TcpMonitor.cs | 6 +- .../BasicScheduledMeterReadingService.cs | 10 +- .../Attributes/KafkaSubscribeAttribute.cs | 37 ++- .../Consumer/ConsumerService.cs | 313 +++++++++++++++++- .../Consumer/IConsumerService.cs | 8 +- .../HeadersFilter.cs | 30 ++ .../JsonSerializer.cs | 88 +++++ .../KafkaSubcribesExtensions.cs | 20 +- .../Producer/ProducerService.cs | 49 ++- .../Abstracts/BaseProtocolPlugin.cs | 4 +- 11 files changed, 535 insertions(+), 64 deletions(-) create mode 100644 src/JiShe.CollectBus.KafkaProducer/HeadersFilter.cs create mode 100644 src/JiShe.CollectBus.KafkaProducer/JsonSerializer.cs diff --git a/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs b/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs index 2a60803..1ca731b 100644 --- a/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs +++ b/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs @@ -90,7 +90,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); result.Status = true; result.Msg = "操作成功"; result.Data.ValidData = true; @@ -136,7 +136,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); } @@ -186,7 +186,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); result.Status = true; result.Msg = "操作成功"; result.Data.ValidData = true; @@ -224,7 +224,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); } result.Status = true; @@ -316,7 +316,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); if (isManual) { @@ -384,7 +384,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); result.Status = true; result.Msg = "操作成功"; @@ -420,7 +420,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); result.Status = true; result.Msg = "操作成功"; @@ -456,7 +456,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); result.Status = true; result.Msg = "操作成功"; @@ -491,7 +491,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); result.Status = true; result.Msg = "操作成功"; @@ -527,7 +527,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); result.Status = true; result.Msg = "操作成功"; return result; @@ -585,7 +585,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); } result.Status = true; @@ -662,7 +662,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); } result.Status = true; result.Msg = "操作成功"; @@ -699,7 +699,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); } result.Status = true; result.Msg = "操作成功"; @@ -735,7 +735,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); } result.Status = true; result.Msg = "操作成功"; @@ -769,7 +769,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); result.Status = true; result.Msg = "操作成功"; return result; @@ -804,7 +804,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); } result.Status = true; result.Msg = "操作成功"; @@ -867,7 +867,7 @@ namespace JiShe.CollectBus.EnergySystem Message = bytes, Type = IssuedEventType.Data, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); } result.Status = true; diff --git a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs index 50b41b0..6cbc8c4 100644 --- a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs +++ b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs @@ -178,7 +178,7 @@ namespace JiShe.CollectBus.Plugins //await _producerBus.PublishAsync(ProtocolConst.SubscriberLoginReceivedEventName, messageReceivedLoginEvent); - await _producerService.ProduceAsync(ProtocolConst.SubscriberLoginReceivedEventName, messageReceivedLoginEvent.Serialize()); + await _producerService.ProduceAsync(ProtocolConst.SubscriberLoginReceivedEventName, messageReceivedLoginEvent); //await _producerBus.Publish( messageReceivedLoginEvent); } @@ -227,7 +227,7 @@ namespace JiShe.CollectBus.Plugins }; //await _producerBus.PublishAsync(ProtocolConst.SubscriberHeartbeatReceivedEventName, messageReceivedHeartbeatEvent); - await _producerService.ProduceAsync(ProtocolConst.SubscriberHeartbeatReceivedEventName, messageReceivedHeartbeatEvent.Serialize()); + await _producerService.ProduceAsync(ProtocolConst.SubscriberHeartbeatReceivedEventName, messageReceivedHeartbeatEvent); //await _producerBus.Publish(messageReceivedHeartbeatEvent); } @@ -271,7 +271,7 @@ namespace JiShe.CollectBus.Plugins MessageHexString = messageHexString, DeviceNo = deviceNo, MessageId = NewId.NextGuid().ToString() - }.Serialize()); + }); } } } diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index 8fa4f5e..d830707 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -382,7 +382,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading }; //_ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName, tempMsg); - _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName, tempMsg.Serialize()); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerOneMinuteIssuedEventName, tempMsg); //_= _producerBus.Publish(tempMsg); @@ -448,7 +448,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading }; //_ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName, tempMsg); - _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName, tempMsg.Serialize()); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFiveMinuteIssuedEventName, tempMsg); //_ = _producerBus.Publish(tempMsg); @@ -514,7 +514,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading }; //_ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg); - _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg.Serialize()); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg); //_ = _producerBus.Publish(tempMsg); @@ -809,7 +809,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading } int partition = DeviceGroupBalanceControl.GetDeviceGroupId(taskRecord.FocusAddress); - await _producerService.ProduceAsync(topicName, partition, taskRecord.Serialize()); + await _producerService.ProduceAsync(topicName, partition, taskRecord); } private async Task AmmerterCreatePublishTask(int timeDensity, MeterTypeEnum meterType) @@ -851,7 +851,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading }; //_ = _producerBus.PublishDelayAsync(TimeSpan.FromMicroseconds(500), ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg); - _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg.Serialize()); + _ = _producerService.ProduceAsync(ProtocolConst.AmmeterSubscriberWorkerFifteenMinuteIssuedEventName, tempMsg); //_ = _producerBus.Publish(tempMsg); diff --git a/src/JiShe.CollectBus.KafkaProducer/Attributes/KafkaSubscribeAttribute.cs b/src/JiShe.CollectBus.KafkaProducer/Attributes/KafkaSubscribeAttribute.cs index 32f652e..df75b89 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Attributes/KafkaSubscribeAttribute.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Attributes/KafkaSubscribeAttribute.cs @@ -26,20 +26,51 @@ namespace JiShe.CollectBus.Kafka.Attributes /// /// 任务数(默认是多少个分区多少个任务) + /// 如设置订阅指定Partition则任务数始终为1 /// public int TaskCount { get; set; } = -1; - public KafkaSubscribeAttribute(string topic, string groupId = "default") + /// + /// 批量处理数量 + /// + public int BatchSize { get; set; } = 100; + + /// + /// 是否启用批量处理 + /// + public bool EnableBatch { get; set; } = false; + + /// + /// 批次超时时间 + /// + public TimeSpan? BatchTimeout { get; set; }=null; + + /// + /// 订阅主题 + /// + /// batchTimeout格式:("00:05:00") + public KafkaSubscribeAttribute(string topic, string groupId = "default", bool enableBatch = false, int batchSize = 100, string? batchTimeout = null) { this.Topic = topic; this.GroupId = groupId; + this.EnableBatch = enableBatch; + this.BatchSize = batchSize; + this.BatchTimeout = batchTimeout != null? TimeSpan.Parse(batchTimeout): null; } - public KafkaSubscribeAttribute(string topic, int partition, string groupId = "default") + /// + /// 订阅主题 + /// + /// batchTimeout格式:("00:05:00") + public KafkaSubscribeAttribute(string topic, int partition, string groupId = "default", bool enableBatch = false, int batchSize = 100, string? batchTimeout = null) { - this.Topic = topic ; + this.Topic = topic; this.Partition = partition; this.GroupId = groupId; + this.TaskCount = 1; + this.EnableBatch = enableBatch; + this.BatchSize = batchSize; + this.BatchTimeout = batchTimeout != null ? TimeSpan.Parse(batchTimeout) : null; } } } diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs index 9a2142e..5547919 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs @@ -10,6 +10,8 @@ using System.Collections.Concurrent; using System.Text.RegularExpressions; using NUglify.Html; using Serilog; +using System; +using System.Text; namespace JiShe.CollectBus.Kafka.Consumer { @@ -19,6 +21,7 @@ namespace JiShe.CollectBus.Kafka.Consumer private readonly IConfiguration _configuration; private readonly ConcurrentDictionary _consumerStore = new(); + private class KafkaConsumer where TKey : notnull where TValue : class { } public ConsumerService(IConfiguration configuration, ILogger logger) { @@ -38,6 +41,7 @@ namespace JiShe.CollectBus.Kafka.Consumer { var config = BuildConsumerConfig(groupId); return new ConsumerBuilder(config) + .SetValueDeserializer(new JsonSerializer()) .SetLogHandler((_, log) => _logger.LogInformation($"消费者Log: {log.Message}")) .SetErrorHandler((_, e) => _logger.LogError($"消费者错误: {e.Reason}")) .Build(); @@ -54,7 +58,8 @@ namespace JiShe.CollectBus.Kafka.Consumer AutoOffsetReset = AutoOffsetReset.Earliest, EnableAutoCommit = false, // 禁止AutoCommit EnablePartitionEof = true, // 启用分区末尾标记 - AllowAutoCreateTopics= true // 启用自动创建 + AllowAutoCreateTopics= true, // 启用自动创建 + FetchMaxBytes = 1024 * 1024 * 50 // 增加拉取大小（50MB） }; if (enableAuth) @@ -105,7 +110,7 @@ namespace JiShe.CollectBus.Kafka.Consumer /// public async Task SubscribeAsync(string[] topics, Func> messageHandler, string? groupId = null) where TKey : notnull where TValue : class { - var consumerKey = typeof((TKey, TValue)); + var consumerKey = typeof(KafkaConsumer); var cts = new CancellationTokenSource(); var consumer = _consumerStore.GetOrAdd(consumerKey, _ => @@ -123,15 +128,28 @@ namespace JiShe.CollectBus.Kafka.Consumer try { var result = consumer.Consume(cts.Token); - if (result == null) continue; - if (result.Message.Value == null) continue; + if (result == null || result.Message==null || result.Message.Value == null) + { + _logger.LogWarning($"Kafka消费: {result?.Topic} 分区 {result?.Partition} 值为NULL"); + consumer.Commit(result); // 手动提交 + continue; + } if (result.IsPartitionEOF) { _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition); - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1),cts.Token); + continue; + } + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + // 检查 Header 是否符合条件 + if (!headersFilter.Match(result.Message.Headers)) + { + _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} Header未匹配", result.Topic, result.Partition); + + //consumer.Commit(result); // 提交偏移量 + // 跳过消息 continue; } - bool sucess= await messageHandler(result.Message.Key, result.Message.Value); if (sucess) { @@ -159,7 +177,7 @@ namespace JiShe.CollectBus.Kafka.Consumer /// public async Task SubscribeAsync(string[] topics, Func> messageHandler, string? groupId) where TValue : class { - var consumerKey = typeof((Ignore, TValue)); + var consumerKey = typeof(KafkaConsumer); var cts = new CancellationTokenSource(); var consumer = _consumerStore.GetOrAdd(consumerKey, _=> @@ -177,15 +195,27 @@ namespace JiShe.CollectBus.Kafka.Consumer try { var result = consumer.Consume(cts.Token); - if (result == null) continue; - if (result.Message == null) continue; + if (result == null || result.Message==null || result.Message.Value == null) + { + _logger.LogWarning($"Kafka消费: {result?.Topic} 分区 {result?.Partition} 值为NULL"); + consumer.Commit(result); // 手动提交 + continue; + } if (result.IsPartitionEOF) { _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition); - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(100, cts.Token); + continue; + } + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + // 检查 Header 是否符合条件 + if (!headersFilter.Match(result.Message.Headers)) + { + _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} Header值未匹配", result.Topic, result.Partition); + //consumer.Commit(result); // 提交偏移量 + // 跳过消息 continue; } - bool sucess = await messageHandler(result.Message.Value); if (sucess) consumer.Commit(result); // 手动提交 @@ -199,6 +229,267 @@ namespace JiShe.CollectBus.Kafka.Consumer await Task.CompletedTask; } + + /// + /// 批量订阅消息 + /// + /// 消息Key类型 + /// 消息Value类型 + /// 主题 + /// 批量消息处理函数 + /// 消费组ID + /// 批次大小 + /// 批次超时时间 + public async Task SubscribeBatchAsync(string topic, Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class + { + await SubscribeBatchAsync(new[] { topic }, messageBatchHandler, groupId, batchSize, batchTimeout); + } + + /// + /// 批量订阅消息 + /// + /// 消息Key类型 + /// 消息Value类型 + /// 主题列表 + /// 批量消息处理函数 + /// 消费组ID + /// 批次大小 + /// 批次超时时间 + public async Task SubscribeBatchAsync(string[] topics,Func, Task> messageBatchHandler, string? groupId = null,int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class + { + var consumerKey = typeof(KafkaConsumer); + var cts = new CancellationTokenSource(); + + var consumer = _consumerStore.GetOrAdd(consumerKey, _ => + ( + CreateConsumer(groupId), + cts + )).Consumer as IConsumer; + + consumer!.Subscribe(topics); + + var timeout = batchTimeout ?? TimeSpan.FromSeconds(5); // 默认超时时间调整为5秒 + + _ = Task.Run(async () => + { + var messages = new List<(TValue Value, TopicPartitionOffset Offset)>(); + var startTime = DateTime.UtcNow; + + while (!cts.IsCancellationRequested) + { + try + { + // 非阻塞快速累积消息 + while (messages.Count < batchSize && (DateTime.UtcNow - startTime) < timeout) + { + var result = consumer.Consume(TimeSpan.Zero); // 非阻塞调用 + + if (result != null) + { + if (result.IsPartitionEOF) + { + _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); + } + else if (result.Message.Value != null) + { + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + // 检查 Header 是否符合条件 + if (!headersFilter.Match(result.Message.Headers)) + { + _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} Header未匹配", result.Topic, result.Partition); + //consumer.Commit(result); // 提交偏移量 + // 跳过消息 + continue; + } + messages.Add((result.Message.Value, result.TopicPartitionOffset)); + //messages.Add(result.Message.Value); + } + } + else + { + // 无消息时短暂等待 + await Task.Delay(10, cts.Token); + } + } + + // 处理批次 + if (messages.Count > 0) + { + bool success = await messageBatchHandler(messages.Select(m => m.Value)); + if (success) + { + var offsetsByPartition = new Dictionary(); + foreach (var msg in messages) + { + var tp = msg.Offset.TopicPartition; + var offset = msg.Offset.Offset; + if (!offsetsByPartition.TryGetValue(tp, out var currentMax) || offset > currentMax) + { + offsetsByPartition[tp] = offset; + } + } + + var offsetsToCommit = offsetsByPartition + .Select(kv => new TopicPartitionOffset(kv.Key, new Offset(kv.Value + 1))) + .ToList(); + consumer.Commit(offsetsToCommit); + } + messages.Clear(); + } + + startTime = DateTime.UtcNow; + } + catch (ConsumeException ex) + { + _logger.LogError(ex, $"消息消费失败: {ex.Error.Reason}"); + } + catch (OperationCanceledException) + { + // 任务取消，正常退出 + } + catch (Exception ex) + { + _logger.LogError(ex, "处理批量消息时发生未知错误"); + } + } + }, cts.Token); + + await Task.CompletedTask; + } + + + /// + /// 批量订阅消息 + /// + /// 消息Value类型 + /// 主题列表 + /// 批量消息处理函数 + /// 消费组ID + /// 批次大小 + /// 批次超时时间 + /// 消费等待时间 + public async Task SubscribeBatchAsync(string topic, Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class + { + await SubscribeBatchAsync(new[] { topic }, messageBatchHandler, groupId, batchSize, batchTimeout, consumeTimeout); + + } + + + /// + /// 批量订阅消息 + /// + /// 消息Value类型 + /// 主题列表 + /// 批量消息处理函数 + /// 消费组ID + /// 批次大小 + /// 批次超时时间 + /// 消费等待时间 + public async Task SubscribeBatchAsync(string[] topics,Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100,TimeSpan? batchTimeout = null,TimeSpan? consumeTimeout = null)where TValue : class + { + var consumerKey = typeof(KafkaConsumer); + var cts = new CancellationTokenSource(); + + var consumer = _consumerStore.GetOrAdd(consumerKey, _ => + ( + CreateConsumer(groupId), + cts + )).Consumer as IConsumer; + + consumer!.Subscribe(topics); + + var timeout = batchTimeout ?? TimeSpan.FromSeconds(5); // 默认超时时间调整为5秒 + + _ = Task.Run(async () => + { + var messages = new List<(TValue Value, TopicPartitionOffset Offset)>(); + //var messages = new List>(); + var startTime = DateTime.UtcNow; + + while (!cts.IsCancellationRequested) + { + try + { + // 非阻塞快速累积消息 + while (messages.Count < batchSize && (DateTime.UtcNow - startTime) < timeout) + { + var result = consumer.Consume(TimeSpan.Zero); // 非阻塞调用 + + if (result != null) + { + if (result.IsPartitionEOF) + { + _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); + } + else if (result.Message.Value != null) + { + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + // 检查 Header 是否符合条件 + if (!headersFilter.Match(result.Message.Headers)) + { + //consumer.Commit(result); // 提交偏移量 + // 跳过消息 + continue; + } + messages.Add((result.Message.Value, result.TopicPartitionOffset)); + //messages.Add(result.Message.Value); + } + } + else + { + // 无消息时短暂等待 + await Task.Delay(10, cts.Token); + } + } + + // 处理批次 + if (messages.Count > 0) + { + bool success = await messageBatchHandler(messages.Select(m => m.Value)); + if (success) + { + var offsetsByPartition = new Dictionary(); + foreach (var msg in messages) + { + var tp = msg.Offset.TopicPartition; + var offset = msg.Offset.Offset; + if (!offsetsByPartition.TryGetValue(tp, out var currentMax) || offset > currentMax) + { + offsetsByPartition[tp] = offset; + } + } + + var offsetsToCommit = offsetsByPartition + .Select(kv => new TopicPartitionOffset(kv.Key, new Offset(kv.Value + 1))) + .ToList(); + consumer.Commit(offsetsToCommit); + } + messages.Clear(); + } + + startTime = DateTime.UtcNow; + } + catch (ConsumeException ex) + { + _logger.LogError(ex, $"消息消费失败: {ex.Error.Reason}"); + } + catch (OperationCanceledException) + { + // 任务取消，正常退出 + } + catch (Exception ex) + { + _logger.LogError(ex, "处理批量消息时发生未知错误"); + } + } + }, cts.Token); + + await Task.CompletedTask; + } + + /// /// 取消消息订阅 /// diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/IConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/IConsumerService.cs index 5cfae2c..3925014 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/IConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/IConsumerService.cs @@ -1,4 +1,5 @@ -using System; +using Confluent.Kafka; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -32,6 +33,11 @@ namespace JiShe.CollectBus.Kafka.Consumer /// Task SubscribeAsync(string[] topics, Func> messageHandler, string? groupId = null) where TValue : class; + + Task SubscribeBatchAsync(string topic, Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class; + + Task SubscribeBatchAsync(string[] topics, Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class; + void Unsubscribe() where TKey : notnull where TValue : class; } } diff --git a/src/JiShe.CollectBus.KafkaProducer/HeadersFilter.cs b/src/JiShe.CollectBus.KafkaProducer/HeadersFilter.cs new file mode 100644 index 0000000..0790f9f --- /dev/null +++ b/src/JiShe.CollectBus.KafkaProducer/HeadersFilter.cs @@ -0,0 +1,30 @@ +using Confluent.Kafka; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JiShe.CollectBus.Kafka +{ + /// + /// 消息头过滤器 + /// + public class HeadersFilter : Dictionary + { + /// + /// 判断Headers是否匹配 + /// + /// + /// + public bool Match(Headers headers) + { + foreach (var kvp in this) + { + if (!headers.TryGetLastBytes(kvp.Key, out var value) || !value.SequenceEqual(kvp.Value)) + return false; + } + return true; + } + } +} diff --git a/src/JiShe.CollectBus.KafkaProducer/JsonSerializer.cs b/src/JiShe.CollectBus.KafkaProducer/JsonSerializer.cs new file mode 100644 index 0000000..83f58a3 --- /dev/null +++ b/src/JiShe.CollectBus.KafkaProducer/JsonSerializer.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Text.Json; +using Confluent.Kafka; +using System.Text.Json.Serialization; +using System.Text.Encodings.Web; + +namespace JiShe.CollectBus.Kafka +{ + /// + /// JSON 序列化器（支持泛型） + /// + public class JsonSerializer : ISerializer, IDeserializer + { + private static readonly JsonSerializerOptions _options = new JsonSerializerOptions + { + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + WriteIndented = false,// 设置格式化输出 + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,// 允许特殊字符 + IgnoreReadOnlyFields = true, + IgnoreReadOnlyProperties = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString, // 允许数字字符串 + AllowTrailingCommas = true, // 忽略尾随逗号 + ReadCommentHandling = JsonCommentHandling.Skip, // 忽略注释 + PropertyNameCaseInsensitive = true, // 属性名称大小写不敏感 + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // 属性名称使用驼峰命名规则 + Converters = { new DateTimeJsonConverter() } // 注册你的自定义转换器, + }; + + public byte[] Serialize(T data, SerializationContext context) + { + if (data == null) + return null; + + try + { + return JsonSerializer.SerializeToUtf8Bytes(data, _options); + } + catch (Exception ex) + { + throw new InvalidOperationException("Kafka序列化失败", ex); + } + } + + public T Deserialize(ReadOnlySpan data, bool isNull, SerializationContext context) + { + if (isNull) + return default; + + try + { + return JsonSerializer.Deserialize(data, _options); + } + catch (Exception ex) + { + throw new InvalidOperationException("Kafka反序列化失败", ex); + } + } + } + + + public class DateTimeJsonConverter : JsonConverter + { + private readonly string _dateFormatString; + public DateTimeJsonConverter() + { + _dateFormatString = "yyyy-MM-dd HH:mm:ss"; + } + + public DateTimeJsonConverter(string dateFormatString) + { + _dateFormatString = dateFormatString; + } + + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return DateTime.Parse(reader.GetString()); + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString(_dateFormatString)); + } + } +} diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index cfe5eee..8860061 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -102,7 +102,7 @@ namespace JiShe.CollectBus.Kafka { var consumerService = provider.GetRequiredService(); - await consumerService.SubscribeAsync(attr.Topic, async (message) => + await consumerService.SubscribeAsync(attr.Topic, async (message) => { try { @@ -126,21 +126,21 @@ namespace JiShe.CollectBus.Kafka /// /// /// - private static async Task ProcessMessageAsync(string message, MethodInfo method, object subscribe) + private static async Task ProcessMessageAsync(dynamic message, MethodInfo method, object subscribe) { var parameters = method.GetParameters(); bool isGenericTask = method.ReturnType.IsGenericType && method.ReturnType.GetGenericTypeDefinition() == typeof(Task<>); bool existParameters = parameters.Length > 0; - dynamic? messageObj= null; - if (existParameters) - { - var paramType = parameters[0].ParameterType; - messageObj = paramType == typeof(string) ? message : message.Deserialize(paramType); - } + //dynamic? messageObj= null; + //if (existParameters) + //{ + // var paramType = parameters[0].ParameterType; + // messageObj = paramType == typeof(string) ? message : message.Deserialize(paramType); + //} if (isGenericTask) { - object? result = await (Task)method.Invoke(subscribe, existParameters? new[] { messageObj }:null)!; + object? result = await (Task)method.Invoke(subscribe, existParameters? new[] { message } :null)!; if (result is ISubscribeAck ackResult) { return ackResult.Ack; @@ -148,7 +148,7 @@ namespace JiShe.CollectBus.Kafka } else { - object? result = method.Invoke(subscribe, existParameters ? new[] { messageObj } : null); + object? result = method.Invoke(subscribe, existParameters ? new[] { message } : null); if (result is ISubscribeAck ackResult) { return ackResult.Ack; diff --git a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs index 8c069a9..0ddf36b 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs @@ -17,7 +17,8 @@ namespace JiShe.CollectBus.Kafka.Producer { private readonly ILogger _logger; private readonly IConfiguration _configuration; - private readonly ConcurrentDictionary, object> _producerCache = new(); + private readonly ConcurrentDictionary _producerCache = new(); + private class KafkaProducer where TKey : notnull where TValue : class { } public ProducerService(IConfiguration configuration,ILogger logger) { @@ -32,14 +33,13 @@ namespace JiShe.CollectBus.Kafka.Producer /// /// /// - private IProducer GetProducer() + private IProducer GetProducer(Type typeKey) { - var typeKey = Tuple.Create(typeof(TKey), typeof(TValue))!; - return (IProducer)_producerCache.GetOrAdd(typeKey, _ => { var config = BuildProducerConfig(); return new ProducerBuilder(config) + .SetValueSerializer(new JsonSerializer()) // Value 使用自定义 JSON 序列化 .SetLogHandler((_, msg) => _logger.Log(ConvertLogLevel(msg.Level), msg.Message)) .Build(); }); @@ -103,8 +103,17 @@ namespace JiShe.CollectBus.Kafka.Producer /// public async Task ProduceAsync(string topic, TKey key, TValue value)where TKey : notnull where TValue : class { - var producer = GetProducer(); - await producer.ProduceAsync(topic, new Message { Key = key, Value = value }); + var typeKey = typeof(KafkaProducer); + var producer = GetProducer(typeKey); + var message = new Message + { + Key = key, + Value = value, + Headers = new Headers{ + { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } + } + }; + await producer.ProduceAsync(topic, message); } /// @@ -116,8 +125,16 @@ namespace JiShe.CollectBus.Kafka.Producer /// public async Task ProduceAsync(string topic, TValue value) where TValue : class { - var producer = GetProducer(); - await producer.ProduceAsync(topic, new Message { Value = value }); + var typeKey = typeof(KafkaProducer); + var producer = GetProducer(typeKey); + var message = new Message + { + Value = value, + Headers = new Headers{ + { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } + } + }; + await producer.ProduceAsync(topic, message); } /// @@ -136,9 +153,13 @@ namespace JiShe.CollectBus.Kafka.Producer var message = new Message { Key = key, - Value = value + Value = value, + Headers = new Headers{ + { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } + } }; - var producer = GetProducer(); + var typeKey = typeof(KafkaProducer); + var producer = GetProducer(typeKey); if (partition.HasValue) { var topicPartition = new TopicPartition(topic, partition.Value); @@ -166,9 +187,13 @@ namespace JiShe.CollectBus.Kafka.Producer { var message = new Message { - Value = value + Value = value, + Headers = new Headers{ + { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } + } }; - var producer = GetProducer(); + var typeKey = typeof(KafkaProducer); + var producer = GetProducer(typeKey); if (partition.HasValue) { var topicPartition = new TopicPartition(topic, partition.Value); diff --git a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs index 963d89b..c43fa70 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs +++ b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs @@ -92,7 +92,7 @@ namespace JiShe.CollectBus.Protocol.Contracts.Abstracts var bytes = Build3761SendData.BuildSendCommandBytes(reqParam); //await _producerBus.PublishAsync(ProtocolConst.SubscriberLoginIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }); - await _producerService.ProduceAsync(ProtocolConst.SubscriberLoginIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }.Serialize()); + await _producerService.ProduceAsync(ProtocolConst.SubscriberLoginIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }); //await _producerBus.Publish(new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Login, MessageId = messageReceived.MessageId }); } @@ -133,7 +133,7 @@ namespace JiShe.CollectBus.Protocol.Contracts.Abstracts var bytes = Build3761SendData.BuildSendCommandBytes(reqParam); //await _producerBus.PublishAsync(ProtocolConst.SubscriberHeartbeatIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }); - await _producerService.ProduceAsync(ProtocolConst.SubscriberHeartbeatIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }.Serialize()); + await _producerService.ProduceAsync(ProtocolConst.SubscriberHeartbeatIssuedEventName, new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }); //await _producerBus.Publish(new IssuedEventMessage { ClientId = messageReceived.ClientId, DeviceNo = messageReceived.DeviceNo, Message = bytes, Type = IssuedEventType.Heartbeat, MessageId = messageReceived.MessageId }); } From 78f9ef349adb24c57086345672c0a63de9442cf7 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Wed, 16 Apr 2025 18:46:51 +0800 Subject: [PATCH 11/27] =?UTF-8?q?=E4=BC=98=E5=8C=96kafka=20=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E9=BB=98=E8=AE=A4=E5=BA=8F=E5=88=97=E5=8C=96=E5=92=8C?= =?UTF-8?q?=E5=8F=8D=E5=BA=8F=E5=88=97=E5=8C=96=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=89=B9=E9=87=8F=E6=B6=88=E8=B4=B9=20=E5=A2=9E=E5=8A=A0Header?= =?UTF-8?q?=E6=B6=88=E8=B4=B9=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Consumer/ConsumerService.cs | 4 ++-- .../Consumer/IConsumerService.cs | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs index 5547919..0625304 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs @@ -131,7 +131,7 @@ namespace JiShe.CollectBus.Kafka.Consumer if (result == null || result.Message==null || result.Message.Value == null) { _logger.LogWarning($"Kafka消费: {result?.Topic} 分区 {result?.Partition} 值为NULL"); - consumer.Commit(result); // 手动提交 + //consumer.Commit(result); // 手动提交 continue; } if (result.IsPartitionEOF) @@ -198,7 +198,7 @@ namespace JiShe.CollectBus.Kafka.Consumer if (result == null || result.Message==null || result.Message.Value == null) { _logger.LogWarning($"Kafka消费: {result?.Topic} 分区 {result?.Partition} 值为NULL"); - consumer.Commit(result); // 手动提交 + //consumer.Commit(result); // 手动提交 continue; } if (result.IsPartitionEOF) diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/IConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/IConsumerService.cs index 3925014..d86dba8 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/IConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/IConsumerService.cs @@ -33,10 +33,13 @@ namespace JiShe.CollectBus.Kafka.Consumer /// Task SubscribeAsync(string[] topics, Func> messageHandler, string? groupId = null) where TValue : class; + Task SubscribeBatchAsync(string[] topics, Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class; Task SubscribeBatchAsync(string topic, Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class; - Task SubscribeBatchAsync(string[] topics, Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class; + Task SubscribeBatchAsync(string topic, Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class; + + Task SubscribeBatchAsync(string[] topics, Func, Task> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class; void Unsubscribe() where TKey : notnull where TValue : class; } From 3790f3918e4bb9cc4511e1fb7ef1bf4875b17a31 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Wed, 16 Apr 2025 20:41:52 +0800 Subject: [PATCH 12/27] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Header=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E8=BF=87=E6=BB=A4=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/JiShe.CollectBus.Host/appsettings.json | 1 + .../Consumer/ConsumerService.cs | 64 +++++++++++-------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/src/JiShe.CollectBus.Host/appsettings.json b/src/JiShe.CollectBus.Host/appsettings.json index a837636..91ff4ea 100644 --- a/src/JiShe.CollectBus.Host/appsettings.json +++ b/src/JiShe.CollectBus.Host/appsettings.json @@ -83,6 +83,7 @@ }, "Kafka": { "BootstrapServers": "192.168.1.9:29092,192.168.1.9:39092,192.168.1.9:49092", + "EnableFilter": true, "EnableAuthorization": false, "SecurityProtocol": "SASL_PLAINTEXT", "SaslMechanism": "PLAIN", diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs index 0625304..da9258c 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs @@ -140,15 +140,16 @@ namespace JiShe.CollectBus.Kafka.Consumer await Task.Delay(TimeSpan.FromSeconds(1),cts.Token); continue; } - var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; - // 检查 Header 是否符合条件 - if (!headersFilter.Match(result.Message.Headers)) + if (bool.Parse(_configuration["KafkaConsumer:EnableFilter"]!)) { - _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} Header未匹配", result.Topic, result.Partition); - - //consumer.Commit(result); // 提交偏移量 - // 跳过消息 - continue; + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + // 检查 Header 是否符合条件 + if (!headersFilter.Match(result.Message.Headers)) + { + //consumer.Commit(result); // 提交偏移量 + // 跳过消息 + continue; + } } bool sucess= await messageHandler(result.Message.Key, result.Message.Value); if (sucess) @@ -207,14 +208,16 @@ namespace JiShe.CollectBus.Kafka.Consumer await Task.Delay(100, cts.Token); continue; } - var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; - // 检查 Header 是否符合条件 - if (!headersFilter.Match(result.Message.Headers)) + if (bool.Parse(_configuration["KafkaConsumer:EnableFilter"]!)) { - _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} Header值未匹配", result.Topic, result.Partition); - //consumer.Commit(result); // 提交偏移量 - // 跳过消息 - continue; + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + // 检查 Header 是否符合条件 + if (!headersFilter.Match(result.Message.Headers)) + { + //consumer.Commit(result); // 提交偏移量 + // 跳过消息 + continue; + } } bool sucess = await messageHandler(result.Message.Value); if (sucess) @@ -293,14 +296,16 @@ namespace JiShe.CollectBus.Kafka.Consumer } else if (result.Message.Value != null) { - var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; - // 检查 Header 是否符合条件 - if (!headersFilter.Match(result.Message.Headers)) + if (bool.Parse(_configuration["KafkaConsumer:EnableFilter"]!)) { - _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} Header未匹配", result.Topic, result.Partition); - //consumer.Commit(result); // 提交偏移量 - // 跳过消息 - continue; + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + // 检查 Header 是否符合条件 + if (!headersFilter.Match(result.Message.Headers)) + { + //consumer.Commit(result); // 提交偏移量 + // 跳过消息 + continue; + } } messages.Add((result.Message.Value, result.TopicPartitionOffset)); //messages.Add(result.Message.Value); @@ -425,13 +430,16 @@ namespace JiShe.CollectBus.Kafka.Consumer } else if (result.Message.Value != null) { - var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; - // 检查 Header 是否符合条件 - if (!headersFilter.Match(result.Message.Headers)) + if (bool.Parse(_configuration["KafkaConsumer:EnableFilter"]!)) { - //consumer.Commit(result); // 提交偏移量 - // 跳过消息 - continue; + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + // 检查 Header 是否符合条件 + if (!headersFilter.Match(result.Message.Headers)) + { + //consumer.Commit(result); // 提交偏移量 + // 跳过消息 + continue; + } } messages.Add((result.Message.Value, result.TopicPartitionOffset)); //messages.Add(result.Message.Value); From 849e0f9ac21c9531aae6a6d175c9fefdbcec71a2 Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Wed, 16 Apr 2025 23:51:27 +0800 Subject: [PATCH 13/27] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RedisDataCache/RedisDataCacheService.cs | 208 +++++++++++------- .../BasicScheduledMeterReadingService.cs | 74 +++---- .../Extensions/EnumerableExtensions.cs | 21 ++ .../Ammeters/AmmeterInfo.cs | 17 +- src/JiShe.CollectBus.Host/appsettings.json | 2 +- 5 files changed, 201 insertions(+), 121 deletions(-) diff --git a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs index 4d78706..8f50017 100644 --- a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs +++ b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs @@ -12,6 +12,9 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using Volo.Abp.DependencyInjection; +using static FreeSql.Internal.GlobalFilter; +using static System.Runtime.InteropServices.JavaScript.JSType; +using static Volo.Abp.UI.Navigation.DefaultMenuNames.Application; namespace JiShe.CollectBus.RedisDataCache { @@ -118,7 +121,7 @@ namespace JiShe.CollectBus.RedisDataCache await semaphore.WaitAsync(); _ = Task.Run(() => - { + { using (var pipe = Instance.StartPipe()) { foreach (var item in batch) @@ -372,129 +375,172 @@ namespace JiShe.CollectBus.RedisDataCache /// 最后一个索引 /// 最后一个唯一标识 /// 排序方式 - /// + /// public async Task> GetAllPagedData( - string redisHashCacheKey, - string redisZSetScoresIndexCacheKey, - int pageSize = 1000, - decimal? lastScore = null, - string lastMember = null, - bool descending = true) - where T : DeviceCacheBasicModel + string redisHashCacheKey, + string redisZSetScoresIndexCacheKey, + int pageSize = 1000, + decimal? lastScore = null, + string lastMember = null, + bool descending = true) + where T : DeviceCacheBasicModel { // 参数校验（保持不变） - if (string.IsNullOrWhiteSpace(redisHashCacheKey) || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + if (string.IsNullOrWhiteSpace(redisHashCacheKey) || + string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) { - _logger.LogError($"{nameof(GetAllPagedData)} 参数异常，-101"); + _logger.LogError("参数异常: HashKey或ZSetKey为空"); return null; } if (pageSize < 1 || pageSize > 10000) - { - _logger.LogError($"{nameof(GetAllPagedData)} 分页大小应在1-10000之间，-102"); - return null; - } + throw new ArgumentException("分页大小应在1-10000之间"); + // 更新后的Lua脚本 var luaScript = @" - local command = ARGV[1] - local range_start = ARGV[2] - local range_end = ARGV[3] - local limit = tonumber(ARGV[4]) - local last_score = ARGV[5] - local last_member = ARGV[6] +local command = ARGV[1] +local range_start = ARGV[2] +local range_end = ARGV[3] +local limit = tonumber(ARGV[4]) +local last_score = ARGV[5] +local last_member = ARGV[6] - -- 处理相同分数下的字典序分页 - if last_score ~= '' and last_member ~= '' then +-- 调整range_start，当有last_score且没有last_member时 +if last_score ~= '' and last_member == '' then + if command == 'ZRANGEBYSCORE' then + range_start = '('..last_score + else + range_start = '('..last_score + end +end + +local members +if command == 'ZRANGEBYSCORE' then + members = redis.call(command, KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit) +else + members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit) +end + +if #members == 0 then return {0,{},{},{}} end + +local result_members = {} +local result_scores = {} + +-- 如果有last_member，进行过滤 +if last_member ~= '' and last_score ~= '' then + for i = 1, #members, 2 do + local member = members[i] + local score = members[i+1] + local include = true + + if score == last_score then if command == 'ZRANGEBYSCORE' then - range_start = '(' .. last_score - range_end = '(' .. last_score .. ' ' .. last_member + -- 升序：member必须 > last_member + if member <= last_member then + include = false + end else - range_start = '(' .. last_score .. ' ' .. last_member - range_end = '(' .. last_score + -- 降序：member必须 < last_member + if member >= last_member then + include = false + end end end - -- 执行范围查询 - local members - if command == 'ZRANGEBYSCORE' then - members = redis.call(command, KEYS[1], range_start, range_end, - 'WITHSCORES', 'LIMIT', 0, limit) - else - members = redis.call(command, KEYS[1], range_end, range_start, - 'WITHSCORES', 'LIMIT', 0, limit) + if include then + table.insert(result_members, member) + table.insert(result_scores, score) end + end +else + for i = 1, #members, 2 do + table.insert(result_members, members[i]) + table.insert(result_scores, members[i+1]) + end +end - -- 提取成员和分数 - local result_members = {} - local result_scores = {} - for i = 1, #members, 2 do - table.insert(result_members, members[i]) - table.insert(result_scores, members[i+1]) - end +-- 截取前limit条 +local count = #result_members +if count > limit then + result_members = {unpack(result_members, 1, limit)} + result_scores = {unpack(result_scores, 1, limit)} +end - -- 获取Hash数据 - local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members)) +if #result_members == 0 then + return {0, {}, {}, {}} +end - return { - #result_members, - result_members, - result_scores, - hash_data - }"; +-- 获取Hash数据 +local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members)) +return {#result_members, result_members, result_scores, hash_data}"; - //正确设置范围参数 + // 修复点：根据是否传递lastMember决定rangeStart是否排他 string rangeStart, rangeEnd; if (descending) { - rangeStart = lastScore.HasValue ? $"({lastScore}" : "+inf"; - rangeEnd = "-inf"; // 降序时固定为最小值 + rangeStart = lastScore.HasValue + ? (string.IsNullOrEmpty(lastMember) ? "(" + lastScore.Value.ToString() : lastScore.Value.ToString()) + : "+inf"; + rangeEnd = "-inf"; } else { - rangeStart = lastScore.HasValue ? $"({lastScore}" : "-inf"; - rangeEnd = "+inf"; // 升序时固定为最大值 + rangeStart = lastScore.HasValue + ? (string.IsNullOrEmpty(lastMember) ? "(" + lastScore.Value.ToString() : lastScore.Value.ToString()) + : "-inf"; + rangeEnd = "+inf"; } - var result = (object[])await Instance.EvalAsync( - luaScript, + // 执行Lua脚本（保持不变） + var scriptResult = (object[])await Instance.EvalAsync(luaScript, new[] { redisZSetScoresIndexCacheKey, redisHashCacheKey }, new object[] { - descending ? "ZREVRANGEBYSCORE" : "ZRANGEBYSCORE", - rangeStart, - rangeEnd, - (pageSize + 1).ToString(), // 多取1条用于判断hasNext - lastScore?.ToString() ?? "", - lastMember ?? "" + descending ? "ZREVRANGEBYSCORE" : "ZRANGEBYSCORE", + rangeStart, + rangeEnd, + (pageSize + 1).ToString(), + lastScore?.ToString() ?? "", + lastMember ?? "" }); - if ((long)result[0] == 0) + // 处理空结果（保持不变） + if ((long)scriptResult[0] == 0) return new BusCacheGlobalPagedResult { Items = new List() }; - // 处理结果集 - var members = ((object[])result[1]).Cast().ToList(); - var scores = ((object[])result[2]).Cast().Select(decimal.Parse).ToList(); - var hashData = ((object[])result[3]).Cast().ToList(); + // 数据提取（保持不变） + var members = ((object[])scriptResult[1]).Cast().ToList(); + var scores = ((object[])scriptResult[2]).Cast().Select(decimal.Parse).ToList(); + var hashData = ((object[])scriptResult[3]).Cast().ToList(); - //合并有效数据并处理游标 - var validItems = members.Zip(hashData, (m, h) => - !string.IsNullOrWhiteSpace(h) ? BusJsonSerializer.Deserialize(h) : null) - .Where(x => x != null) - .Take(pageSize + 1) - .ToList(); + // 反序列化处理（保持不变） + var validItems = members.Select((m, i) => + { + try + { + return !string.IsNullOrEmpty(hashData[i]) + ? BusJsonSerializer.Deserialize(hashData[i]) + : null; + } + catch (Exception ex) + { + _logger.LogError($"反序列化失败: {m} - {ex.Message}"); + return null; + } + }).Where(x => x != null).Take(pageSize + 1).ToList(); + // 分页逻辑（保持不变） var hasNext = validItems.Count > pageSize; var actualItems = hasNext ? validItems.Take(pageSize) : validItems; - // 计算下一页起始点 - string nextMember = null; + // 计算下一页锚点（保持不变） decimal? nextScore = null; - if (hasNext) + string nextMember = null; + if (hasNext && actualItems.Any()) { - // 获取实际返回的最后一条有效数据 - var lastValidIndex = actualItems.Count() - 1; - nextMember = members[lastValidIndex]; - nextScore = scores[lastValidIndex]; + var lastIndex = Math.Min(members.Count - 1, pageSize); + nextScore = scores[lastIndex]; + nextMember = members[lastIndex]; } return new BusCacheGlobalPagedResult diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index ce4635b..4f8ae1f 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -213,50 +213,50 @@ namespace JiShe.CollectBus.ScheduledMeterReading public virtual async Task InitAmmeterCacheData(string gatherCode = "") { #if DEBUG - //var timeDensity = "15"; - //string tempCacheMeterInfoKey = $"CollectBus:{"{0}:{1}"}:MeterInfo:{"{2}"}:{"{3}"}"; - ////获取缓存中的电表信息 - //var redisKeyList = $"{string.Format(tempCacheMeterInfoKey, SystemType, "JiSheCollectBus", MeterTypeEnum.Ammeter, timeDensity)}*"; + var timeDensity = "15"; + string tempCacheMeterInfoKey = $"CollectBus:{"{0}:{1}"}:MeterInfo:{"{2}"}:{"{3}"}"; + //获取缓存中的电表信息 + var redisKeyList = $"{string.Format(tempCacheMeterInfoKey, SystemType, "JiSheCollectBus", MeterTypeEnum.Ammeter, timeDensity)}*"; - //var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); - //var tempMeterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); - ////List focusAddressDataLista = new List(); - //List meterInfos = new List(); - //foreach (var item in tempMeterInfos) - //{ - // var tempData = item.Adapt(); - // tempData.FocusId = item.FocusID; - // tempData.MeterId = item.Id; - // meterInfos.Add(tempData); - // //focusAddressDataLista.Add(item.FocusAddress); - //} + var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); + var tempMeterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); + //List focusAddressDataLista = new List(); + List meterInfos = new List(); + foreach (var item in tempMeterInfos) + { + var tempData = item.Adapt(); + tempData.FocusId = item.FocusID; + tempData.MeterId = item.Id; + meterInfos.Add(tempData); + //focusAddressDataLista.Add(item.FocusAddress); + } //DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); - var timeDensity = "15"; - var redisCacheMeterInfoHashKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; - var redisCacheMeterInfoSetIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; - var redisCacheMeterInfoZSetScoresIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + //var timeDensity = "15"; + //var redisCacheMeterInfoHashKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + //var redisCacheMeterInfoSetIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + //var redisCacheMeterInfoZSetScoresIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; - decimal? cursor = null; - string member = null; - bool hasNext; - List meterInfos = new List(); - do - { - var page = await _redisDataCacheService.GetAllPagedData( - redisCacheMeterInfoHashKeyTemp, - redisCacheMeterInfoZSetScoresIndexKeyTemp, - pageSize: 1000, - lastScore: cursor, - lastMember: member); + //decimal? cursor = null; + //string member = null; + //bool hasNext; + //List meterInfos = new List(); + //do + //{ + // var page = await _redisDataCacheService.GetAllPagedData( + // redisCacheMeterInfoHashKeyTemp, + // redisCacheMeterInfoZSetScoresIndexKeyTemp, + // pageSize: 1000, + // lastScore: cursor, + // lastMember: member); - meterInfos.AddRange(page.Items); - cursor = page.HasNext ? page.NextScore : null; - member = page.HasNext ? page.NextMember : null; - hasNext = page.HasNext; - } while (hasNext); + // meterInfos.AddRange(page.Items); + // cursor = page.HasNext ? page.NextScore : null; + // member = page.HasNext ? page.NextMember : null; + // hasNext = page.HasNext; + //} while (hasNext); #else var meterInfos = await GetAmmeterInfoList(gatherCode); diff --git a/src/JiShe.CollectBus.Common/Extensions/EnumerableExtensions.cs b/src/JiShe.CollectBus.Common/Extensions/EnumerableExtensions.cs index 80d2f23..b17e9c2 100644 --- a/src/JiShe.CollectBus.Common/Extensions/EnumerableExtensions.cs +++ b/src/JiShe.CollectBus.Common/Extensions/EnumerableExtensions.cs @@ -89,5 +89,26 @@ namespace JiShe.CollectBus.Common.Extensions if (buffer.Count > 0) yield return buffer; } + + //public static IEnumerable> Batch(this IEnumerable source, int batchSize) + //{ + // if (batchSize <= 0) + // throw new ArgumentOutOfRangeException(nameof(batchSize)); + + // using var enumerator = source.GetEnumerator(); + // while (enumerator.MoveNext()) + // { + // yield return GetBatch(enumerator, batchSize); + // } + //} + + //private static IEnumerable GetBatch(IEnumerator enumerator, int batchSize) + //{ + // do + // { + // yield return enumerator.Current; + // batchSize--; + // } while (batchSize > 0 && enumerator.MoveNext()); + //} } } diff --git a/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs b/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs index 4ac667d..3df01b5 100644 --- a/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs +++ b/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs @@ -1,4 +1,5 @@ -using JiShe.CollectBus.Common.Models; +using FreeSql.DataAnnotations; +using JiShe.CollectBus.Common.Models; using System; using System.Collections.Generic; using System.Linq; @@ -8,7 +9,19 @@ using System.Threading.Tasks; namespace JiShe.CollectBus.Ammeters { public class AmmeterInfo: DeviceCacheBasicModel - { + { + /// + /// 关系映射标识，用于ZSet的Member字段和Set的Value字段，具体值可以根据不同业务场景进行定义 + /// + [Column(IsIgnore = true)] + public override string MemberID => $"{FocusId}:{MeterId}"; + + /// + /// ZSet排序索引分数值，具体值可以根据不同业务场景进行定义，例如时间戳 + /// + [Column(IsIgnore = true)] + public override long ScoreValue => ((long)FocusId << 32) | (uint)DateTime.Now.Ticks; + /// /// 电表名称 /// diff --git a/src/JiShe.CollectBus.Host/appsettings.json b/src/JiShe.CollectBus.Host/appsettings.json index bb09204..ae2025b 100644 --- a/src/JiShe.CollectBus.Host/appsettings.json +++ b/src/JiShe.CollectBus.Host/appsettings.json @@ -129,7 +129,7 @@ "OpenDebugMode": true, "UseTableSessionPoolByDefault": false }, - "ServerTagName": "JiSheCollectBus3", + "ServerTagName": "JiSheCollectBus2", "KafkaReplicationFactor": 3, "NumPartitions": 30 } \ No newline at end of file From 78127f7cea9975aa131ba007cf6b3d46ef6b9494 Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Thu, 17 Apr 2025 08:20:54 +0800 Subject: [PATCH 14/27] =?UTF-8?q?=E8=A7=A3=E5=86=B3Redis=E5=A4=A7=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E6=95=B0=E6=8D=AE=E8=AF=BB=E5=8F=96=E6=80=BB=E6=95=B0?= =?UTF-8?q?=E4=B8=8D=E5=87=86=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C=E5=B9=B6?= =?UTF-8?q?=E5=B0=8610=E4=B8=87=E8=AE=B0=E5=BD=95=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=8E=A7=E5=88=B6=E5=9C=A823=E7=A7=92?= =?UTF-8?q?=E4=BB=A5=E5=86=85=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RedisDataCache/RedisDataCacheService.cs | 115 +++++++----------- .../BasicScheduledMeterReadingService.cs | 77 ++++++------ src/JiShe.CollectBus.Host/appsettings.json | 2 +- 3 files changed, 87 insertions(+), 107 deletions(-) diff --git a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs index 8f50017..e846ada 100644 --- a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs +++ b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs @@ -385,7 +385,7 @@ namespace JiShe.CollectBus.RedisDataCache bool descending = true) where T : DeviceCacheBasicModel { - // 参数校验（保持不变） + // 参数校验 if (string.IsNullOrWhiteSpace(redisHashCacheKey) || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) { @@ -396,7 +396,6 @@ namespace JiShe.CollectBus.RedisDataCache if (pageSize < 1 || pageSize > 10000) throw new ArgumentException("分页大小应在1-10000之间"); - // 更新后的Lua脚本 var luaScript = @" local command = ARGV[1] local range_start = ARGV[2] @@ -405,93 +404,74 @@ local limit = tonumber(ARGV[4]) local last_score = ARGV[5] local last_member = ARGV[6] --- 调整range_start，当有last_score且没有last_member时 -if last_score ~= '' and last_member == '' then - if command == 'ZRANGEBYSCORE' then - range_start = '('..last_score - else - range_start = '('..last_score - end -end - +-- 获取原始数据 local members if command == 'ZRANGEBYSCORE' then - members = redis.call(command, KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit) + members = redis.call(command, KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit * 2) else - members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit) + members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit * 2) end -if #members == 0 then return {0,{},{},{}} end - -local result_members = {} -local result_scores = {} - --- 如果有last_member，进行过滤 -if last_member ~= '' and last_score ~= '' then - for i = 1, #members, 2 do - local member = members[i] - local score = members[i+1] - local include = true - - if score == last_score then - if command == 'ZRANGEBYSCORE' then - -- 升序：member必须 > last_member - if member <= last_member then - include = false - end +-- 过滤数据 +local filtered_members = {} +local count = 0 +for i = 1, #members, 2 do + local member = members[i] + local score = members[i+1] + local include = true + if last_score ~= '' and last_member ~= '' then + if command == 'ZRANGEBYSCORE' then + -- 升序：score > last_score 或 (score == last_score 且 member > last_member) + if score == last_score then + include = member > last_member else - -- 降序：member必须 < last_member - if member >= last_member then - include = false - end + include = tonumber(score) > tonumber(last_score) + end + else + -- 降序：score < last_score 或 (score == last_score 且 member < last_member) + if score == last_score then + include = member < last_member + else + include = tonumber(score) < tonumber(last_score) end end - - if include then - table.insert(result_members, member) - table.insert(result_scores, score) + end + if include then + table.insert(filtered_members, member) + table.insert(filtered_members, score) + count = count + 1 + if count >= limit then + break end end -else - for i = 1, #members, 2 do - table.insert(result_members, members[i]) - table.insert(result_scores, members[i+1]) - end end --- 截取前limit条 -local count = #result_members -if count > limit then - result_members = {unpack(result_members, 1, limit)} - result_scores = {unpack(result_scores, 1, limit)} +-- 提取有效数据 +local result_members, result_scores = {}, {} +for i=1,#filtered_members,2 do + table.insert(result_members, filtered_members[i]) + table.insert(result_scores, filtered_members[i+1]) end -if #result_members == 0 then - return {0, {}, {}, {}} -end +if #result_members == 0 then return {0,{},{},{}} end -- 获取Hash数据 local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members)) return {#result_members, result_members, result_scores, hash_data}"; - // 修复点：根据是否传递lastMember决定rangeStart是否排他 + // 调整范围构造逻辑（移除排他符号） string rangeStart, rangeEnd; if (descending) { - rangeStart = lastScore.HasValue - ? (string.IsNullOrEmpty(lastMember) ? "(" + lastScore.Value.ToString() : lastScore.Value.ToString()) - : "+inf"; + rangeStart = lastScore.HasValue ? lastScore.Value.ToString() : "+inf"; rangeEnd = "-inf"; } else { - rangeStart = lastScore.HasValue - ? (string.IsNullOrEmpty(lastMember) ? "(" + lastScore.Value.ToString() : lastScore.Value.ToString()) - : "-inf"; + rangeStart = lastScore.HasValue ? lastScore.Value.ToString() : "-inf"; rangeEnd = "+inf"; } - // 执行Lua脚本（保持不变） var scriptResult = (object[])await Instance.EvalAsync(luaScript, new[] { redisZSetScoresIndexCacheKey, redisHashCacheKey }, new object[] @@ -499,21 +479,19 @@ return {#result_members, result_members, result_scores, hash_data}"; descending ? "ZREVRANGEBYSCORE" : "ZRANGEBYSCORE", rangeStart, rangeEnd, - (pageSize + 1).ToString(), + (pageSize + 1).ToString(), // 获取pageSize+1条以判断是否有下一页 lastScore?.ToString() ?? "", lastMember ?? "" }); - // 处理空结果（保持不变） if ((long)scriptResult[0] == 0) return new BusCacheGlobalPagedResult { Items = new List() }; - // 数据提取（保持不变） + // 处理结果集 var members = ((object[])scriptResult[1]).Cast().ToList(); var scores = ((object[])scriptResult[2]).Cast().Select(decimal.Parse).ToList(); var hashData = ((object[])scriptResult[3]).Cast().ToList(); - // 反序列化处理（保持不变） var validItems = members.Select((m, i) => { try @@ -527,18 +505,17 @@ return {#result_members, result_members, result_scores, hash_data}"; _logger.LogError($"反序列化失败: {m} - {ex.Message}"); return null; } - }).Where(x => x != null).Take(pageSize + 1).ToList(); + }).Where(x => x != null).ToList(); - // 分页逻辑（保持不变） var hasNext = validItems.Count > pageSize; var actualItems = hasNext ? validItems.Take(pageSize) : validItems; - // 计算下一页锚点（保持不变） + // 修正分页锚点索引 decimal? nextScore = null; string nextMember = null; if (hasNext && actualItems.Any()) { - var lastIndex = Math.Min(members.Count - 1, pageSize); + var lastIndex = actualItems.Count() - 1; // 使用actualItems的最后一个索引 nextScore = scores[lastIndex]; nextMember = members[lastIndex]; } @@ -551,7 +528,7 @@ return {#result_members, result_members, result_scores, hash_data}"; NextMember = nextMember, TotalCount = await GetTotalCount(redisZSetScoresIndexCacheKey), PageSize = pageSize, - }; + }; } ///// diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index 4f8ae1f..d32cdb2 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -213,50 +213,53 @@ namespace JiShe.CollectBus.ScheduledMeterReading public virtual async Task InitAmmeterCacheData(string gatherCode = "") { #if DEBUG - var timeDensity = "15"; - string tempCacheMeterInfoKey = $"CollectBus:{"{0}:{1}"}:MeterInfo:{"{2}"}:{"{3}"}"; - //获取缓存中的电表信息 - var redisKeyList = $"{string.Format(tempCacheMeterInfoKey, SystemType, "JiSheCollectBus", MeterTypeEnum.Ammeter, timeDensity)}*"; + //var timeDensity = "15"; + //string tempCacheMeterInfoKey = $"CollectBus:{"{0}:{1}"}:MeterInfo:{"{2}"}:{"{3}"}"; + ////获取缓存中的电表信息 + //var redisKeyList = $"{string.Format(tempCacheMeterInfoKey, SystemType, "JiSheCollectBus", MeterTypeEnum.Ammeter, timeDensity)}*"; - var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); - var tempMeterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); - //List focusAddressDataLista = new List(); - List meterInfos = new List(); - foreach (var item in tempMeterInfos) - { - var tempData = item.Adapt(); - tempData.FocusId = item.FocusID; - tempData.MeterId = item.Id; - meterInfos.Add(tempData); - //focusAddressDataLista.Add(item.FocusAddress); - } + //var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); + //var tempMeterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, timeDensity, MeterTypeEnum.Ammeter); + ////List focusAddressDataLista = new List(); + //List meterInfos = new List(); + //foreach (var item in tempMeterInfos) + //{ + // var tempData = item.Adapt(); + // tempData.FocusId = item.FocusID; + // tempData.MeterId = item.Id; + // meterInfos.Add(tempData); + // //focusAddressDataLista.Add(item.FocusAddress); + //} //DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); - //var timeDensity = "15"; - //var redisCacheMeterInfoHashKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; - //var redisCacheMeterInfoSetIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; - //var redisCacheMeterInfoZSetScoresIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + var timeDensity = "15"; + var redisCacheMeterInfoHashKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + var redisCacheMeterInfoSetIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + var redisCacheMeterInfoZSetScoresIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; + var timer1 = Stopwatch.StartNew(); + decimal? cursor = null; + string member = null; + bool hasNext; + List meterInfos = new List(); + do + { + var page = await _redisDataCacheService.GetAllPagedData( + redisCacheMeterInfoHashKeyTemp, + redisCacheMeterInfoZSetScoresIndexKeyTemp, + pageSize: 1000, + lastScore: cursor, + lastMember: member); - //decimal? cursor = null; - //string member = null; - //bool hasNext; - //List meterInfos = new List(); - //do - //{ - // var page = await _redisDataCacheService.GetAllPagedData( - // redisCacheMeterInfoHashKeyTemp, - // redisCacheMeterInfoZSetScoresIndexKeyTemp, - // pageSize: 1000, - // lastScore: cursor, - // lastMember: member); + meterInfos.AddRange(page.Items); + cursor = page.HasNext ? page.NextScore : null; + member = page.HasNext ? page.NextMember : null; + hasNext = page.HasNext; + } while (hasNext); - // meterInfos.AddRange(page.Items); - // cursor = page.HasNext ? page.NextScore : null; - // member = page.HasNext ? page.NextMember : null; - // hasNext = page.HasNext; - //} while (hasNext); + timer1.Stop(); + _logger.LogError($"读取数据更花费时间{timer1.ElapsedMilliseconds}毫秒"); #else var meterInfos = await GetAmmeterInfoList(gatherCode); diff --git a/src/JiShe.CollectBus.Host/appsettings.json b/src/JiShe.CollectBus.Host/appsettings.json index ae2025b..bb09204 100644 --- a/src/JiShe.CollectBus.Host/appsettings.json +++ b/src/JiShe.CollectBus.Host/appsettings.json @@ -129,7 +129,7 @@ "OpenDebugMode": true, "UseTableSessionPoolByDefault": false }, - "ServerTagName": "JiSheCollectBus2", + "ServerTagName": "JiSheCollectBus3", "KafkaReplicationFactor": 3, "NumPartitions": 30 } \ No newline at end of file From cadb69f4b97ed27c46890319904789705431fd5d Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Thu, 17 Apr 2025 08:29:32 +0800 Subject: [PATCH 15/27] =?UTF-8?q?=E5=B0=8610=E4=B8=87Redis=E7=9A=84Hash?= =?UTF-8?q?=E8=A1=A8=E6=95=B0=E6=8D=AE=E8=AF=BB=E5=8F=96=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E5=88=B013=E7=A7=92=E4=BB=A5=E5=86=85=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RedisDataCache/RedisDataCacheService.cs | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs index e846ada..67a435d 100644 --- a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs +++ b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs @@ -492,20 +492,14 @@ return {#result_members, result_members, result_scores, hash_data}"; var scores = ((object[])scriptResult[2]).Cast().Select(decimal.Parse).ToList(); var hashData = ((object[])scriptResult[3]).Cast().ToList(); - var validItems = members.Select((m, i) => + var validItems = members.AsParallel() + .Select((m, i) => { - try - { - return !string.IsNullOrEmpty(hashData[i]) - ? BusJsonSerializer.Deserialize(hashData[i]) - : null; - } - catch (Exception ex) - { - _logger.LogError($"反序列化失败: {m} - {ex.Message}"); - return null; - } - }).Where(x => x != null).ToList(); + try { return BusJsonSerializer.Deserialize(hashData[i]); } + catch { return null; } + }) + .Where(x => x != null) + .ToList(); var hasNext = validItems.Count > pageSize; var actualItems = hasNext ? validItems.Take(pageSize) : validItems; From 4737c5b53d2a23240c73d9591a0e6fc4b26cd9ba Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Thu, 17 Apr 2025 09:10:01 +0800 Subject: [PATCH 16/27] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RedisDataCache/RedisDataCacheService.cs | 133 +++++++++--------- 1 file changed, 68 insertions(+), 65 deletions(-) diff --git a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs index 67a435d..a634789 100644 --- a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs +++ b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs @@ -7,6 +7,7 @@ using JiShe.CollectBus.Common.Models; using JiShe.CollectBus.FreeRedisProvider; using Microsoft.Extensions.Logging; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -121,7 +122,7 @@ namespace JiShe.CollectBus.RedisDataCache await semaphore.WaitAsync(); _ = Task.Run(() => - { + { using (var pipe = Instance.StartPipe()) { foreach (var item in batch) @@ -363,7 +364,7 @@ namespace JiShe.CollectBus.RedisDataCache where T : DeviceCacheBasicModel { throw new Exception(); - } + } /// /// 通过ZSET索引获取数据 @@ -385,79 +386,81 @@ namespace JiShe.CollectBus.RedisDataCache bool descending = true) where T : DeviceCacheBasicModel { - // 参数校验 - if (string.IsNullOrWhiteSpace(redisHashCacheKey) || - string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + // 参数校验增强 + if (string.IsNullOrWhiteSpace(redisHashCacheKey) || string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) { - _logger.LogError("参数异常: HashKey或ZSetKey为空"); + _logger.LogError($"{nameof(GetAllPagedData)} 参数异常，-101"); return null; } if (pageSize < 1 || pageSize > 10000) - throw new ArgumentException("分页大小应在1-10000之间"); + { + _logger.LogError($"{nameof(GetAllPagedData)} 分页大小应在1-10000之间，-102"); + return null; + } var luaScript = @" -local command = ARGV[1] -local range_start = ARGV[2] -local range_end = ARGV[3] -local limit = tonumber(ARGV[4]) -local last_score = ARGV[5] -local last_member = ARGV[6] + local command = ARGV[1] + local range_start = ARGV[2] + local range_end = ARGV[3] + local limit = tonumber(ARGV[4]) + local last_score = ARGV[5] + local last_member = ARGV[6] --- 获取原始数据 -local members -if command == 'ZRANGEBYSCORE' then - members = redis.call(command, KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit * 2) -else - members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit * 2) -end + -- 获取原始数据 + local members + if command == 'ZRANGEBYSCORE' then + members = redis.call(command, KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit * 2) + else + members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, 'WITHSCORES', 'LIMIT', 0, limit * 2) + end --- 过滤数据 -local filtered_members = {} -local count = 0 -for i = 1, #members, 2 do - local member = members[i] - local score = members[i+1] - local include = true - if last_score ~= '' and last_member ~= '' then - if command == 'ZRANGEBYSCORE' then - -- 升序：score > last_score 或 (score == last_score 且 member > last_member) - if score == last_score then - include = member > last_member - else - include = tonumber(score) > tonumber(last_score) - end - else - -- 降序：score < last_score 或 (score == last_score 且 member < last_member) - if score == last_score then - include = member < last_member - else - include = tonumber(score) < tonumber(last_score) - end - end - end - if include then - table.insert(filtered_members, member) - table.insert(filtered_members, score) - count = count + 1 - if count >= limit then - break - end - end -end + -- 过滤数据 + local filtered_members = {} + local count = 0 + for i = 1, #members, 2 do + local member = members[i] + local score = members[i+1] + local include = true + if last_score ~= '' and last_member ~= '' then + if command == 'ZRANGEBYSCORE' then + -- 升序：score > last_score 或 (score == last_score 且 member > last_member) + if score == last_score then + include = member > last_member + else + include = tonumber(score) > tonumber(last_score) + end + else + -- 降序：score < last_score 或 (score == last_score 且 member < last_member) + if score == last_score then + include = member < last_member + else + include = tonumber(score) < tonumber(last_score) + end + end + end + if include then + table.insert(filtered_members, member) + table.insert(filtered_members, score) + count = count + 1 + if count >= limit then + break + end + end + end --- 提取有效数据 -local result_members, result_scores = {}, {} -for i=1,#filtered_members,2 do - table.insert(result_members, filtered_members[i]) - table.insert(result_scores, filtered_members[i+1]) -end + -- 提取有效数据 + local result_members, result_scores = {}, {} + for i=1,#filtered_members,2 do + table.insert(result_members, filtered_members[i]) + table.insert(result_scores, filtered_members[i+1]) + end -if #result_members == 0 then return {0,{},{},{}} end + if #result_members == 0 then return {0,{},{},{}} end --- 获取Hash数据 -local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members)) -return {#result_members, result_members, result_scores, hash_data}"; + -- 获取Hash数据 + local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members)) + return {#result_members, result_members, result_scores, hash_data}"; // 调整范围构造逻辑（移除排他符号） string rangeStart, rangeEnd; @@ -504,7 +507,7 @@ return {#result_members, result_members, result_scores, hash_data}"; var hasNext = validItems.Count > pageSize; var actualItems = hasNext ? validItems.Take(pageSize) : validItems; - // 修正分页锚点索引 + //分页锚点索引 decimal? nextScore = null; string nextMember = null; if (hasNext && actualItems.Any()) @@ -522,7 +525,7 @@ return {#result_members, result_members, result_scores, hash_data}"; NextMember = nextMember, TotalCount = await GetTotalCount(redisZSetScoresIndexCacheKey), PageSize = pageSize, - }; + }; } ///// From 13e986168e5dd00e04270ba6b94062bbad744cfa Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Thu, 17 Apr 2025 11:29:26 +0800 Subject: [PATCH 17/27] =?UTF-8?q?=E7=99=BE=E4=B8=87=E7=BA=A7=E7=9A=84?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=95=B0=E6=8D=AE=E9=9B=86=E4=B8=AD=E5=99=A8?= =?UTF-8?q?=E6=89=80=E5=9C=A8=E5=88=86=E7=BB=84=E5=8D=95=E7=8B=AC=E4=BF=9D?= =?UTF-8?q?=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RedisDataCache/IRedisDataCacheService.cs | 57 ++-- .../RedisDataCache/RedisDataCacheService.cs | 263 +++++++++++++++++- .../Samples/SampleAppService.cs | 25 +- .../BasicScheduledMeterReadingService.cs | 154 ++++++---- .../Consts/RedisConst.cs | 14 +- .../DeviceGroupBalanceControl.cs | 12 +- .../Models/DeviceCacheBasicModel.cs | 2 +- .../Ammeters/AmmeterInfo.cs | 2 +- .../MeterReadingTelemetryPacketInfo.cs | 141 ++++++++++ 9 files changed, 558 insertions(+), 112 deletions(-) create mode 100644 src/JiShe.CollectBus.Domain/IotSystems/MeterReadingRecords/MeterReadingTelemetryPacketInfo.cs diff --git a/src/JiShe.CollectBus.Application.Contracts/RedisDataCache/IRedisDataCacheService.cs b/src/JiShe.CollectBus.Application.Contracts/RedisDataCache/IRedisDataCacheService.cs index c6f3613..5f0e8f8 100644 --- a/src/JiShe.CollectBus.Application.Contracts/RedisDataCache/IRedisDataCacheService.cs +++ b/src/JiShe.CollectBus.Application.Contracts/RedisDataCache/IRedisDataCacheService.cs @@ -113,7 +113,7 @@ namespace JiShe.CollectBus.Application.Contracts /// - /// 通过ZSET索引获取数据 + /// 通过ZSET索引获取数据，支持10万级别数据处理，控制在13秒以内。 /// /// /// 主数据存储Hash缓存Key @@ -133,6 +133,17 @@ namespace JiShe.CollectBus.Application.Contracts where T : DeviceCacheBasicModel; + /// + /// 优化后的分页获取方法（支持百万级数据） + /// + Task> GetAllPagedDataOptimized( + string redisHashCacheKey, + string redisZSetScoresIndexCacheKey, + int pageSize = 1000, + decimal? lastScore = null, + string lastMember = null, + bool descending = true) where T : DeviceCacheBasicModel; + ///// ///// 游标分页查询 ///// @@ -149,28 +160,28 @@ namespace JiShe.CollectBus.Application.Contracts // string excludeMember, // bool descending); - ///// - ///// 批量获取指定分页的数据 - ///// - ///// - ///// Hash表缓存key - ///// Hash表字段集合 - ///// - //Task> BatchGetData( - // string redisHashCacheKey, - // IEnumerable members) - // where T : DeviceCacheBasicModel; + ///// + ///// 批量获取指定分页的数据 + ///// + ///// + ///// Hash表缓存key + ///// Hash表字段集合 + ///// + //Task> BatchGetData( + // string redisHashCacheKey, + // IEnumerable members) + // where T : DeviceCacheBasicModel; - ///// - ///// 获取下一页游标 - ///// - ///// 排序索引ZSET缓存Key - ///// 最后一个唯一标识 - ///// 排序方式 - ///// - //Task GetNextScore( - // string redisZSetScoresIndexCacheKey, - // string lastMember, - // bool descending); + ///// + ///// 获取下一页游标 + ///// + ///// 排序索引ZSET缓存Key + ///// 最后一个唯一标识 + ///// 排序方式 + ///// + //Task GetNextScore( + // string redisZSetScoresIndexCacheKey, + // string lastMember, + // bool descending); } } diff --git a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs index a634789..3c96410 100644 --- a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs +++ b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs @@ -70,13 +70,13 @@ namespace JiShe.CollectBus.RedisDataCache using (var trans = Instance.Multi()) { // 主数据存储Hash - trans.HSet(redisHashCacheKey, data.MemberID, data.Serialize()); + trans.HSet(redisHashCacheKey, data.MemberId, data.Serialize()); // 集中器号分组索引Set缓存 - trans.SAdd(redisSetIndexCacheKey, data.MemberID); + trans.SAdd(redisSetIndexCacheKey, data.MemberId); // 集中器与表计信息排序索引ZSET缓存Key - trans.ZAdd(redisZSetScoresIndexCacheKey, data.ScoreValue, data.MemberID); + trans.ZAdd(redisZSetScoresIndexCacheKey, data.ScoreValue, data.MemberId); var results = trans.Exec(); @@ -128,13 +128,13 @@ namespace JiShe.CollectBus.RedisDataCache foreach (var item in batch) { // 主数据存储Hash - pipe.HSet(redisHashCacheKey, item.MemberID, item.Serialize()); + pipe.HSet(redisHashCacheKey, item.MemberId, item.Serialize()); // Set索引缓存 - pipe.SAdd(redisSetIndexCacheKey, item.MemberID); + pipe.SAdd(redisSetIndexCacheKey, item.MemberId); // ZSET索引缓存Key - pipe.ZAdd(redisZSetScoresIndexCacheKey, item.ScoreValue, item.MemberID); + pipe.ZAdd(redisZSetScoresIndexCacheKey, item.ScoreValue, item.MemberId); } pipe.EndPipe(); } @@ -192,11 +192,11 @@ namespace JiShe.CollectBus.RedisDataCache redisZSetScoresIndexCacheKey }; - var result = await Instance.EvalAsync(luaScript, keys, new[] { data.MemberID }); + var result = await Instance.EvalAsync(luaScript, keys, new[] { data.MemberId }); if ((int)result == 0) { - _logger.LogError($"{nameof(RemoveCacheDataAsync)} 删除指定Key{redisHashCacheKey}的{data.MemberID}数据失败,-102"); + _logger.LogError($"{nameof(RemoveCacheDataAsync)} 删除指定Key{redisHashCacheKey}的{data.MemberId}数据失败,-102"); } } @@ -248,13 +248,13 @@ namespace JiShe.CollectBus.RedisDataCache }, new object[] { - newData.MemberID, + newData.MemberId, newData.Serialize() }); if ((int)result == 0) { - _logger.LogError($"{nameof(ModifyDataAsync)} 更新指定Key{redisHashCacheKey}的{newData.MemberID}数据失败,-102"); + _logger.LogError($"{nameof(ModifyDataAsync)} 更新指定Key{redisHashCacheKey}的{newData.MemberId}数据失败,-102"); } } @@ -328,7 +328,7 @@ namespace JiShe.CollectBus.RedisDataCache }, new object[] { - newData.MemberID, + newData.MemberId, oldMemberId, newData.Serialize(), newData.ScoreValue.ToString() ?? "", @@ -336,7 +336,7 @@ namespace JiShe.CollectBus.RedisDataCache if ((int)result == 0) { - _logger.LogError($"{nameof(ModifyDataAsync)} 更新指定Key{redisHashCacheKey}的{newData.MemberID}数据失败,-102"); + _logger.LogError($"{nameof(ModifyDataAsync)} 更新指定Key{redisHashCacheKey}的{newData.MemberId}数据失败,-102"); } } @@ -364,10 +364,245 @@ namespace JiShe.CollectBus.RedisDataCache where T : DeviceCacheBasicModel { throw new Exception(); - } + } + /// - /// 通过ZSET索引获取数据 + /// 优化后的分页获取方法（支持百万级数据） + /// + public async Task> GetAllPagedDataOptimized( + string redisHashCacheKey, + string redisZSetScoresIndexCacheKey, + int pageSize = 1000, + decimal? lastScore = null, + string lastMember = null, + bool descending = true) where T : DeviceCacheBasicModel + { + // 参数校验 + if (string.IsNullOrWhiteSpace(redisHashCacheKey) || + string.IsNullOrWhiteSpace(redisZSetScoresIndexCacheKey)) + { + _logger.LogError("Invalid parameters in {Method}", nameof(GetAllPagedDataOptimized)); + return new BusCacheGlobalPagedResult { Items = new List() }; + } + + pageSize = Math.Clamp(pageSize, 1, 10000); + + const string luaScript = @" + local command = ARGV[1] + local range_start = ARGV[2] + local range_end = ARGV[3] + local limit = tonumber(ARGV[4]) + local last_score = ARGV[5] + local last_member = ARGV[6] + + -- 获取扩展数据（3倍分页大小） + local members + if command == 'ZRANGEBYSCORE' then + members = redis.call('ZRANGEBYSCORE', KEYS[1], range_start, range_end, + 'WITHSCORES', 'LIMIT', 0, limit * 5) + else + members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, + 'WITHSCORES', 'LIMIT', 0, limit * 5) + end + + -- 精确分页过滤 + local filtered = {} + local count = 0 + local start_index = 1 + + -- 存在锚点时寻找起始位置 + if last_score ~= '' and last_member ~= '' then + for i=1,#members,2 do + local score = members[i+1] + local member = members[i] + + if command == 'ZRANGEBYSCORE' then + if tonumber(score) > tonumber(last_score) then + start_index = i + break + elseif tonumber(score) == tonumber(last_score) then + if member > last_member then + start_index = i + break + end + end + else + if tonumber(score) < tonumber(last_score) then + start_index = i + break + elseif tonumber(score) == tonumber(last_score) then + if member < last_member then + start_index = i + break + end + end + end + end + end + + -- 收集有效数据 + for i=start_index,#members,2 do + if count >= limit then break end + table.insert(filtered, members[i]) + table.insert(filtered, members[i+1]) + count = count + 1 + end + + -- 提取有效数据 + local result_members = {} + local result_scores = {} + for i=1,#filtered,2 do + table.insert(result_members, filtered[i]) + table.insert(result_scores, filtered[i+1]) + end + + if #result_members == 0 then return {0,{},{},{}} end + + -- 获取Hash数据 + local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members)) + return {#result_members, result_members, result_scores, hash_data}"; + + // 构造查询范围（包含等于） + string rangeStart, rangeEnd; + if (descending) + { + rangeStart = lastScore.HasValue ? lastScore.Value.ToString() : "+inf"; + rangeEnd = "-inf"; + } + else + { + rangeStart = lastScore.HasValue ? lastScore.Value.ToString() : "-inf"; + rangeEnd = "+inf"; + } + + try + { + var scriptResult = (object[])await Instance.EvalAsync( + luaScript, + new[] { redisZSetScoresIndexCacheKey, redisHashCacheKey }, + new object[] + { + descending ? "ZREVRANGEBYSCORE" : "ZRANGEBYSCORE", + rangeStart, + rangeEnd, + pageSize, + lastScore?.ToString() ?? "", + lastMember ?? "" + }); + + var itemCount = (long)scriptResult[0]; + if (itemCount == 0) + return new BusCacheGlobalPagedResult { Items = new List() }; + + // 处理结果集 + var members = ((object[])scriptResult[1]).Cast().ToList(); + var scores = ((object[])scriptResult[2]).Cast() + .Select(decimal.Parse).ToList(); + var hashData = ((object[])scriptResult[3]).Cast().ToList(); + + var validItems = members.AsParallel() + .Select((m, i) => + { + try { return BusJsonSerializer.Deserialize(hashData[i]); } + catch { return null; } + }) + .Where(x => x != null) + .ToList(); + + // 精确分页控制 + var hasNext = validItems.Count >= pageSize; + var actualItems = validItems.Take(pageSize).ToList(); + + // 计算下一页锚点（必须基于原始排序） + decimal? nextScore = null; + string nextMember = null; + if (hasNext && actualItems.Count > 0) + { + var lastValidIndex = Math.Min(pageSize - 1, members.Count - 1); + nextScore = scores[lastValidIndex]; + nextMember = members[lastValidIndex]; + } + + return new BusCacheGlobalPagedResult + { + Items = actualItems, + HasNext = hasNext, + NextScore = nextScore, + NextMember = nextMember, + TotalCount = await GetTotalCount(redisZSetScoresIndexCacheKey), + PageSize = pageSize + }; + } + catch (Exception ex) + { + _logger.LogError(ex, "分页查询异常"); + return new BusCacheGlobalPagedResult { Items = new List() }; + } + } + + /// + /// 并行分页导出方法（百万级数据支持） + /// + public async Task> FullExportParallel( + string hashKey, + string zsetKey, + int parallelDegree = 10, + int pageSize = 5000) where T : DeviceCacheBasicModel + { + var result = new ConcurrentBag(); + var totalCount = await GetTotalCount(zsetKey); + var totalPages = (int)Math.Ceiling(totalCount / (double)pageSize); + + var semaphore = new SemaphoreSlim(parallelDegree); + var tasks = new List(); + + decimal? lastScore = null; + string lastMember = null; + var isDescending = true; + + for (int page = 0; page < totalPages; page++) + { + await semaphore.WaitAsync(); + + tasks.Add(Task.Run(async () => + { + try + { + var pageResult = await GetAllPagedData( + hashKey, + zsetKey, + pageSize, + lastScore, + lastMember, + isDescending); + + foreach (var item in pageResult.Items) + { + result.Add(item); + } + + // 更新分页锚点 + if (pageResult.HasNext) + { + lastScore = pageResult.NextScore; + lastMember = pageResult.NextMember; + } + } + finally + { + semaphore.Release(); + } + })); + } + + await Task.WhenAll(tasks); + return result.ToList(); + } + + + /// + /// 通过ZSET索引获取数据，支持10万级别数据处理，控制在13秒以内。 /// /// /// 主数据存储Hash缓存Key diff --git a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs index 3af4b6a..3658557 100644 --- a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs +++ b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs @@ -191,40 +191,41 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS } /// - /// 测试单个测点数据项 + /// 测试Redis批量读取10万条数据性能 /// /// [HttpGet] public async Task TestRedisCacheGetAllPagedData() { var timeDensity = "15"; - string SystemType = ""; + string SystemType = "Energy"; string ServerTagName = "JiSheCollectBus2"; var redisCacheMeterInfoHashKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}"; var redisCacheMeterInfoSetIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}"; var redisCacheMeterInfoZSetScoresIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}"; - var timer = Stopwatch.StartNew(); + var timer1 = Stopwatch.StartNew(); decimal? cursor = null; string member = null; bool hasNext; List meterInfos = new List(); do { - var page = await _redisDataCacheService - .GetAllPagedData( - redisCacheMeterInfoHashKeyTemp, - redisCacheMeterInfoZSetScoresIndexKeyTemp); + var page = await _redisDataCacheService.GetAllPagedData( + redisCacheMeterInfoHashKeyTemp, + redisCacheMeterInfoZSetScoresIndexKeyTemp, + pageSize: 1000, + lastScore: cursor, + lastMember: member); meterInfos.AddRange(page.Items); - cursor = page.NextScore; - member = page.NextMember; + cursor = page.HasNext ? page.NextScore : null; + member = page.HasNext ? page.NextMember : null; hasNext = page.HasNext; } while (hasNext); - timer.Stop(); - - _logger.LogInformation($"{nameof(TestRedisCacheGetAllPagedData)} 获取电表缓存数据完成，耗时{timer.ElapsedMilliseconds}毫秒"); + timer1.Stop(); + _logger.LogError($"读取数据更花费时间{timer1.ElapsedMilliseconds}毫秒"); } diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index d32cdb2..27de191 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -15,6 +15,7 @@ using JiShe.CollectBus.IotSystems.MeterReadingRecords; using JiShe.CollectBus.IotSystems.Watermeter; using JiShe.CollectBus.Kafka.Producer; using JiShe.CollectBus.Protocol.Contracts; +using JiShe.CollectBus.RedisDataCache; using JiShe.CollectBus.Repository.MeterReadingRecord; using Mapster; using Microsoft.Extensions.Logging; @@ -23,6 +24,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using static FreeSql.Internal.GlobalFilter; namespace JiShe.CollectBus.ScheduledMeterReading { @@ -121,32 +123,45 @@ namespace JiShe.CollectBus.ScheduledMeterReading _logger.LogInformation($"{nameof(CreateToBeIssueTasks)} 构建待处理的下发指令任务处理时Key=>{item}时间节点不在当前时间范围内,103"); continue; } - - - - //获取缓存中的表信息 - var redisKeyList = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, ServerTagName, meteryType, timeDensity)}*"; - var oneMinutekeyList = await FreeRedisProvider.Instance.KeysAsync(redisKeyList); - if (oneMinutekeyList == null || oneMinutekeyList.Length <= 0) - { - _logger.LogError($"{nameof(CreateToBeIssueTasks)} {timeDensity}分钟采集待下发任务创建失败，没有获取到缓存信息,-104"); - return; - } - + var meterTypes = EnumExtensions.ToEnumDictionary(); if (meteryType == MeterTypeEnum.Ammeter.ToString()) { - // 解析结果（结果为嵌套数组） - var meterInfos = await GetMeterRedisCacheListData(oneMinutekeyList, SystemType, ServerTagName, $"{timeDensity}", meterTypes[meteryType]); + var timer = Stopwatch.StartNew(); + + //获取对应频率中的所有电表信息 + var redisCacheMeterInfoHashKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}"; + var redisCacheMeterInfoSetIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}"; + var redisCacheMeterInfoZSetScoresIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}"; + + List meterInfos = new List(); + decimal? cursor = null; + string member = null; + bool hasNext; + do + { + var page = await _redisDataCacheService.GetAllPagedData( + redisCacheMeterInfoHashKeyTemp, + redisCacheMeterInfoZSetScoresIndexKeyTemp, + pageSize: 1000, + lastScore: cursor, + lastMember: member); + + meterInfos.AddRange(page.Items); + cursor = page.HasNext ? page.NextScore : null; + member = page.HasNext ? page.NextMember : null; + hasNext = page.HasNext; + } while (hasNext); + if (meterInfos == null || meterInfos.Count <= 0) { + timer.Stop(); _logger.LogError($"{nameof(CreateToBeIssueTasks)} {timeDensity}分钟采集待下发任务创建失败，没有获取到缓存信息,-105"); return; } //await AmmerterScheduledMeterReadingIssued(timeDensity, meterInfos); - var timer = Stopwatch.StartNew(); //处理数据 //await DeviceGroupBalanceControl.ProcessGenericListAsync( @@ -163,14 +178,14 @@ namespace JiShe.CollectBus.ScheduledMeterReading await DeviceGroupBalanceControl.ProcessWithThrottleAsync( items: meterInfos, deviceIdSelector: data => data.FocusAddress, - processor: data => + processor: (data,groupIndex) => { - _ = AmmerterCreatePublishTask(timeDensity, data); + _ = AmmerterCreatePublishTask(timeDensity, data, groupIndex,tasksToBeIssueModel.NextTaskTime.ToString("yyyyMMddHHmmss")); } ); timer.Stop(); - _logger.LogInformation($"{nameof(CreateToBeIssueTasks)} {timeDensity}分钟采集待下发任务创建完成,{timer.ElapsedMilliseconds},{oneMinutekeyList.Length}"); + _logger.LogInformation($"{nameof(CreateToBeIssueTasks)} {timeDensity}分钟采集待下发任务创建完成,{timer.ElapsedMilliseconds},总共{meterInfos.Count}表计信息"); } else if (meteryType == MeterTypeEnum.WaterMeter.ToString()) @@ -230,37 +245,65 @@ namespace JiShe.CollectBus.ScheduledMeterReading // meterInfos.Add(tempData); // //focusAddressDataLista.Add(item.FocusAddress); //} - - //DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); - + + + var timeDensity = "15"; var redisCacheMeterInfoHashKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoHashKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; var redisCacheMeterInfoSetIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoSetIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; var redisCacheMeterInfoZSetScoresIndexKeyTemp = $"{string.Format(RedisConst.CacheMeterInfoZSetScoresIndexKey, SystemType, "JiSheCollectBus2", MeterTypeEnum.Ammeter, timeDensity)}"; - var timer1 = Stopwatch.StartNew(); - decimal? cursor = null; - string member = null; - bool hasNext; List meterInfos = new List(); - do + List focusAddressDataLista = new List(); + var timer1 = Stopwatch.StartNew(); + //decimal? cursor = null; + //string member = null; + //bool hasNext; + //do + //{ + // var page = await _redisDataCacheService.GetAllPagedDataOptimized( + // redisCacheMeterInfoHashKeyTemp, + // redisCacheMeterInfoZSetScoresIndexKeyTemp, + // pageSize: 1000, + // lastScore: cursor, + // lastMember: member); + + // meterInfos.AddRange(page.Items); + // cursor = page.HasNext ? page.NextScore : null; + // member = page.HasNext ? page.NextMember : null; + // hasNext = page.HasNext; + //} while (hasNext); + + var allIds = new HashSet(); + decimal? score = null; + string member = null; + + while (true) { - var page = await _redisDataCacheService.GetAllPagedData( + var page = await _redisDataCacheService.GetAllPagedDataOptimized( redisCacheMeterInfoHashKeyTemp, redisCacheMeterInfoZSetScoresIndexKeyTemp, pageSize: 1000, - lastScore: cursor, + lastScore: score, lastMember: member); meterInfos.AddRange(page.Items); - cursor = page.HasNext ? page.NextScore : null; - member = page.HasNext ? page.NextMember : null; - hasNext = page.HasNext; - } while (hasNext); + focusAddressDataLista.AddRange(page.Items.Select(d=>d.FocusAddress)); + foreach (var item in page.Items) + { + if (!allIds.Add(item.MemberId)) + throw new Exception("Duplicate data found!"); + } + if (!page.HasNext) break; + score = page.NextScore; + member = page.NextMember; + } + timer1.Stop(); _logger.LogError($"读取数据更花费时间{timer1.ElapsedMilliseconds}毫秒"); - + //DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); + //return; #else var meterInfos = await GetAmmeterInfoList(gatherCode); #endif @@ -656,9 +699,11 @@ namespace JiShe.CollectBus.ScheduledMeterReading /// /// 采集频率 /// 集中器号hash分组的集中器集合数据 + /// 集中器所在分组 + /// 时间格式的任务批次名称 /// private async Task AmmerterCreatePublishTask(int timeDensity - , AmmeterInfo ammeterInfo) + , AmmeterInfo ammeterInfo,int groupIndex,string taskBatch) { var handlerPacketBuilder = TelemetryPacketBuilder.AFNHandlersDictionary; //todo 检查需要待补抄的电表的时间点信息，保存到需要待补抄的缓存中。如果此线程异常，该如何补偿？ @@ -666,7 +711,9 @@ namespace JiShe.CollectBus.ScheduledMeterReading var currentTime = DateTime.Now; var pendingCopyReadTime = currentTime.AddMinutes(timeDensity); //构建缓存任务key，依然 表计类型+采集频率+集中器地址，存hash类型 - var redisCacheKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoHashKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}{ammeterInfo.FocusAddress}"; + var redisCacheTelemetryPacketInfoHashKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoHashKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity, groupIndex, taskBatch)}"; + var redisCacheTelemetryPacketInfoSetIndexKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoSetIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity, groupIndex, taskBatch)}"; + var redisCacheTelemetryPacketInfoZSetScoresIndexKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoZSetScoresIndexKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity, groupIndex, taskBatch)}"; if (string.IsNullOrWhiteSpace(ammeterInfo.ItemCodes)) { @@ -747,7 +794,8 @@ namespace JiShe.CollectBus.ScheduledMeterReading } } - Dictionary keyValuePairs = new Dictionary(); + //Dictionary keyValuePairs = new Dictionary(); + List taskList = new List(); foreach (var tempItem in tempCodes) { @@ -802,7 +850,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading - var meterReadingRecords = new MeterReadingRecords() + var meterReadingRecords = new MeterReadingTelemetryPacketInfo() { ProjectID = ammeterInfo.ProjectID, DatabaseBusiID = ammeterInfo.DatabaseBusiID, @@ -812,7 +860,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading MeterId = ammeterInfo.MeterId, MeterType = MeterTypeEnum.Ammeter, FocusAddress = ammeterInfo.FocusAddress, - FocusID = ammeterInfo.FocusId, + FocusId = ammeterInfo.FocusId, AFN = aFN, Fn = fn, ItemCode = tempItem, @@ -822,9 +870,10 @@ namespace JiShe.CollectBus.ScheduledMeterReading IssuedMessageId = GuidGenerator.Create().ToString(), IssuedMessageHexString = Convert.ToHexString(dataInfos), }; - meterReadingRecords.CreateDataId(GuidGenerator.Create()); - keyValuePairs.TryAdd($"{ammeterInfo.MeterId}_{tempItem}", meterReadingRecords); + //meterReadingRecords.CreateDataId(GuidGenerator.Create()); + + taskList.Add(meterReadingRecords); } //TimeSpan timeSpan = TimeSpan.FromMicroseconds(5); //await Task.Delay(timeSpan); @@ -832,14 +881,25 @@ namespace JiShe.CollectBus.ScheduledMeterReading //return keyValuePairs; // await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs); - using (var pipe = FreeRedisProvider.Instance.StartPipe()) + //using (var pipe = FreeRedisProvider.Instance.StartPipe()) + //{ + // pipe.HSet(redisCacheKey, keyValuePairs); + // object[] ret = pipe.EndPipe(); + //} + if (taskList == null + || taskList.Count() <= 0 + || string.IsNullOrWhiteSpace(redisCacheTelemetryPacketInfoHashKey) + || string.IsNullOrWhiteSpace(redisCacheTelemetryPacketInfoSetIndexKey) + || string.IsNullOrWhiteSpace(redisCacheTelemetryPacketInfoZSetScoresIndexKey)) { - pipe.HSet(redisCacheKey, keyValuePairs); - object[] ret = pipe.EndPipe(); + _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 写入参数异常，{redisCacheTelemetryPacketInfoHashKey}：{redisCacheTelemetryPacketInfoSetIndexKey}：{redisCacheTelemetryPacketInfoZSetScoresIndexKey}，-101"); + return; } - - - await Task.CompletedTask; + await _redisDataCacheService.BatchInsertDataAsync( + redisCacheTelemetryPacketInfoHashKey, + redisCacheTelemetryPacketInfoSetIndexKey, + redisCacheTelemetryPacketInfoZSetScoresIndexKey, + taskList); } /// @@ -1088,7 +1148,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading IssuedMessageId = GuidGenerator.Create().ToString(), IssuedMessageHexString = Convert.ToHexString(dataInfos), }; - meterReadingRecords.CreateDataId(GuidGenerator.Create()); + //meterReadingRecords.CreateDataId(GuidGenerator.Create()); keyValuePairs.TryAdd($"{ammeter.MeterId}_{tempItem}", meterReadingRecords); } diff --git a/src/JiShe.CollectBus.Common/Consts/RedisConst.cs b/src/JiShe.CollectBus.Common/Consts/RedisConst.cs index dce5307..7ac170b 100644 --- a/src/JiShe.CollectBus.Common/Consts/RedisConst.cs +++ b/src/JiShe.CollectBus.Common/Consts/RedisConst.cs @@ -49,23 +49,23 @@ namespace JiShe.CollectBus.Common.Consts /// /// 缓存待下发的指令生产任务数据,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 /// - public const string CacheTasksToBeIssuedKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TaskInfo}:{"{2}"}{"{3}"}"; + public const string CacheTasksToBeIssuedKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TaskInfo}:{"{2}"}:{"{3}"}"; public const string TelemetryPacket = "TelemetryPacket"; /// - /// 缓存表计下发指令数据集,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// 缓存表计下发指令数据集,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率，{4}=>集中器所在分组，{5}=>时间格式的任务批次 /// - public const string CacheTelemetryPacketInfoHashKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:{"{3}"}"; + public const string CacheTelemetryPacketInfoHashKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:{"{3}"}:{"{4}"}:{"{5}"}"; /// - /// 缓存表计下发指令数据集索引Set缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// 缓存表计下发指令数据集索引Set缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率，{4}=>集中器所在分组，{5}=>时间格式的任务批次 /// - public const string CacheTelemetryPacketInfoSetIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:SetIndex:{"{3}"}"; + public const string CacheTelemetryPacketInfoSetIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:SetIndex:{"{3}"}:{"{4}"}:{"{5}"}"; /// - /// 缓存表计下发指令数据集排序索引ZSET缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率 + /// 缓存表计下发指令数据集排序索引ZSET缓存Key,{0}=>系统类型，{1}=>应用服务部署标记，{2}=>表计类别，{3}=>采集频率，{4}=>集中器所在分组，{5}=>时间格式的任务批次 /// - public const string CacheTelemetryPacketInfoZSetScoresIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:ZSetScoresIndex:{"{3}"}"; + public const string CacheTelemetryPacketInfoZSetScoresIndexKey = $"{CacheBasicDirectoryKey}{"{0}:{1}"}:{TelemetryPacket}:{"{2}"}:ZSetScoresIndex:{"{3}"}:{"{4}"}:{"{5}"}"; ///// ///// 缓存设备平衡关系映射结果,{0}=>系统类型，{1}=>应用服务部署标记 diff --git a/src/JiShe.CollectBus.Common/DeviceBalanceControl/DeviceGroupBalanceControl.cs b/src/JiShe.CollectBus.Common/DeviceBalanceControl/DeviceGroupBalanceControl.cs index 4be0f4f..06b7d70 100644 --- a/src/JiShe.CollectBus.Common/DeviceBalanceControl/DeviceGroupBalanceControl.cs +++ b/src/JiShe.CollectBus.Common/DeviceBalanceControl/DeviceGroupBalanceControl.cs @@ -161,7 +161,6 @@ namespace JiShe.CollectBus.Common.DeviceBalanceControl MaxDegreeOfParallelism = maxThreads.Value, }; - TimeSpan timeSpan = TimeSpan.FromMicroseconds(5); await Task.Run(() => { Parallel.For(0, cache.CachedGroups.Length, options, async groupId => @@ -169,8 +168,7 @@ namespace JiShe.CollectBus.Common.DeviceBalanceControl var queue = groupQueues[groupId]; while (queue.TryDequeue(out T item)) { - await Task.Delay(timeSpan); - processor(item, Thread.CurrentThread.ManagedThreadId); + processor(item, groupId); } }); }); @@ -183,14 +181,14 @@ namespace JiShe.CollectBus.Common.DeviceBalanceControl /// 已经分组的设备信息 /// 部分或者全部的已经分组的设备集合 /// 从泛型对象提取deviceId - /// 处理委托（参数：当前对象，线程ID） + /// 处理委托（参数：当前对象，分组ID） /// 可选最佳并发度 /// /// public static async Task ProcessWithThrottleAsync( List items, Func deviceIdSelector, - Action processor, + Action processor, int? maxConcurrency = null) { var cache = _currentCache ?? throw new InvalidOperationException("缓存未初始化"); @@ -244,7 +242,7 @@ namespace JiShe.CollectBus.Common.DeviceBalanceControl /// /// 分组异步处理（带节流） /// - private static async Task ProcessItemAsync(T item, Action processor, int groupId) + private static async Task ProcessItemAsync(T item, Action processor, int groupId) { // 使用内存缓存降低CPU负载 await Task.Yield(); // 立即释放当前线程 @@ -255,7 +253,7 @@ namespace JiShe.CollectBus.Common.DeviceBalanceControl { ExecutionContext.Run(context!, state => { - processor(item); + processor(item,groupId); }, null); }); } diff --git a/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs b/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs index 335c17c..d397151 100644 --- a/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs +++ b/src/JiShe.CollectBus.Common/Models/DeviceCacheBasicModel.cs @@ -24,7 +24,7 @@ namespace JiShe.CollectBus.Common.Models /// /// 关系映射标识，用于ZSet的Member字段和Set的Value字段，具体值可以根据不同业务场景进行定义 /// - public virtual string MemberID => $"{FocusId}:{MeterId}"; + public virtual string MemberId => $"{FocusId}:{MeterId}"; /// /// ZSet排序索引分数值，具体值可以根据不同业务场景进行定义，例如时间戳 diff --git a/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs b/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs index 3df01b5..8b082bb 100644 --- a/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs +++ b/src/JiShe.CollectBus.Domain/Ammeters/AmmeterInfo.cs @@ -14,7 +14,7 @@ namespace JiShe.CollectBus.Ammeters /// 关系映射标识，用于ZSet的Member字段和Set的Value字段，具体值可以根据不同业务场景进行定义 /// [Column(IsIgnore = true)] - public override string MemberID => $"{FocusId}:{MeterId}"; + public override string MemberId => $"{FocusId}:{MeterId}"; /// /// ZSet排序索引分数值，具体值可以根据不同业务场景进行定义，例如时间戳 diff --git a/src/JiShe.CollectBus.Domain/IotSystems/MeterReadingRecords/MeterReadingTelemetryPacketInfo.cs b/src/JiShe.CollectBus.Domain/IotSystems/MeterReadingRecords/MeterReadingTelemetryPacketInfo.cs new file mode 100644 index 0000000..c3f75d3 --- /dev/null +++ b/src/JiShe.CollectBus.Domain/IotSystems/MeterReadingRecords/MeterReadingTelemetryPacketInfo.cs @@ -0,0 +1,141 @@ +using JiShe.CollectBus.Common.Enums; +using JiShe.CollectBus.Common.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Volo.Abp.Domain.Entities; +using Volo.Abp.Domain.Entities.Auditing; + +namespace JiShe.CollectBus.IotSystems.MeterReadingRecords +{ + /// + /// 抄读任务Redis缓存数据记录 + /// + public class MeterReadingTelemetryPacketInfo : DeviceCacheBasicModel + { + /// + /// 关系映射标识，用于ZSet的Member字段和Set的Value字段，具体值可以根据不同业务场景进行定义 + /// + public override string MemberId => $"{FocusId}:{MeterId}:{ItemCode}"; + + /// + /// ZSet排序索引分数值，具体值可以根据不同业务场景进行定义，例如时间戳 + /// + public override long ScoreValue => ((long)FocusId << 32) | (uint)DateTime.Now.Ticks; + + + /// + /// 是否手动操作 + /// + public bool ManualOrNot { get; set; } + + /// + /// 任务数据唯一标记 + /// + public string TaskMark { get; set; } + + /// + /// 时间戳标记，IoTDB时间列处理，上报通过构建标记获取唯一标记匹配时间戳。 + /// + public long Timestamps { get; set; } + + /// + /// 是否超时 + /// + public bool IsTimeout { get; set; } = false; + + /// + /// 待抄读时间 + /// + public DateTime PendingCopyReadTime { get; set; } + + + /// + /// 集中器地址 + /// + public string FocusAddress { get; set; } + + /// + /// 表地址 + /// + public string MeterAddress { get; set; } + + /// + /// 表类型 + /// + public MeterTypeEnum MeterType { get; set; } + + /// + /// 项目ID + /// + public int ProjectID { get; set; } + + /// + /// 数据库业务ID + /// + public int DatabaseBusiID { get; set; } + + /// + /// AFN功能码 + /// + public AFN AFN { get; set; } + + /// + /// 抄读功能码 + /// + public int Fn { get; set; } + + /// + /// 抄读计量点 + /// + public int Pn { get; set; } + + /// + /// 采集项编码 + /// + public string ItemCode { get; set;} + + + /// + /// 创建时间 + /// + public DateTime CreationTime { get; set; } + + /// + /// 下发消息内容 + /// + public string IssuedMessageHexString { get; set; } + + /// + /// 下发消息Id + /// + public string IssuedMessageId { get; set; } + + /// + /// 消息上报内容 + /// + public string? ReceivedMessageHexString { get; set; } + + /// + /// 消息上报时间 + /// + public DateTime? ReceivedTime { get; set; } + + /// + /// 上报消息Id + /// + public string ReceivedMessageId { get; set; } + + /// + /// 上报报文解析备注，异常情况下才有 + /// + public string ReceivedRemark { get; set; } + + //public void CreateDataId(Guid Id) + //{ + // this.Id = Id; + //} + } +} From 8fd5f985ab23cd794438dfa3ebfd554726996b0e Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Thu, 17 Apr 2025 11:42:35 +0800 Subject: [PATCH 18/27] =?UTF-8?q?=E6=9A=82=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CollectBusKafkaModule.cs | 42 ++++++++++++++- .../Consumer/ConsumerService.cs | 43 ++++++--------- .../JiShe.CollectBus.Kafka.csproj | 4 -- .../KafkaOptionConfig.cs | 53 +++++++++++++++++++ .../KafkaSubcribesExtensions.cs | 50 +++++++---------- .../Producer/ProducerService.cs | 27 +++++----- 6 files changed, 144 insertions(+), 75 deletions(-) create mode 100644 src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs diff --git a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs index fe0e866..48a8c35 100644 --- a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs +++ b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs @@ -16,6 +16,15 @@ namespace JiShe.CollectBus.Kafka { public override void ConfigureServices(ServiceConfigurationContext context) { + var configuration = context.Services.GetConfiguration(); + var kafkaSection = configuration.GetSection("Kafka"); + KafkaOptionConfig kafkaOptionConfig = new KafkaOptionConfig (); + kafkaSection.Bind(kafkaOptionConfig); + if (configuration["ServerTagName"] != null) + { + kafkaOptionConfig.ServerTagName = configuration["ServerTagName"]!; + } + context.Services.AddSingleton(kafkaOptionConfig); // 注册Producer context.Services.AddSingleton(); // 注册Consumer @@ -25,8 +34,39 @@ namespace JiShe.CollectBus.Kafka public override void OnApplicationInitialization(ApplicationInitializationContext context) { var app = context.GetApplicationBuilder(); - app.UseKafkaSubscribers(Assembly.Load("JiShe.CollectBus.Application")); + // 程序运行目录 + var assemblyPath = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location); + if (!string.IsNullOrWhiteSpace(assemblyPath)) + { + var dllFiles = Directory.GetFiles(assemblyPath, "*.dll"); + var kafkaSubscriberAssemblies = new List(); + foreach (var file in dllFiles) + { + try + { + // 跳过已加载的程序集 + var assemblyName = AssemblyName.GetAssemblyName(file); + var existingAssembly = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => a.GetName().FullName == assemblyName.FullName); + var assembly = existingAssembly ?? Assembly.LoadFrom(file); + // 检查程序集是否包含 IKafkaSubscribe 的实现类 + var hasSubscriber = assembly.GetTypes() + .Any(type => + typeof(IKafkaSubscribe).IsAssignableFrom(type) && // 实现接口 + !type.IsAbstract && !type.IsInterface); // 排除抽象类和接口本身 + + if (hasSubscriber) + { + kafkaSubscriberAssemblies.Add(assembly); + } + } + catch{} + app.UseKafkaSubscribers(kafkaSubscriberAssemblies.ToArray()); + } + } + // 获取程序集 + //app.UseKafkaSubscribers(new[] { Assembly.Load("JiShe.CollectBus.Application")}); } } } diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs index da9258c..eeb5661 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs @@ -1,16 +1,7 @@ using Confluent.Kafka; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using JiShe.CollectBus.Kafka.Attributes; -using Volo.Abp.DependencyInjection; -using JiShe.CollectBus.Kafka.AdminClient; -using static Confluent.Kafka.ConfigPropertyNames; using System.Collections.Concurrent; -using System.Text.RegularExpressions; -using NUglify.Html; -using Serilog; -using System; using System.Text; namespace JiShe.CollectBus.Kafka.Consumer @@ -21,12 +12,14 @@ namespace JiShe.CollectBus.Kafka.Consumer private readonly IConfiguration _configuration; private readonly ConcurrentDictionary _consumerStore = new(); + private readonly KafkaOptionConfig _kafkaOptionConfig; private class KafkaConsumer where TKey : notnull where TValue : class { } - public ConsumerService(IConfiguration configuration, ILogger logger) + public ConsumerService(IConfiguration configuration, ILogger logger, KafkaOptionConfig kafkaOptionConfig) { _configuration = configuration; _logger = logger; + _kafkaOptionConfig = kafkaOptionConfig; } #region private 私有方法 @@ -49,11 +42,9 @@ namespace JiShe.CollectBus.Kafka.Consumer private ConsumerConfig BuildConsumerConfig(string? groupId = null) { - var enableAuth = bool.Parse(_configuration["Kafka:EnableAuthorization"]!); - var config = new ConsumerConfig { - BootstrapServers = _configuration["Kafka:BootstrapServers"], + BootstrapServers = _kafkaOptionConfig.BootstrapServers, GroupId = groupId ?? "default", AutoOffsetReset = AutoOffsetReset.Earliest, EnableAutoCommit = false, // 禁止AutoCommit @@ -62,12 +53,12 @@ namespace JiShe.CollectBus.Kafka.Consumer FetchMaxBytes = 1024 * 1024 * 50 // 增加拉取大小（50MB） }; - if (enableAuth) + if (_kafkaOptionConfig.EnableAuthorization) { - config.SecurityProtocol = SecurityProtocol.SaslPlaintext; - config.SaslMechanism = SaslMechanism.Plain; - config.SaslUsername = _configuration["Kafka:SaslUserName"]; - config.SaslPassword = _configuration["Kafka:SaslPassword"]; + config.SecurityProtocol = _kafkaOptionConfig.SecurityProtocol; + config.SaslMechanism = _kafkaOptionConfig.SaslMechanism; + config.SaslUsername = _kafkaOptionConfig.SaslUserName; + config.SaslPassword = _kafkaOptionConfig.SaslPassword; } return config; @@ -140,9 +131,9 @@ namespace JiShe.CollectBus.Kafka.Consumer await Task.Delay(TimeSpan.FromSeconds(1),cts.Token); continue; } - if (bool.Parse(_configuration["KafkaConsumer:EnableFilter"]!)) + if (_kafkaOptionConfig.EnableFilter) { - var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } }; // 检查 Header 是否符合条件 if (!headersFilter.Match(result.Message.Headers)) { @@ -208,9 +199,9 @@ namespace JiShe.CollectBus.Kafka.Consumer await Task.Delay(100, cts.Token); continue; } - if (bool.Parse(_configuration["KafkaConsumer:EnableFilter"]!)) + if (_kafkaOptionConfig.EnableFilter) { - var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } }; // 检查 Header 是否符合条件 if (!headersFilter.Match(result.Message.Headers)) { @@ -296,9 +287,9 @@ namespace JiShe.CollectBus.Kafka.Consumer } else if (result.Message.Value != null) { - if (bool.Parse(_configuration["KafkaConsumer:EnableFilter"]!)) + if (_kafkaOptionConfig.EnableFilter) { - var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } }; // 检查 Header 是否符合条件 if (!headersFilter.Match(result.Message.Headers)) { @@ -430,9 +421,9 @@ namespace JiShe.CollectBus.Kafka.Consumer } else if (result.Message.Value != null) { - if (bool.Parse(_configuration["KafkaConsumer:EnableFilter"]!)) + if (_kafkaOptionConfig.EnableFilter) { - var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } }; + var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } }; // 检查 Header 是否符合条件 if (!headersFilter.Match(result.Message.Headers)) { diff --git a/src/JiShe.CollectBus.KafkaProducer/JiShe.CollectBus.Kafka.csproj b/src/JiShe.CollectBus.KafkaProducer/JiShe.CollectBus.Kafka.csproj index 9518b0b..cef24d5 100644 --- a/src/JiShe.CollectBus.KafkaProducer/JiShe.CollectBus.Kafka.csproj +++ b/src/JiShe.CollectBus.KafkaProducer/JiShe.CollectBus.Kafka.csproj @@ -12,8 +12,4 @@ - - - - diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs new file mode 100644 index 0000000..50ae47e --- /dev/null +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs @@ -0,0 +1,53 @@ +using Confluent.Kafka; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JiShe.CollectBus.Kafka +{ + public class KafkaOptionConfig + { + /// + /// kafka地址 + /// + public string BootstrapServers { get; set; } = null!; + + /// + /// 服务器标识 + /// + public string ServerTagName { get; set; }= "KafkaFilterKey"; + + /// + /// 是否开启过滤器 + /// + public bool EnableFilter { get; set; }= true; + + /// + /// 是否开启认证 + /// + public bool EnableAuthorization { get; set; } = false; + + /// + /// 安全协议 + /// + public SecurityProtocol SecurityProtocol { get; set; } = SecurityProtocol.SaslPlaintext; + + /// + /// 认证方式 + /// + public SaslMechanism SaslMechanism { get; set; }= SaslMechanism.Plain; + + /// + /// 用户名 + /// + public string? SaslUserName { get; set; } + + /// + /// 密码 + /// + public string? SaslPassword { get; set; } + + } +} diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index 8860061..1f83540 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -1,7 +1,4 @@ using Confluent.Kafka; -using DeviceDetectorNET; -using JiShe.CollectBus.Common.Enums; -using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Kafka.AdminClient; using JiShe.CollectBus.Kafka.Attributes; using JiShe.CollectBus.Kafka.Consumer; @@ -9,16 +6,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Primitives; -using Newtonsoft.Json; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; using System.Reflection; -using System.Text; -using System.Threading.Tasks; -using static Confluent.Kafka.ConfigPropertyNames; namespace JiShe.CollectBus.Kafka { @@ -29,14 +17,8 @@ namespace JiShe.CollectBus.Kafka /// /// /// - public static void UseKafkaSubscribers(this IApplicationBuilder app, Assembly assembly) + public static void UseKafkaSubscribers(this IApplicationBuilder app, Assembly[] assemblys) { - var subscribeTypes = assembly.GetTypes() - .Where(t => typeof(IKafkaSubscribe).IsAssignableFrom(t)) - .ToList(); - - if (subscribeTypes.Count == 0) return; - var provider = app.ApplicationServices; var lifetime = provider.GetRequiredService(); @@ -45,18 +27,26 @@ namespace JiShe.CollectBus.Kafka var logger = provider.GetRequiredService>(); int threadCount = 0; int topicCount = 0; - foreach (var subscribeType in subscribeTypes) + foreach (Assembly assembly in assemblys) { - var subscribes = provider.GetServices(subscribeType).ToList(); - subscribes.ForEach(subscribe => { - - if(subscribe is IKafkaSubscribe) - { - Tuple tuple= BuildKafkaSubscriber(subscribe, provider, logger); - threadCount+= tuple.Item1; - topicCount+= tuple.Item2; - } - }); + var subscribeTypes = assembly.GetTypes() + .Where(t => typeof(IKafkaSubscribe).IsAssignableFrom(t)) + .ToList(); + + if (subscribeTypes.Count == 0) return; + foreach (var subscribeType in subscribeTypes) + { + var subscribes = provider.GetServices(subscribeType).ToList(); + subscribes.ForEach(subscribe => { + + if (subscribe is IKafkaSubscribe) + { + Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger); + threadCount += tuple.Item1; + topicCount += tuple.Item2; + } + }); + } } logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程"); }); diff --git a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs index 0ddf36b..27702e0 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs @@ -19,11 +19,12 @@ namespace JiShe.CollectBus.Kafka.Producer private readonly IConfiguration _configuration; private readonly ConcurrentDictionary _producerCache = new(); private class KafkaProducer where TKey : notnull where TValue : class { } - - public ProducerService(IConfiguration configuration,ILogger logger) + private readonly KafkaOptionConfig _kafkaOptionConfig; + public ProducerService(IConfiguration configuration,ILogger logger, KafkaOptionConfig kafkaOptionConfig) { _configuration = configuration; _logger = logger; + _kafkaOptionConfig = kafkaOptionConfig; } #region private 私有方法 @@ -51,11 +52,9 @@ namespace JiShe.CollectBus.Kafka.Producer /// private ProducerConfig BuildProducerConfig() { - var enableAuth = bool.Parse(_configuration["Kafka:EnableAuthorization"]!); - var config = new ProducerConfig { - BootstrapServers = _configuration["Kafka:BootstrapServers"], + BootstrapServers = _kafkaOptionConfig.BootstrapServers, AllowAutoCreateTopics = true, QueueBufferingMaxKbytes = 2_097_151, // 修改缓冲区最大为2GB，默认为1GB CompressionType = CompressionType.Lz4, // 配置使用压缩算法LZ4，其他：gzip/snappy/zstd @@ -66,12 +65,12 @@ namespace JiShe.CollectBus.Kafka.Producer MessageTimeoutMs = 120000, // 消息发送超时时间为2分钟,设置值MessageTimeoutMs > LingerMs }; - if (enableAuth) + if (_kafkaOptionConfig.EnableAuthorization) { - config.SecurityProtocol = SecurityProtocol.SaslPlaintext; - config.SaslMechanism = SaslMechanism.Plain; - config.SaslUsername = _configuration["Kafka:SaslUserName"]; - config.SaslPassword = _configuration["Kafka:SaslPassword"]; + config.SecurityProtocol = _kafkaOptionConfig.SecurityProtocol; + config.SaslMechanism = _kafkaOptionConfig.SaslMechanism; + config.SaslUsername = _kafkaOptionConfig.SaslUserName; + config.SaslPassword = _kafkaOptionConfig.SaslPassword; } return config; @@ -110,7 +109,7 @@ namespace JiShe.CollectBus.Kafka.Producer Key = key, Value = value, Headers = new Headers{ - { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } + { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } } }; await producer.ProduceAsync(topic, message); @@ -131,7 +130,7 @@ namespace JiShe.CollectBus.Kafka.Producer { Value = value, Headers = new Headers{ - { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } + { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } } }; await producer.ProduceAsync(topic, message); @@ -155,7 +154,7 @@ namespace JiShe.CollectBus.Kafka.Producer Key = key, Value = value, Headers = new Headers{ - { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } + { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } } }; var typeKey = typeof(KafkaProducer); @@ -189,7 +188,7 @@ namespace JiShe.CollectBus.Kafka.Producer { Value = value, Headers = new Headers{ - { "route-key", Encoding.UTF8.GetBytes(_configuration["ServerTagName"]!) } + { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } } }; var typeKey = typeof(KafkaProducer); From fa18377f107288f31731531a19fbd6d62ccda845 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Thu, 17 Apr 2025 11:53:29 +0800 Subject: [PATCH 19/27] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/JiShe.CollectBus.Host/appsettings.json | 4 +- .../CollectBusKafkaModule.cs | 58 +++++++------- .../KafkaSubcribesExtensions.cs | 76 +++++++++++++------ .../Abstracts/BaseProtocolPlugin.cs | 1 + 4 files changed, 86 insertions(+), 53 deletions(-) diff --git a/src/JiShe.CollectBus.Host/appsettings.json b/src/JiShe.CollectBus.Host/appsettings.json index f830481..7aa3947 100644 --- a/src/JiShe.CollectBus.Host/appsettings.json +++ b/src/JiShe.CollectBus.Host/appsettings.json @@ -86,8 +86,8 @@ "BootstrapServers": "192.168.1.9:29092,192.168.1.9:39092,192.168.1.9:49092", "EnableFilter": true, "EnableAuthorization": false, - "SecurityProtocol": "SASL_PLAINTEXT", - "SaslMechanism": "PLAIN", + "SecurityProtocol": "SaslPlaintext", + "SaslMechanism": "Plain", "SaslUserName": "lixiao", "SaslPassword": "lixiao1980" //"Topic": { diff --git a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs index 48a8c35..80739a6 100644 --- a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs +++ b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs @@ -35,38 +35,38 @@ namespace JiShe.CollectBus.Kafka { var app = context.GetApplicationBuilder(); // 程序运行目录 - var assemblyPath = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location); - if (!string.IsNullOrWhiteSpace(assemblyPath)) - { - var dllFiles = Directory.GetFiles(assemblyPath, "*.dll"); - var kafkaSubscriberAssemblies = new List(); - foreach (var file in dllFiles) - { - try - { - // 跳过已加载的程序集 - var assemblyName = AssemblyName.GetAssemblyName(file); - var existingAssembly = AppDomain.CurrentDomain.GetAssemblies() - .FirstOrDefault(a => a.GetName().FullName == assemblyName.FullName); - var assembly = existingAssembly ?? Assembly.LoadFrom(file); + //var assemblyPath = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location); + //if (!string.IsNullOrWhiteSpace(assemblyPath)) + //{ + // var dllFiles = Directory.GetFiles(assemblyPath, "*.dll"); + // var kafkaSubscriberAssemblies = new List(); + // foreach (var file in dllFiles) + // { + // try + // { + // // 跳过已加载的程序集 + // var assemblyName = AssemblyName.GetAssemblyName(file); + // var existingAssembly = AppDomain.CurrentDomain.GetAssemblies() + // .FirstOrDefault(a => a.GetName().FullName == assemblyName.FullName); + // var assembly = existingAssembly ?? Assembly.LoadFrom(file); - // 检查程序集是否包含 IKafkaSubscribe 的实现类 - var hasSubscriber = assembly.GetTypes() - .Any(type => - typeof(IKafkaSubscribe).IsAssignableFrom(type) && // 实现接口 - !type.IsAbstract && !type.IsInterface); // 排除抽象类和接口本身 + // // 检查程序集是否包含 IKafkaSubscribe 的实现类 + // var hasSubscriber = assembly.GetTypes() + // .Any(type => + // typeof(IKafkaSubscribe).IsAssignableFrom(type) && // 实现接口 + // !type.IsAbstract && !type.IsInterface); // 排除抽象类和接口本身 - if (hasSubscriber) - { - kafkaSubscriberAssemblies.Add(assembly); - } - } - catch{} - app.UseKafkaSubscribers(kafkaSubscriberAssemblies.ToArray()); - } - } + // if (hasSubscriber) + // { + // kafkaSubscriberAssemblies.Add(assembly); + // } + // } + // catch{} + // app.UseKafkaSubscribers(kafkaSubscriberAssemblies.ToArray()); + // } + //} // 获取程序集 - //app.UseKafkaSubscribers(new[] { Assembly.Load("JiShe.CollectBus.Application")}); + app.UseKafkaSubscribers(Assembly.Load("JiShe.CollectBus.Application")); } } } diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index 1f83540..4c69b6a 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -12,12 +12,47 @@ namespace JiShe.CollectBus.Kafka { public static class KafkaSubcribesExtensions { - /// - /// 添加Kafka订阅 - /// - /// - /// - public static void UseKafkaSubscribers(this IApplicationBuilder app, Assembly[] assemblys) + ///// + ///// 添加Kafka订阅 + ///// + ///// + ///// + //public static void UseKafkaSubscribers(this IApplicationBuilder app, Assembly[] assemblys) + //{ + // var provider = app.ApplicationServices; + // var lifetime = provider.GetRequiredService(); + + // lifetime.ApplicationStarted.Register(() => + // { + // var logger = provider.GetRequiredService>(); + // int threadCount = 0; + // int topicCount = 0; + // foreach (Assembly assembly in assemblys) + // { + // var subscribeTypes = assembly.GetTypes() + // .Where(t => typeof(IKafkaSubscribe).IsAssignableFrom(t)) + // .ToList(); + + // if (subscribeTypes.Count == 0) return; + // foreach (var subscribeType in subscribeTypes) + // { + // var subscribes = provider.GetServices(subscribeType).ToList(); + // subscribes.ForEach(subscribe => { + + // if (subscribe is IKafkaSubscribe) + // { + // Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger); + // threadCount += tuple.Item1; + // topicCount += tuple.Item2; + // } + // }); + // } + // } + // logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程"); + // }); + //} + + public static void UseKafkaSubscribers(this IApplicationBuilder app, Assembly assembly) { var provider = app.ApplicationServices; var lifetime = provider.GetRequiredService(); @@ -27,26 +62,23 @@ namespace JiShe.CollectBus.Kafka var logger = provider.GetRequiredService>(); int threadCount = 0; int topicCount = 0; - foreach (Assembly assembly in assemblys) - { - var subscribeTypes = assembly.GetTypes() + var subscribeTypes = assembly.GetTypes() .Where(t => typeof(IKafkaSubscribe).IsAssignableFrom(t)) .ToList(); - if (subscribeTypes.Count == 0) return; - foreach (var subscribeType in subscribeTypes) - { - var subscribes = provider.GetServices(subscribeType).ToList(); - subscribes.ForEach(subscribe => { + if (subscribeTypes.Count == 0) return; + foreach (var subscribeType in subscribeTypes) + { + var subscribes = provider.GetServices(subscribeType).ToList(); + subscribes.ForEach(subscribe => { - if (subscribe is IKafkaSubscribe) - { - Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger); - threadCount += tuple.Item1; - topicCount += tuple.Item2; - } - }); - } + if (subscribe is IKafkaSubscribe) + { + Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger); + threadCount += tuple.Item1; + topicCount += tuple.Item2; + } + }); } logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程"); }); diff --git a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs index 159f0fc..52ef129 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs +++ b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs @@ -12,6 +12,7 @@ using JiShe.CollectBus.IotSystems.MessageReceiveds; using JiShe.CollectBus.IotSystems.Protocols; using MassTransit; using DotNetCore.CAP; +using JiShe.CollectBus.Kafka.Producer; namespace JiShe.CollectBus.Protocol.Contracts.Abstracts { From 96e066376f9b08f00e35d119ac1df61f7894a0c6 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Thu, 17 Apr 2025 13:01:26 +0800 Subject: [PATCH 20/27] =?UTF-8?q?=E5=A2=9E=E5=8A=A0kafka=E8=AE=A2=E9=98=85?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=8F=96=E7=A8=8B=E5=BA=8F=E6=89=80=E6=9C=89?= =?UTF-8?q?=E7=9A=84=E8=AE=A2=E9=98=85=E8=80=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CollectBusKafkaModule.cs | 37 +------- .../IKafkaSubscribe.cs | 6 ++ .../KafkaSubcribesExtensions.cs | 92 +++++++++++-------- 3 files changed, 64 insertions(+), 71 deletions(-) diff --git a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs index 80739a6..0444412 100644 --- a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs +++ b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs @@ -34,39 +34,12 @@ namespace JiShe.CollectBus.Kafka public override void OnApplicationInitialization(ApplicationInitializationContext context) { var app = context.GetApplicationBuilder(); - // 程序运行目录 - //var assemblyPath = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location); - //if (!string.IsNullOrWhiteSpace(assemblyPath)) - //{ - // var dllFiles = Directory.GetFiles(assemblyPath, "*.dll"); - // var kafkaSubscriberAssemblies = new List(); - // foreach (var file in dllFiles) - // { - // try - // { - // // 跳过已加载的程序集 - // var assemblyName = AssemblyName.GetAssemblyName(file); - // var existingAssembly = AppDomain.CurrentDomain.GetAssemblies() - // .FirstOrDefault(a => a.GetName().FullName == assemblyName.FullName); - // var assembly = existingAssembly ?? Assembly.LoadFrom(file); - - // // 检查程序集是否包含 IKafkaSubscribe 的实现类 - // var hasSubscriber = assembly.GetTypes() - // .Any(type => - // typeof(IKafkaSubscribe).IsAssignableFrom(type) && // 实现接口 - // !type.IsAbstract && !type.IsInterface); // 排除抽象类和接口本身 - - // if (hasSubscriber) - // { - // kafkaSubscriberAssemblies.Add(assembly); - // } - // } - // catch{} - // app.UseKafkaSubscribers(kafkaSubscriberAssemblies.ToArray()); - // } - //} + + // 注册Subscriber + app.ApplicationServices.UseKafkaSubscribers(); + // 获取程序集 - app.UseKafkaSubscribers(Assembly.Load("JiShe.CollectBus.Application")); + //app.UseKafkaSubscribers(Assembly.Load("JiShe.CollectBus.Application")); } } } diff --git a/src/JiShe.CollectBus.KafkaProducer/IKafkaSubscribe.cs b/src/JiShe.CollectBus.KafkaProducer/IKafkaSubscribe.cs index 3ccea65..39e5789 100644 --- a/src/JiShe.CollectBus.KafkaProducer/IKafkaSubscribe.cs +++ b/src/JiShe.CollectBus.KafkaProducer/IKafkaSubscribe.cs @@ -6,6 +6,12 @@ using System.Threading.Tasks; namespace JiShe.CollectBus.Kafka { + /// + /// Kafka订阅者 + /// + /// 订阅者需要继承此接口并需要依赖注入，并使用标记 + /// + /// public interface IKafkaSubscribe { } diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index 4c69b6a..8f2974a 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -1,8 +1,10 @@ using Confluent.Kafka; +using JiShe.CollectBus.Kafka; using JiShe.CollectBus.Kafka.AdminClient; using JiShe.CollectBus.Kafka.Attributes; using JiShe.CollectBus.Kafka.Consumer; using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Mvc.ApplicationParts; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -12,45 +14,57 @@ namespace JiShe.CollectBus.Kafka { public static class KafkaSubcribesExtensions { - ///// - ///// 添加Kafka订阅 - ///// - ///// - ///// - //public static void UseKafkaSubscribers(this IApplicationBuilder app, Assembly[] assemblys) - //{ - // var provider = app.ApplicationServices; - // var lifetime = provider.GetRequiredService(); + /// + /// 添加Kafka订阅 + /// + /// + /// + public static void UseKafkaSubscribers(this IServiceProvider provider) + { + var lifetime = provider.GetRequiredService(); - // lifetime.ApplicationStarted.Register(() => - // { - // var logger = provider.GetRequiredService>(); - // int threadCount = 0; - // int topicCount = 0; - // foreach (Assembly assembly in assemblys) - // { - // var subscribeTypes = assembly.GetTypes() - // .Where(t => typeof(IKafkaSubscribe).IsAssignableFrom(t)) - // .ToList(); - - // if (subscribeTypes.Count == 0) return; - // foreach (var subscribeType in subscribeTypes) - // { - // var subscribes = provider.GetServices(subscribeType).ToList(); - // subscribes.ForEach(subscribe => { - - // if (subscribe is IKafkaSubscribe) - // { - // Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger); - // threadCount += tuple.Item1; - // topicCount += tuple.Item2; - // } - // }); - // } - // } - // logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程"); - // }); - //} + lifetime.ApplicationStarted.Register(() => + { + var logger = provider.GetRequiredService>(); + int threadCount = 0; + int topicCount = 0; + var assemblyPath = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location); + if (string.IsNullOrWhiteSpace(assemblyPath)) + { + logger.LogInformation($"kafka订阅未能找到程序路径"); + return; + } + var dllFiles = Directory.GetFiles(assemblyPath, "*.dll"); + foreach (var file in dllFiles) + { + // 跳过已加载的程序集 + var assemblyName = AssemblyName.GetAssemblyName(file); + var existingAssembly = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => a.GetName().FullName == assemblyName.FullName); + var assembly = existingAssembly ?? Assembly.LoadFrom(file); + // 实现IKafkaSubscribe接口 + var subscribeTypes = assembly.GetTypes().Where(type => + typeof(IKafkaSubscribe).IsAssignableFrom(type) && + !type.IsAbstract && !type.IsInterface).ToList(); ; + if (subscribeTypes.Count == 0) + continue; + foreach (var subscribeType in subscribeTypes) + { + var subscribes = provider.GetServices(subscribeType).ToList(); + subscribes.ForEach(subscribe => + { + if (subscribe!=null) + { + Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger); + threadCount += tuple.Item1; + topicCount += tuple.Item2; + } + }); + } + } + logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程"); + }); + } public static void UseKafkaSubscribers(this IApplicationBuilder app, Assembly assembly) { @@ -72,7 +86,7 @@ namespace JiShe.CollectBus.Kafka var subscribes = provider.GetServices(subscribeType).ToList(); subscribes.ForEach(subscribe => { - if (subscribe is IKafkaSubscribe) + if (subscribe != null) { Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger); threadCount += tuple.Item1; From 6c8ffb3ae5429f18d925dda5b04fea80610fc7d6 Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Thu, 17 Apr 2025 13:35:08 +0800 Subject: [PATCH 21/27] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/JiShe.CollectBus.Host/appsettings.json | 8 +++---- .../CollectBusIoTDBModule.cs | 9 ++++++- .../CollectBusKafkaModule.cs | 24 +++++++++++++------ .../JiShe.CollectBus.Kafka.csproj | 4 ++++ .../KafkaOptionConfig.cs | 10 ++++++++ .../KafkaSubcribesExtensions.cs | 9 ++++--- 6 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/JiShe.CollectBus.Host/appsettings.json b/src/JiShe.CollectBus.Host/appsettings.json index 7aa3947..735ce0d 100644 --- a/src/JiShe.CollectBus.Host/appsettings.json +++ b/src/JiShe.CollectBus.Host/appsettings.json @@ -89,7 +89,9 @@ "SecurityProtocol": "SaslPlaintext", "SaslMechanism": "Plain", "SaslUserName": "lixiao", - "SaslPassword": "lixiao1980" + "SaslPassword": "lixiao1980", + "KafkaReplicationFactor": 3, + "NumPartitions": 30 //"Topic": { // "ReplicationFactor": 3, // "NumPartitions": 1000 @@ -130,9 +132,7 @@ "OpenDebugMode": true, "UseTableSessionPoolByDefault": false }, - "ServerTagName": "JiSheCollectBus3", - "KafkaReplicationFactor": 3, - "NumPartitions": 30, + "ServerTagName": "JiSheCollectBus3", "Cassandra": { "ReplicationStrategy": { "Class": "NetworkTopologyStrategy", //策略为NetworkTopologyStrategy时才会有多个数据中心，SimpleStrategy用在只有一个数据中心的情况下 diff --git a/src/JiShe.CollectBus.IoTDBProvider/CollectBusIoTDBModule.cs b/src/JiShe.CollectBus.IoTDBProvider/CollectBusIoTDBModule.cs index 444ab4e..62c63c4 100644 --- a/src/JiShe.CollectBus.IoTDBProvider/CollectBusIoTDBModule.cs +++ b/src/JiShe.CollectBus.IoTDBProvider/CollectBusIoTDBModule.cs @@ -1,6 +1,7 @@ using JiShe.CollectBus.IoTDBProvider.Context; using JiShe.CollectBus.IoTDBProvider.Interface; using JiShe.CollectBus.IoTDBProvider.Provider; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using System; using System.Collections.Generic; @@ -8,6 +9,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using Volo.Abp.Modularity; +using static Thrift.Server.TThreadPoolAsyncServer; namespace JiShe.CollectBus.IoTDBProvider { @@ -15,7 +17,12 @@ namespace JiShe.CollectBus.IoTDBProvider { public override void ConfigureServices(ServiceConfigurationContext context) { - context.Services.Configure(context.Services.GetConfiguration().GetSection(nameof(IoTDBOptions))); + + var configuration = context.Services.GetConfiguration(); + Configure(options => + { + configuration.GetSection(nameof(IoTDBOptions)).Bind(options); + }); // 注册上下文为Scoped context.Services.AddScoped(); diff --git a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs index 80739a6..5740c7f 100644 --- a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs +++ b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs @@ -1,4 +1,5 @@ using Confluent.Kafka; +using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Kafka.Consumer; using JiShe.CollectBus.Kafka.Producer; using Microsoft.AspNetCore.Builder; @@ -17,14 +18,23 @@ namespace JiShe.CollectBus.Kafka public override void ConfigureServices(ServiceConfigurationContext context) { var configuration = context.Services.GetConfiguration(); - var kafkaSection = configuration.GetSection("Kafka"); - KafkaOptionConfig kafkaOptionConfig = new KafkaOptionConfig (); - kafkaSection.Bind(kafkaOptionConfig); - if (configuration["ServerTagName"] != null) + //var kafkaSection = configuration.GetSection(CommonConst.Kafka); + //KafkaOptionConfig kafkaOptionConfig = new KafkaOptionConfig (); + //kafkaSection.Bind(kafkaOptionConfig); + //if (configuration[CommonConst.ServerTagName] != null) + //{ + // kafkaOptionConfig.ServerTagName = configuration[CommonConst.ServerTagName]!; + //} + //context.Services.AddSingleton(kafkaOptionConfig); + + context.Services.Configure(context.Services.GetConfiguration().GetSection(CommonConst.Kafka)); + + Configure(options => { - kafkaOptionConfig.ServerTagName = configuration["ServerTagName"]!; - } - context.Services.AddSingleton(kafkaOptionConfig); + configuration.GetSection(CommonConst.Kafka).Bind(options); + }); + + // 注册Producer context.Services.AddSingleton(); // 注册Consumer diff --git a/src/JiShe.CollectBus.KafkaProducer/JiShe.CollectBus.Kafka.csproj b/src/JiShe.CollectBus.KafkaProducer/JiShe.CollectBus.Kafka.csproj index cef24d5..9518b0b 100644 --- a/src/JiShe.CollectBus.KafkaProducer/JiShe.CollectBus.Kafka.csproj +++ b/src/JiShe.CollectBus.KafkaProducer/JiShe.CollectBus.Kafka.csproj @@ -12,4 +12,8 @@ + + + + diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs index 50ae47e..28b80a5 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs @@ -19,6 +19,16 @@ namespace JiShe.CollectBus.Kafka /// public string ServerTagName { get; set; }= "KafkaFilterKey"; + /// + /// kafka主题副本数量 + /// + public int KafkaReplicationFactor { get; set; } + + /// + /// kafka主题分区数量 + /// + public int NumPartitions { get; set; } + /// /// 是否开启过滤器 /// diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index 4c69b6a..d04ad4f 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -1,8 +1,10 @@ using Confluent.Kafka; +using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Kafka.AdminClient; using JiShe.CollectBus.Kafka.Attributes; using JiShe.CollectBus.Kafka.Consumer; using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -95,12 +97,13 @@ namespace JiShe.CollectBus.Kafka .Select(m => new { Method = m, Attribute = m.GetCustomAttribute() }) .Where(x => x.Attribute != null) .ToArray(); - + var configuration = provider.GetRequiredService(); int threadCount = 0; foreach (var sub in subscribedMethods) { - var adminClientService = provider.GetRequiredService(); - int partitionCount = sub.Attribute!.TaskCount==-1?adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic) : sub.Attribute!.TaskCount; + int partitionCount = configuration.GetValue(CommonConst.NumPartitions); + //var adminClientService = provider.GetRequiredService(); + //int partitionCount = sub.Attribute!.TaskCount==-1?adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic) : sub.Attribute!.TaskCount; if (partitionCount <= 0) partitionCount = 1; for (int i = 0; i < partitionCount; i++) From 4c5f7231bf7dcd267ab30ac8c4c214c40630a653 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Thu, 17 Apr 2025 13:41:53 +0800 Subject: [PATCH 22/27] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=9C=AA=E5=A4=9A?= =?UTF-8?q?=E4=BD=99=E7=9A=84=E9=85=8D=E7=BD=AE=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../KafkaOptions.cs | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 src/JiShe.CollectBus.KafkaProducer/KafkaOptions.cs diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaOptions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaOptions.cs deleted file mode 100644 index d946cc8..0000000 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaOptions.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace JiShe.CollectBus.Kafka -{ - public class KafkaOptions - { - public string BootstrapServers { get; set; } - public string GroupId { get; set; } - public Dictionary ProducerConfig { get; set; } = new(); - public Dictionary ConsumerConfig { get; set; } = new(); - public Dictionary AdminConfig { get; set; } = new(); - } -} From 6c0ce01634bcb6ee49fe6759c7406b1be486f4b4 Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Thu, 17 Apr 2025 13:54:18 +0800 Subject: [PATCH 23/27] =?UTF-8?q?=E4=BC=98=E5=8C=96Kafka=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E9=A1=B9=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BasicScheduledMeterReadingService.cs | 14 ++++++++------ .../EnergySystemScheduledMeterReadingService.cs | 12 +++++++----- .../CollectBusCassandraModule.cs | 4 ++-- .../Helpers/BusJsonSerializer.cs | 6 ++++-- src/JiShe.CollectBus.Host/appsettings.json | 3 ++- .../CollectBusKafkaModule.cs | 2 +- .../Consumer/ConsumerService.cs | 5 +++-- .../Producer/ProducerService.cs | 5 +++-- 8 files changed, 30 insertions(+), 21 deletions(-) diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index eac2a92..9184a62 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -13,12 +13,14 @@ using JiShe.CollectBus.IoTDBProvider; using JiShe.CollectBus.IotSystems.MessageIssueds; using JiShe.CollectBus.IotSystems.MeterReadingRecords; using JiShe.CollectBus.IotSystems.Watermeter; +using JiShe.CollectBus.Kafka; using JiShe.CollectBus.Kafka.Producer; using JiShe.CollectBus.Protocol.Contracts; using JiShe.CollectBus.RedisDataCache; using JiShe.CollectBus.Repository.MeterReadingRecord; using Mapster; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using System; using System.Collections.Generic; using System.Diagnostics; @@ -38,22 +40,22 @@ namespace JiShe.CollectBus.ScheduledMeterReading private readonly IMeterReadingRecordRepository _meterReadingRecordRepository; private readonly IProducerService _producerService; private readonly IRedisDataCacheService _redisDataCacheService; - private readonly ICapPublisher _producerBus; + private readonly KafkaOptionConfig _kafkaOptions; public BasicScheduledMeterReadingService( ILogger logger, - ICapPublisher producerBus, IMeterReadingRecordRepository meterReadingRecordRepository, IProducerService producerService, IRedisDataCacheService redisDataCacheService, - IIoTDBProvider dbProvider) + IIoTDBProvider dbProvider, + IOptions kafkaOptions) { - _producerBus = producerBus; _logger = logger; _dbProvider = dbProvider; _meterReadingRecordRepository = meterReadingRecordRepository; _producerService = producerService; _redisDataCacheService = redisDataCacheService; + _kafkaOptions = kafkaOptions.Value; } /// @@ -302,7 +304,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading timer1.Stop(); _logger.LogError($"读取数据更花费时间{timer1.ElapsedMilliseconds}毫秒"); - //DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista); + //DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista, _kafkaOptions.NumPartitions); //return; #else var meterInfos = await GetAmmeterInfoList(gatherCode); @@ -428,7 +430,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading } else { - DeviceGroupBalanceControl.InitializeCache(focusAddressDataList); + DeviceGroupBalanceControl.InitializeCache(focusAddressDataList, _kafkaOptions.NumPartitions); } timer.Stop(); diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs index 25c5476..cfb0193 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/EnergySystemScheduledMeterReadingService.cs @@ -15,6 +15,7 @@ using JiShe.CollectBus.IotSystems.Devices; using JiShe.CollectBus.IotSystems.MessageIssueds; using JiShe.CollectBus.IotSystems.MeterReadingRecords; using JiShe.CollectBus.IotSystems.Watermeter; +using JiShe.CollectBus.Kafka; using JiShe.CollectBus.Kafka.Producer; using JiShe.CollectBus.Repository; using JiShe.CollectBus.Repository.MeterReadingRecord; @@ -23,6 +24,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Volo.Abp.Domain.Repositories; using Volo.Abp.Uow; @@ -38,19 +40,19 @@ namespace JiShe.CollectBus.ScheduledMeterReading string serverTagName = string.Empty; public EnergySystemScheduledMeterReadingService( ILogger logger, - ICapPublisher producerBus, IIoTDBProvider dbProvider, + IIoTDBProvider dbProvider, IMeterReadingRecordRepository meterReadingRecordRepository, - IConfiguration configuration, + IOptions kafkaOptions, IProducerService producerService, IRedisDataCacheService redisDataCacheService) : base(logger, - producerBus, meterReadingRecordRepository, producerService, redisDataCacheService, - dbProvider) + dbProvider, + kafkaOptions) { - serverTagName = configuration.GetValue(CommonConst.ServerTagName)!; + serverTagName = kafkaOptions.Value.ServerTagName; } public sealed override string SystemType => SystemTypeConst.Energy; diff --git a/src/JiShe.CollectBus.Cassandra/CollectBusCassandraModule.cs b/src/JiShe.CollectBus.Cassandra/CollectBusCassandraModule.cs index 2502420..2d70f07 100644 --- a/src/JiShe.CollectBus.Cassandra/CollectBusCassandraModule.cs +++ b/src/JiShe.CollectBus.Cassandra/CollectBusCassandraModule.cs @@ -17,13 +17,13 @@ namespace JiShe.CollectBus.Cassandra { Configure(context.Services.GetConfiguration().GetSection("Cassandra")); - context.AddCassandra(); + // context.AddCassandra(); } public override void OnApplicationInitialization(ApplicationInitializationContext context) { - context.UseCassandra(); + // context.UseCassandra(); } } } diff --git a/src/JiShe.CollectBus.Common/Helpers/BusJsonSerializer.cs b/src/JiShe.CollectBus.Common/Helpers/BusJsonSerializer.cs index f938fd8..713501e 100644 --- a/src/JiShe.CollectBus.Common/Helpers/BusJsonSerializer.cs +++ b/src/JiShe.CollectBus.Common/Helpers/BusJsonSerializer.cs @@ -37,7 +37,8 @@ namespace JiShe.CollectBus.Common.Helpers ReadCommentHandling = JsonCommentHandling.Skip, // 忽略注释 PropertyNameCaseInsensitive = true, // 属性名称大小写不敏感 PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // 属性名称使用驼峰命名规则 - Converters = { new DateTimeJsonConverter() } // 注册你的自定义转换器, + Converters = { new DateTimeJsonConverter() }, // 注册你的自定义转换器, + DefaultBufferSize = 4096, }; } @@ -77,7 +78,8 @@ namespace JiShe.CollectBus.Common.Helpers ReadCommentHandling = JsonCommentHandling.Skip, // 忽略注释 PropertyNameCaseInsensitive = true, // 属性名称大小写不敏感 PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // 属性名称使用驼峰命名规则 - Converters = { new DateTimeJsonConverter() } // 注册你的自定义转换器, + Converters = { new DateTimeJsonConverter() }, // 注册你的自定义转换器, + DefaultBufferSize = 4096, }; } diff --git a/src/JiShe.CollectBus.Host/appsettings.json b/src/JiShe.CollectBus.Host/appsettings.json index 735ce0d..501a2a0 100644 --- a/src/JiShe.CollectBus.Host/appsettings.json +++ b/src/JiShe.CollectBus.Host/appsettings.json @@ -91,7 +91,8 @@ "SaslUserName": "lixiao", "SaslPassword": "lixiao1980", "KafkaReplicationFactor": 3, - "NumPartitions": 30 + "NumPartitions": 30, + "ServerTagName": "JiSheCollectBus3" //"Topic": { // "ReplicationFactor": 3, // "NumPartitions": 1000 diff --git a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs index 5740c7f..20c01fd 100644 --- a/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs +++ b/src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs @@ -27,7 +27,7 @@ namespace JiShe.CollectBus.Kafka //} //context.Services.AddSingleton(kafkaOptionConfig); - context.Services.Configure(context.Services.GetConfiguration().GetSection(CommonConst.Kafka)); + //context.Services.Configure(context.Services.GetConfiguration().GetSection(CommonConst.Kafka)); Configure(options => { diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs index eeb5661..24f2029 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs @@ -1,6 +1,7 @@ using Confluent.Kafka; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using System.Collections.Concurrent; using System.Text; @@ -15,11 +16,11 @@ namespace JiShe.CollectBus.Kafka.Consumer private readonly KafkaOptionConfig _kafkaOptionConfig; private class KafkaConsumer where TKey : notnull where TValue : class { } - public ConsumerService(IConfiguration configuration, ILogger logger, KafkaOptionConfig kafkaOptionConfig) + public ConsumerService(IConfiguration configuration, ILogger logger, IOptions kafkaOptionConfig) { _configuration = configuration; _logger = logger; - _kafkaOptionConfig = kafkaOptionConfig; + _kafkaOptionConfig = kafkaOptionConfig.Value; } #region private 私有方法 diff --git a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs index 27702e0..42fc9cf 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs @@ -8,6 +8,7 @@ using Confluent.Kafka; using JiShe.CollectBus.Kafka.Consumer; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Volo.Abp.DependencyInjection; using YamlDotNet.Serialization; @@ -20,11 +21,11 @@ namespace JiShe.CollectBus.Kafka.Producer private readonly ConcurrentDictionary _producerCache = new(); private class KafkaProducer where TKey : notnull where TValue : class { } private readonly KafkaOptionConfig _kafkaOptionConfig; - public ProducerService(IConfiguration configuration,ILogger logger, KafkaOptionConfig kafkaOptionConfig) + public ProducerService(IConfiguration configuration,ILogger logger, IOptions kafkaOptionConfig) { _configuration = configuration; _logger = logger; - _kafkaOptionConfig = kafkaOptionConfig; + _kafkaOptionConfig = kafkaOptionConfig.Value; } #region private 私有方法 From 72f8222ec27e7c1ff91834a97d93fca5523c5785 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Thu, 17 Apr 2025 13:56:17 +0800 Subject: [PATCH 24/27] =?UTF-8?q?kafka=E6=B6=88=E8=B4=B9=E8=80=85=E8=AE=A2?= =?UTF-8?q?=E9=98=85=E5=A2=9E=E5=8A=A0=E6=89=B9=E9=87=8F=E6=B6=88=E8=B4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../KafkaSubcribesExtensions.cs | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index 73ea14c..e830dc4 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -138,21 +138,42 @@ namespace JiShe.CollectBus.Kafka private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr,MethodInfo method, object subscribe, ILogger logger) { var consumerService = provider.GetRequiredService(); - - await consumerService.SubscribeAsync(attr.Topic, async (message) => + + if (attr.EnableBatch) { - try + await consumerService.SubscribeBatchAsync(attr.Topic, async (message) => { - // 处理消息 - return await ProcessMessageAsync(message, method, subscribe); - } - catch (ConsumeException ex) + try + { + // 处理消息 + return await ProcessMessageAsync(message, method, subscribe); + } + catch (ConsumeException ex) + { + // 处理消费错误 + logger.LogError($"kafka批量消费异常:{ex.Message}"); + } + return await Task.FromResult(false); + }); + } + else + { + await consumerService.SubscribeAsync(attr.Topic, async (message) => { - // 处理消费错误 - logger.LogError($"kafka消费异常:{ex.Message}"); - } - return await Task.FromResult(false); - }); + try + { + // 处理消息 + return await ProcessMessageAsync(message, method, subscribe); + } + catch (ConsumeException ex) + { + // 处理消费错误 + logger.LogError($"kafka消费异常:{ex.Message}"); + } + return await Task.FromResult(false); + }); + } + } From 871ed615a44b314872e90c5ed2a26e895fedfe02 Mon Sep 17 00:00:00 2001 From: ChenYi <296215406@outlook.com> Date: Thu, 17 Apr 2025 14:12:02 +0800 Subject: [PATCH 25/27] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CollectBusApplicationModule.cs | 4 +- .../RedisDataCache/RedisDataCacheService.cs | 174 ++++++------------ .../KafkaOptionConfig.cs | 2 +- 3 files changed, 61 insertions(+), 119 deletions(-) diff --git a/src/JiShe.CollectBus.Application/CollectBusApplicationModule.cs b/src/JiShe.CollectBus.Application/CollectBusApplicationModule.cs index f3ed978..1826fa4 100644 --- a/src/JiShe.CollectBus.Application/CollectBusApplicationModule.cs +++ b/src/JiShe.CollectBus.Application/CollectBusApplicationModule.cs @@ -22,6 +22,7 @@ using Volo.Abp.BackgroundWorkers; using Volo.Abp.BackgroundWorkers.Hangfire; using Volo.Abp.EventBus; using Volo.Abp.Modularity; +using Microsoft.Extensions.Options; namespace JiShe.CollectBus; @@ -69,13 +70,14 @@ public class CollectBusApplicationModule : AbpModule //初始化主题信息 var kafkaAdminClient = context.ServiceProvider.GetRequiredService(); var configuration = context.ServiceProvider.GetRequiredService(); + var kafkaOptions = context.ServiceProvider.GetRequiredService>(); List topics = ProtocolConstExtensions.GetAllTopicNamesByIssued(); topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived()); foreach (var item in topics) { - await kafkaAdminClient.CreateTopicAsync(item, configuration.GetValue(CommonConst.NumPartitions), configuration.GetValue(CommonConst.KafkaReplicationFactor)); + await kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor); } } diff --git a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs index 3c96410..0d9c80d 100644 --- a/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs +++ b/src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs @@ -389,79 +389,79 @@ namespace JiShe.CollectBus.RedisDataCache pageSize = Math.Clamp(pageSize, 1, 10000); const string luaScript = @" - local command = ARGV[1] - local range_start = ARGV[2] - local range_end = ARGV[3] - local limit = tonumber(ARGV[4]) - local last_score = ARGV[5] - local last_member = ARGV[6] + local command = ARGV[1] + local range_start = ARGV[2] + local range_end = ARGV[3] + local limit = tonumber(ARGV[4]) + local last_score = ARGV[5] + local last_member = ARGV[6] - -- 获取扩展数据（3倍分页大小） - local members - if command == 'ZRANGEBYSCORE' then - members = redis.call('ZRANGEBYSCORE', KEYS[1], range_start, range_end, - 'WITHSCORES', 'LIMIT', 0, limit * 5) - else - members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, - 'WITHSCORES', 'LIMIT', 0, limit * 5) - end + -- 获取扩展数据（5倍分页大小） + local members + if command == 'ZRANGEBYSCORE' then + members = redis.call('ZRANGEBYSCORE', KEYS[1], range_start, range_end, + 'WITHSCORES', 'LIMIT', 0, limit * 5) + else + members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, + 'WITHSCORES', 'LIMIT', 0, limit * 5) + end - -- 精确分页过滤 - local filtered = {} - local count = 0 - local start_index = 1 + -- 精确分页过滤 + local filtered = {} + local count = 0 + local start_index = 1 - -- 存在锚点时寻找起始位置 - if last_score ~= '' and last_member ~= '' then - for i=1,#members,2 do - local score = members[i+1] - local member = members[i] + -- 存在锚点时寻找起始位置 + if last_score ~= '' and last_member ~= '' then + for i=1,#members,2 do + local score = members[i+1] + local member = members[i] - if command == 'ZRANGEBYSCORE' then - if tonumber(score) > tonumber(last_score) then - start_index = i - break - elseif tonumber(score) == tonumber(last_score) then - if member > last_member then + if command == 'ZRANGEBYSCORE' then + if tonumber(score) > tonumber(last_score) then start_index = i break + elseif tonumber(score) == tonumber(last_score) then + if member > last_member then + start_index = i + break + end end - end - else - if tonumber(score) < tonumber(last_score) then - start_index = i - break - elseif tonumber(score) == tonumber(last_score) then - if member < last_member then + else + if tonumber(score) < tonumber(last_score) then start_index = i break + elseif tonumber(score) == tonumber(last_score) then + if member < last_member then + start_index = i + break + end end end end end - end - -- 收集有效数据 - for i=start_index,#members,2 do - if count >= limit then break end - table.insert(filtered, members[i]) - table.insert(filtered, members[i+1]) - count = count + 1 - end + -- 收集有效数据 + for i=start_index,#members,2 do + if count >= limit then break end + table.insert(filtered, members[i]) + table.insert(filtered, members[i+1]) + count = count + 1 + end - -- 提取有效数据 - local result_members = {} - local result_scores = {} - for i=1,#filtered,2 do - table.insert(result_members, filtered[i]) - table.insert(result_scores, filtered[i+1]) - end + -- 提取有效数据 + local result_members = {} + local result_scores = {} + for i=1,#filtered,2 do + table.insert(result_members, filtered[i]) + table.insert(result_scores, filtered[i+1]) + end - if #result_members == 0 then return {0,{},{},{}} end + if #result_members == 0 then return {0,{},{},{}} end - -- 获取Hash数据 - local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members)) - return {#result_members, result_members, result_scores, hash_data}"; + -- 获取Hash数据 + local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members)) + return {#result_members, result_members, result_scores, hash_data}"; // 构造查询范围（包含等于） string rangeStart, rangeEnd; @@ -540,67 +540,7 @@ namespace JiShe.CollectBus.RedisDataCache return new BusCacheGlobalPagedResult { Items = new List() }; } } - - /// - /// 并行分页导出方法（百万级数据支持） - /// - public async Task> FullExportParallel( - string hashKey, - string zsetKey, - int parallelDegree = 10, - int pageSize = 5000) where T : DeviceCacheBasicModel - { - var result = new ConcurrentBag(); - var totalCount = await GetTotalCount(zsetKey); - var totalPages = (int)Math.Ceiling(totalCount / (double)pageSize); - - var semaphore = new SemaphoreSlim(parallelDegree); - var tasks = new List(); - - decimal? lastScore = null; - string lastMember = null; - var isDescending = true; - - for (int page = 0; page < totalPages; page++) - { - await semaphore.WaitAsync(); - - tasks.Add(Task.Run(async () => - { - try - { - var pageResult = await GetAllPagedData( - hashKey, - zsetKey, - pageSize, - lastScore, - lastMember, - isDescending); - - foreach (var item in pageResult.Items) - { - result.Add(item); - } - - // 更新分页锚点 - if (pageResult.HasNext) - { - lastScore = pageResult.NextScore; - lastMember = pageResult.NextMember; - } - } - finally - { - semaphore.Release(); - } - })); - } - - await Task.WhenAll(tasks); - return result.ToList(); - } - - + /// /// 通过ZSET索引获取数据，支持10万级别数据处理，控制在13秒以内。 /// diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs index 28b80a5..e592ea2 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs @@ -22,7 +22,7 @@ namespace JiShe.CollectBus.Kafka /// /// kafka主题副本数量 /// - public int KafkaReplicationFactor { get; set; } + public short KafkaReplicationFactor { get; set; } /// /// kafka主题分区数量 From 0ba64a4b903d54c3660f6ad4bb835233dd757696 Mon Sep 17 00:00:00 2001 From: zenghongyao <873884283@qq.com> Date: Thu, 17 Apr 2025 14:39:14 +0800 Subject: [PATCH 26/27] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8F=91=E5=B8=83?= =?UTF-8?q?=E8=AE=A2=E9=98=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EnergySystem/EnergySystemAppService.cs | 1 + .../Plugins/TcpMonitor.cs | 1 + .../Samples/SampleAppService.cs | 2 +- .../Subscribers/SubscriberAppService.cs | 1 + .../Subscribers/WorkerSubscriberAppService.cs | 1 + .../Consts}/ProtocolConst.cs | 2 +- .../Extensions/ProtocolConstExtensions.cs | 5 ++- .../CollectBusHostModule.Configure.cs | 40 ------------------- .../Consumer/ConsumerService.cs | 2 +- .../KafkaSubcribesExtensions.cs | 39 ++++++++++++++---- .../Abstracts/BaseProtocolPlugin.cs | 1 + ...JiShe.CollectBus.Protocol.Contracts.csproj | 4 ++ 12 files changed, 47 insertions(+), 52 deletions(-) rename src/{JiShe.CollectBus.Protocol.Contracts => JiShe.CollectBus.Common/Consts}/ProtocolConst.cs (99%) rename src/{JiShe.CollectBus.Protocol.Contracts => JiShe.CollectBus.Common}/Extensions/ProtocolConstExtensions.cs (95%) diff --git a/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs b/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs index 1ca731b..9fc8dd4 100644 --- a/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs +++ b/src/JiShe.CollectBus.Application/EnergySystem/EnergySystemAppService.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using DeviceDetectorNET.Class.Device; using DotNetCore.CAP; using JiShe.CollectBus.Common.BuildSendDatas; +using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; using JiShe.CollectBus.Common.Helpers; diff --git a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs index 6cbc8c4..c2bd026 100644 --- a/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs +++ b/src/JiShe.CollectBus.Application/Plugins/TcpMonitor.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using DeviceDetectorNET.Parser.Device; using DotNetCore.CAP; using JiShe.CollectBus.Ammeters; +using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; using JiShe.CollectBus.Common.Helpers; diff --git a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs index 15b094a..09bd962 100644 --- a/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs +++ b/src/JiShe.CollectBus.Application/Samples/SampleAppService.cs @@ -265,7 +265,7 @@ public class SampleAppService : CollectBusAppService, ISampleAppService, IKafkaS return aa == null; } - [KafkaSubscribe("test-topic1")] + //[KafkaSubscribe("test-topic1")] public async Task KafkaSubscribeAsync(object obj) { diff --git a/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs b/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs index 3ec6936..ce41db1 100644 --- a/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application/Subscribers/SubscriberAppService.cs @@ -1,4 +1,5 @@ using DotNetCore.CAP; +using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Common.Models; diff --git a/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs b/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs index 47cff47..65ecc01 100644 --- a/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs +++ b/src/JiShe.CollectBus.Application/Subscribers/WorkerSubscriberAppService.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using DeviceDetectorNET.Parser.Device; using DotNetCore.CAP; +using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.IotSystems.Devices; using JiShe.CollectBus.IotSystems.MessageIssueds; diff --git a/src/JiShe.CollectBus.Protocol.Contracts/ProtocolConst.cs b/src/JiShe.CollectBus.Common/Consts/ProtocolConst.cs similarity index 99% rename from src/JiShe.CollectBus.Protocol.Contracts/ProtocolConst.cs rename to src/JiShe.CollectBus.Common/Consts/ProtocolConst.cs index d1e5739..df96c29 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/ProtocolConst.cs +++ b/src/JiShe.CollectBus.Common/Consts/ProtocolConst.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace JiShe.CollectBus.Protocol.Contracts +namespace JiShe.CollectBus.Common.Consts { public class ProtocolConst { diff --git a/src/JiShe.CollectBus.Protocol.Contracts/Extensions/ProtocolConstExtensions.cs b/src/JiShe.CollectBus.Common/Extensions/ProtocolConstExtensions.cs similarity index 95% rename from src/JiShe.CollectBus.Protocol.Contracts/Extensions/ProtocolConstExtensions.cs rename to src/JiShe.CollectBus.Common/Extensions/ProtocolConstExtensions.cs index fc52f31..1dbb301 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/Extensions/ProtocolConstExtensions.cs +++ b/src/JiShe.CollectBus.Common/Extensions/ProtocolConstExtensions.cs @@ -1,4 +1,5 @@ -using JiShe.CollectBus.Common.Enums; +using JiShe.CollectBus.Common.Consts; +using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Extensions; using System; using System.Collections.Generic; @@ -7,7 +8,7 @@ using System.Reflection; using System.Text; using System.Threading.Tasks; -namespace JiShe.CollectBus.Protocol.Contracts +namespace JiShe.CollectBus.Common.Extensions { public class ProtocolConstExtensions { diff --git a/src/JiShe.CollectBus.Host/CollectBusHostModule.Configure.cs b/src/JiShe.CollectBus.Host/CollectBusHostModule.Configure.cs index 35bb46b..82cc214 100644 --- a/src/JiShe.CollectBus.Host/CollectBusHostModule.Configure.cs +++ b/src/JiShe.CollectBus.Host/CollectBusHostModule.Configure.cs @@ -361,45 +361,5 @@ namespace JiShe.CollectBus.Host }); }); } - - /// - /// 配置Kafka主题 - /// - /// - /// - public void ConfigureKafkaTopic(ServiceConfigurationContext context, IConfiguration configuration) - { - var adminClient = new AdminClientBuilder(new AdminClientConfig - { - BootstrapServers = configuration.GetConnectionString(CommonConst.Kafka) - }).Build(); - - try - { - - List topics = ProtocolConstExtensions.GetAllTopicNamesByIssued(); - topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived()); - - List topicSpecifications = new List(); - foreach (var item in topics) - { - topicSpecifications.Add(new TopicSpecification - { - Name = item, - NumPartitions = 3, - ReplicationFactor = 1 - }); - } - adminClient.CreateTopicsAsync(topicSpecifications).ConfigureAwait(false).GetAwaiter().GetResult(); - } - catch (CreateTopicsException e) - { - if (e.Results[0].Error.Code != ErrorCode.TopicAlreadyExists) - { - throw; - } - } - - } } } \ No newline at end of file diff --git a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs index 24f2029..fd11df4 100644 --- a/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs +++ b/src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs @@ -50,7 +50,7 @@ namespace JiShe.CollectBus.Kafka.Consumer AutoOffsetReset = AutoOffsetReset.Earliest, EnableAutoCommit = false, // 禁止AutoCommit EnablePartitionEof = true, // 启用分区末尾标记 - AllowAutoCreateTopics= true, // 启用自动创建 + //AllowAutoCreateTopics= true, // 启用自动创建 FetchMaxBytes = 1024 * 1024 * 50 // 增加拉取大小（50MB） }; diff --git a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs index e830dc4..c28c82c 100644 --- a/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs +++ b/src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs @@ -1,5 +1,6 @@ using Confluent.Kafka; using JiShe.CollectBus.Common.Consts; +using JiShe.CollectBus.Common.Extensions; using JiShe.CollectBus.Kafka.AdminClient; using JiShe.CollectBus.Kafka.Attributes; using JiShe.CollectBus.Kafka.Consumer; @@ -8,6 +9,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using System.Reflection; namespace JiShe.CollectBus.Kafka @@ -23,6 +25,18 @@ namespace JiShe.CollectBus.Kafka { var lifetime = provider.GetRequiredService(); + //初始化主题信息 + var kafkaAdminClient = provider.GetRequiredService(); + var kafkaOptions = provider.GetRequiredService>(); + + List topics = ProtocolConstExtensions.GetAllTopicNamesByIssued(); + topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived()); + + foreach (var item in topics) + { + kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor).ConfigureAwait(false).GetAwaiter().GetResult(); + } + lifetime.ApplicationStarted.Register(() => { var logger = provider.GetRequiredService>(); @@ -55,7 +69,7 @@ namespace JiShe.CollectBus.Kafka { if (subscribe!=null) { - Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger); + Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger, kafkaOptions.Value); threadCount += tuple.Item1; topicCount += tuple.Item2; } @@ -70,6 +84,17 @@ namespace JiShe.CollectBus.Kafka { var provider = app.ApplicationServices; var lifetime = provider.GetRequiredService(); + //初始化主题信息 + var kafkaAdminClient = provider.GetRequiredService(); + var kafkaOptions = provider.GetRequiredService>(); + + List topics = ProtocolConstExtensions.GetAllTopicNamesByIssued(); + topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived()); + + foreach (var item in topics) + { + kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor).ConfigureAwait(false).GetAwaiter().GetResult(); + } lifetime.ApplicationStarted.Register(() => { @@ -88,7 +113,7 @@ namespace JiShe.CollectBus.Kafka if (subscribe != null) { - Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger); + Tuple tuple = BuildKafkaSubscriber(subscribe, provider, logger, kafkaOptions.Value); threadCount += tuple.Item1; topicCount += tuple.Item2; } @@ -103,17 +128,17 @@ namespace JiShe.CollectBus.Kafka /// /// /// - private static Tuple BuildKafkaSubscriber(object subscribe, IServiceProvider provider,ILogger logger) + private static Tuple BuildKafkaSubscriber(object subscribe, IServiceProvider provider,ILogger logger, KafkaOptionConfig kafkaOptionConfig) { var subscribedMethods = subscribe.GetType().GetMethods() .Select(m => new { Method = m, Attribute = m.GetCustomAttribute() }) .Where(x => x.Attribute != null) .ToArray(); - var configuration = provider.GetRequiredService(); + //var configuration = provider.GetRequiredService(); int threadCount = 0; foreach (var sub in subscribedMethods) { - int partitionCount = configuration.GetValue(CommonConst.NumPartitions); + int partitionCount = kafkaOptionConfig.NumPartitions; //var adminClientService = provider.GetRequiredService(); //int partitionCount = sub.Attribute!.TaskCount==-1?adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic) : sub.Attribute!.TaskCount; if (partitionCount <= 0) @@ -154,7 +179,7 @@ namespace JiShe.CollectBus.Kafka logger.LogError($"kafka批量消费异常:{ex.Message}"); } return await Task.FromResult(false); - }); + }, attr.GroupId, attr.BatchSize,attr.BatchTimeout); } else { @@ -171,7 +196,7 @@ namespace JiShe.CollectBus.Kafka logger.LogError($"kafka消费异常:{ex.Message}"); } return await Task.FromResult(false); - }); + }, attr.GroupId); } } diff --git a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs index 52ef129..bc066fe 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs +++ b/src/JiShe.CollectBus.Protocol.Contracts/Abstracts/BaseProtocolPlugin.cs @@ -13,6 +13,7 @@ using JiShe.CollectBus.IotSystems.Protocols; using MassTransit; using DotNetCore.CAP; using JiShe.CollectBus.Kafka.Producer; +using JiShe.CollectBus.Common.Consts; namespace JiShe.CollectBus.Protocol.Contracts.Abstracts { diff --git a/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj b/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj index 3aa7a77..fc0f12e 100644 --- a/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj +++ b/src/JiShe.CollectBus.Protocol.Contracts/JiShe.CollectBus.Protocol.Contracts.csproj @@ -20,4 +20,8 @@ + + + + From f1082ce8a299a1b74b8c016941044410292b30fb Mon Sep 17 00:00:00 2001 From: cli <377476583@qq.com> Date: Thu, 17 Apr 2025 16:19:46 +0800 Subject: [PATCH 27/27] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=BA=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E8=AF=BB=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- JiShe.CollectBus.sln | 7 ------- .../BasicScheduledMeterReadingService.cs | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/JiShe.CollectBus.sln b/JiShe.CollectBus.sln index ea5e278..9c67aae 100644 --- a/JiShe.CollectBus.sln +++ b/JiShe.CollectBus.sln @@ -39,8 +39,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Cassandra", "src\JiShe.CollectBus.Cassandra\JiShe.CollectBus.Cassandra.csproj", "{443B4549-0AC0-4493-8F3E-49C83225DD76}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Kafka.Test", "src\JiShe.CollectBus.Kafka.Test\JiShe.CollectBus.Kafka.Test.csproj", "{FA762E8F-659A-DECF-83D6-5F364144450E}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -115,10 +113,6 @@ Global {443B4549-0AC0-4493-8F3E-49C83225DD76}.Debug|Any CPU.Build.0 = Debug|Any CPU {443B4549-0AC0-4493-8F3E-49C83225DD76}.Release|Any CPU.ActiveCfg = Release|Any CPU {443B4549-0AC0-4493-8F3E-49C83225DD76}.Release|Any CPU.Build.0 = Release|Any CPU - {FA762E8F-659A-DECF-83D6-5F364144450E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FA762E8F-659A-DECF-83D6-5F364144450E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FA762E8F-659A-DECF-83D6-5F364144450E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FA762E8F-659A-DECF-83D6-5F364144450E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -141,7 +135,6 @@ Global {A3F3C092-0A25-450B-BF6A-5983163CBEF5} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545} {A377955E-7EA1-6F29-8CF7-774569E93925} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545} {443B4549-0AC0-4493-8F3E-49C83225DD76} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545} - {FA762E8F-659A-DECF-83D6-5F364144450E} = {649A3FFA-182F-4E56-9717-E6A9A2BEC545} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4324B3B4-B60B-4E3C-91D8-59576B4E26DD} diff --git a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs index 9184a62..4cd968c 100644 --- a/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs +++ b/src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs @@ -304,8 +304,8 @@ namespace JiShe.CollectBus.ScheduledMeterReading timer1.Stop(); _logger.LogError($"读取数据更花费时间{timer1.ElapsedMilliseconds}毫秒"); - //DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista, _kafkaOptions.NumPartitions); - //return; + DeviceGroupBalanceControl.InitializeCache(focusAddressDataLista, _kafkaOptions.NumPartitions); + return; #else var meterInfos = await GetAmmeterInfoList(gatherCode); #endif