admin/JiShe.CollectBus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

优化任务创建，使用线程分组

Browse Source
This commit is contained in:
陈益 2025-04-14 21:56:24 +08:00
parent 8b86381e7a
commit 1284898376
2 changed files with 264 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
src/JiShe.CollectBus.Application/ScheduledMeterReading/BasicScheduledMeterReadingService.cs
View File

@ -3,6 +3,7 @@ using DotNetCore.CAP;
using JiShe.CollectBus.Ammeters; using JiShe.CollectBus.Ammeters;
using JiShe.CollectBus.Common.BuildSendDatas; using JiShe.CollectBus.Common.BuildSendDatas;
using JiShe.CollectBus.Common.Consts; using JiShe.CollectBus.Common.Consts;
using JiShe.CollectBus.Common.DeviceBalanceControl;
using JiShe.CollectBus.Common.Enums; using JiShe.CollectBus.Common.Enums;
using JiShe.CollectBus.Common.Extensions; using JiShe.CollectBus.Common.Extensions;
using JiShe.CollectBus.Common.Helpers; using JiShe.CollectBus.Common.Helpers;
@ -128,19 +129,23 @@ namespace JiShe.CollectBus.ScheduledMeterReading
if (meteryType == MeterTypeEnum.Ammeter.ToString()) if (meteryType == MeterTypeEnum.Ammeter.ToString())
{ {
// 解析结果（结果为嵌套数组） // 解析结果（结果为嵌套数组）
var meterInfos = await GetMeterRedisCacheDictionaryData<AmmeterInfo>(oneMinutekeyList, SystemType, ServerTagName, $"{timeDensity}", meterTypes[meteryType]); var meterInfos = await GetMeterRedisCacheListData<AmmeterInfo>(oneMinutekeyList, SystemType, ServerTagName, $"{timeDensity}", meterTypes[meteryType]);
if (meterInfos == null || meterInfos.Count <= 0) if (meterInfos == null || meterInfos.Count <= 0)
{ {
_logger.LogError($"{nameof(CreateToBeIssueTasks)} {timeDensity}分钟采集待下发任务创建失败，没有获取到缓存信息,-105"); _logger.LogError($"{nameof(CreateToBeIssueTasks)} {timeDensity}分钟采集待下发任务创建失败，没有获取到缓存信息,-105");
return; return;
} }
await AmmerterScheduledMeterReadingIssued(timeDensity, meterInfos); //await AmmerterScheduledMeterReadingIssued(timeDensity, meterInfos);
DeviceGroupBalanceControl.ProcessAllGroups(deviceId => { // 处理数据
Console.WriteLine($"Processing {deviceId} on Thread {Thread.CurrentThread.ManagedThreadId}"); await DeviceGroupBalanceControl.ProcessGenericListAsync(
}); items: meterInfos,
deviceIdSelector: data => data.FocusAddress,
await AmmerterCreatePublishTask(timeDensity, meterInfos); processor: (data, threadId) =>
{
_= AmmerterCreatePublishTask(timeDensity, data);
}
);
} }
else if (meteryType == MeterTypeEnum.WaterMeter.ToString()) else if (meteryType == MeterTypeEnum.WaterMeter.ToString())
{ {
@ -567,7 +572,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading
//根据分组创建线程批处理集中器 //根据分组创建线程批处理集中器
foreach (var group in focusHashGroups) foreach (var group in focusHashGroups)
{ {
await AmmerterCreatePublishTask(timeDensity, group.Value); await AmmerterCreatePublishTask2(timeDensity, group.Value);
} }
} }
catch (Exception) catch (Exception)
@ -582,10 +587,10 @@ namespace JiShe.CollectBus.ScheduledMeterReading
/// 电表创建发布任务 /// 电表创建发布任务
/// </summary> /// </summary>
/// <param name="timeDensity">采集频率</param> /// <param name="timeDensity">采集频率</param>
/// <param name="ammeterGroup">集中器号hash分组的集中器集合数据</param> /// <param name="ammeterInfo">集中器号hash分组的集中器集合数据</param>
/// <returns></returns> /// <returns></returns>
private async Task AmmerterCreatePublishTask(int timeDensity private async Task AmmerterCreatePublishTask(int timeDensity
, List<AmmeterInfo> ammeterGroup) , AmmeterInfo ammeterInfo)
{ {
var handlerPacketBuilder = TelemetryPacketBuilder.AFNHandlersDictionary; var handlerPacketBuilder = TelemetryPacketBuilder.AFNHandlersDictionary;
//todo 检查需要待补抄的电表的时间点信息，保存到需要待补抄的缓存中。如果此线程异常，该如何补偿？ //todo 检查需要待补抄的电表的时间点信息，保存到需要待补抄的缓存中。如果此线程异常，该如何补偿？
@ -593,28 +598,25 @@ namespace JiShe.CollectBus.ScheduledMeterReading
var currentTime = DateTime.Now; var currentTime = DateTime.Now;
var pendingCopyReadTime = currentTime.AddMinutes(timeDensity); var pendingCopyReadTime = currentTime.AddMinutes(timeDensity);
//构建缓存任务key，依然 表计类型+采集频率+集中器地址，存hash类型 //构建缓存任务key，依然 表计类型+采集频率+集中器地址，存hash类型
var redisCacheKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}{ammeterInfo.Key}"; var redisCacheKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}{ammeterInfo.FocusAddress}";
foreach (var ammeterInfo in ammeterGroup)
{
if (string.IsNullOrWhiteSpace(ammeterInfo.ItemCodes)) if (string.IsNullOrWhiteSpace(ammeterInfo.ItemCodes))
{ {
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}数据采集指令生成失败，采集项为空,-101"); _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}数据采集指令生成失败，采集项为空,-101");
continue; return;
} }
//载波的不处理 //载波的不处理
if (ammeterInfo.MeteringPort == (int)MeterLinkProtocolEnum.Carrierwave) if (ammeterInfo.MeteringPort == (int)MeterLinkProtocolEnum.Carrierwave)
{ {
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}数据采集指令生成失败，载波不处理,-102"); _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}数据采集指令生成失败，载波不处理,-102");
continue; return;
} }
if (ammeterInfo.State.Equals(2)) if (ammeterInfo.State.Equals(2))
{ {
_logger.LogWarning($"{nameof(AmmerterCreatePublishTask)} {ammeterInfo.Name} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}状态为禁用，不处理"); _logger.LogWarning($"{nameof(AmmerterCreatePublishTask)} {ammeterInfo.Name} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}状态为禁用，不处理");
continue; return;
} }
////排除1天未在线的集中器生成指令 或 排除集中器配置为自动上报的集中器 ////排除1天未在线的集中器生成指令 或 排除集中器配置为自动上报的集中器
@ -627,22 +629,22 @@ namespace JiShe.CollectBus.ScheduledMeterReading
if (string.IsNullOrWhiteSpace(ammeterInfo.AreaCode)) if (string.IsNullOrWhiteSpace(ammeterInfo.AreaCode))
{ {
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},集中器通信区号为空"); _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},集中器通信区号为空");
continue; return;
} }
if (string.IsNullOrWhiteSpace(ammeterInfo.Address)) if (string.IsNullOrWhiteSpace(ammeterInfo.Address))
{ {
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},集中器通信地址为空"); _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},集中器通信地址为空");
continue; return;
} }
if (Convert.ToInt32(ammeterInfo.Address) > 65535) if (Convert.ToInt32(ammeterInfo.Address) > 65535)
{ {
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},集中器通信地址无效,确保大于65535"); _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},集中器通信地址无效,确保大于65535");
continue; return;
} }
if (ammeterInfo.MeteringCode <= 0 || ammeterInfo.MeteringCode > 2033) if (ammeterInfo.MeteringCode <= 0 || ammeterInfo.MeteringCode > 33)
{ {
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},非有效测量点号({ammeterInfo.MeteringCode})"); _logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},非有效测量点号({ammeterInfo.MeteringCode})");
continue; return;
} }
List<string> tempCodes = ammeterInfo.ItemCodes.Deserialize<List<string>>()!; List<string> tempCodes = ammeterInfo.ItemCodes.Deserialize<List<string>>()!;
@ -669,7 +671,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading
if (tempSubCodes == null || tempSubCodes.Count <= 0) if (tempSubCodes == null || tempSubCodes.Count <= 0)
{ {
_logger.LogInformation($"{nameof(AmmerterCreatePublishTask)} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}自动上报数据主动采集1类数据时数据类型为空"); _logger.LogInformation($"{nameof(AmmerterCreatePublishTask)} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}自动上报数据主动采集1类数据时数据类型为空");
continue; return;
} }
else else
{ {
@ -758,7 +760,6 @@ namespace JiShe.CollectBus.ScheduledMeterReading
} }
await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs); await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs);
} }
}
/// <summary> /// <summary>
/// 电表创建发布任务 /// 电表创建发布任务
@ -1033,7 +1034,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading
} }
//获取下发任务缓存数据 //获取下发任务缓存数据
Dictionary<string, Dictionary<string, MeterReadingRecords>> meterTaskInfos = await GetMeterRedisCacheDictionaryData<MeterReadingRecords>(oneMinutekeyList,SystemType,ServerTagName ,timeDensity.ToString(), MeterTypeEnum.WaterMeter); Dictionary<string, Dictionary<string, MeterReadingRecords>> meterTaskInfos = await GetMeterRedisCacheDictionaryData<MeterReadingRecords>(oneMinutekeyList, SystemType, ServerTagName, timeDensity.ToString(), MeterTypeEnum.WaterMeter);
if (meterTaskInfos == null || meterTaskInfos.Count <= 0) if (meterTaskInfos == null || meterTaskInfos.Count <= 0)
{ {
_logger.LogError($"{nameof(WatermeterScheduledMeterAutoReading)} {timeDensity}分钟采集水表数据处理时没有获取到缓存信息,-102"); _logger.LogError($"{nameof(WatermeterScheduledMeterAutoReading)} {timeDensity}分钟采集水表数据处理时没有获取到缓存信息,-102");

112
src/JiShe.CollectBus.Common/DeviceBalanceControl/DeviceGroupBalanceControl.cs
View File

@ -108,48 +108,104 @@ namespace JiShe.CollectBus.Common.DeviceBalanceControl
} }
/// <summary> /// <summary>
/// 并行处理所有分组设备（每个分组一个处理线程） /// 并行处理泛型数据集（支持动态线程分配）
/// </summary> /// </summary>
public static void ProcessAllGroups<T>(Action<List<T>> processAction) where T : DeviceGroupBasicModel /// <typeparam name="T">已经分组的设备信息</typeparam>
/// <param name="items">部分或者全部的已经分组的设备集合</param>
/// <param name="deviceIdSelector">从泛型对象提取deviceId</param>
/// <param name="processor">处理委托（参数：当前对象，线程ID）</param>
/// <param name="maxThreads">可选线程限制</param>
/// <returns></returns>
/// <exception cref="InvalidOperationException"></exception>
public static async Task ProcessGenericListAsync<T>(
List<T> items, Func<T, string> deviceIdSelector, Action<T, int> processor, int? maxThreads = null)
{ {
var cache = _currentCache; var cache = _currentCache ?? throw new InvalidOperationException("缓存未初始化");
if (cache == null)
throw new InvalidOperationException("缓存未初始化");
// 使用并行选项控制并发度 // 创建分组任务队列
var groupQueues = new ConcurrentQueue<T>[cache.CachedGroups.Length];
for (int i = 0; i < groupQueues.Length; i++)
{
groupQueues[i] = new ConcurrentQueue<T>();
}
// 阶段1：分发数据到分组队列
Parallel.ForEach(items, item =>
{
var deviceId = deviceIdSelector(item);
if (cache.BalancedMapping.TryGetValue(deviceId, out int groupId))
{
groupQueues[groupId].Enqueue(item);
}
});
if ((maxThreads.HasValue && maxThreads.Value > cache.CachedGroups.Length) || maxThreads.HasValue == false)
{
maxThreads = cache.CachedGroups.Length;
}
// 阶段2：并行处理队列
var options = new ParallelOptions var options = new ParallelOptions
{ {
MaxDegreeOfParallelism = cache.CachedGroups.Length // 严格匹配分组数量 MaxDegreeOfParallelism = maxThreads.Value,
}; };
await Task.Run(() =>
{
Parallel.For(0, cache.CachedGroups.Length, options, groupId => Parallel.For(0, cache.CachedGroups.Length, options, groupId =>
{ {
// 获取当前分组的只读副本 var queue = groupQueues[groupId];
var groupDevices = GetGroupSnapshot(cache, groupId); while (queue.TryDequeue(out T item))
{
processAction(groupDevices); processor(item, Thread.CurrentThread.ManagedThreadId);
}
//foreach (var deviceId in groupDevices) });
//{
// //执行处理操作
// processAction(deviceId);
// // 可添加取消检测
// // if (token.IsCancellationRequested) break;
//}
}); });
} }
/// <summary> /// <summary>
/// 获取分组数据快照（线程安全） /// 并行处理所有分组设备（每个分组一个处理线程）
/// </summary> /// </summary>
private static IReadOnlyList<string> GetGroupSnapshot(CacheState cache, int groupId) //public static void ProcessAllGroups<T>(Action<List<T>> processAction) where T : DeviceGroupBasicModel
{ //{
lock (cache.CachedGroups[groupId]) // var cache = _currentCache;
{ // if (cache == null)
return cache.CachedGroups[groupId].ToList(); // 创建内存快照 // throw new InvalidOperationException("缓存未初始化");
}
} // // 使用并行选项控制并发度
// var options = new ParallelOptions
// {
// MaxDegreeOfParallelism = cache.CachedGroups.Length // 严格匹配分组数量
// };
// Parallel.For(0, cache.CachedGroups.Length, options, groupId =>
// {
// // 获取当前分组的只读副本
// var groupDevices = GetGroupSnapshot(cache, groupId);
// processAction(groupDevices);
// //foreach (var deviceId in groupDevices)
// //{
// // //执行处理操作
// // processAction(deviceId);
// // // 可添加取消检测
// // // if (token.IsCancellationRequested) break;
// //}
// });
//}
///// <summary>
///// 获取分组数据快照（线程安全）
///// </summary>
//public static IReadOnlyList<string> GetGroupSnapshot(CacheState cache, int groupId)
//{
// lock (cache.CachedGroups[groupId])
// {
// return cache.CachedGroups[groupId].ToList(); // 创建内存快照
// }
//}
/// <summary> /// <summary>
/// 通过 deviceId 获取所在的分组集合 /// 通过 deviceId 获取所在的分组集合