优化任务创建，使用线程分组
This commit is contained in:
陈益
parent
8b86381e7a
commit
1284898376
@ -3,6 +3,7 @@ using DotNetCore.CAP;
|
||||
using JiShe.CollectBus.Ammeters;
|
||||
using JiShe.CollectBus.Common.BuildSendDatas;
|
||||
using JiShe.CollectBus.Common.Consts;
|
||||
using JiShe.CollectBus.Common.DeviceBalanceControl;
|
||||
using JiShe.CollectBus.Common.Enums;
|
||||
using JiShe.CollectBus.Common.Extensions;
|
||||
using JiShe.CollectBus.Common.Helpers;
|
||||
@ -128,19 +129,23 @@ namespace JiShe.CollectBus.ScheduledMeterReading
|
||||
if (meteryType == MeterTypeEnum.Ammeter.ToString())
|
||||
{
|
||||
// 解析结果（结果为嵌套数组）
|
||||
var meterInfos = await GetMeterRedisCacheDictionaryData<AmmeterInfo>(oneMinutekeyList, SystemType, ServerTagName, $"{timeDensity}", meterTypes[meteryType]);
|
||||
var meterInfos = await GetMeterRedisCacheListData<AmmeterInfo>(oneMinutekeyList, SystemType, ServerTagName, $"{timeDensity}", meterTypes[meteryType]);
|
||||
if (meterInfos == null || meterInfos.Count <= 0)
|
||||
{
|
||||
_logger.LogError($"{nameof(CreateToBeIssueTasks)} {timeDensity}分钟采集待下发任务创建失败，没有获取到缓存信息,-105");
|
||||
return;
|
||||
}
|
||||
await AmmerterScheduledMeterReadingIssued(timeDensity, meterInfos);
|
||||
//await AmmerterScheduledMeterReadingIssued(timeDensity, meterInfos);
|
||||
|
||||
DeviceGroupBalanceControl.ProcessAllGroups(deviceId => {
|
||||
Console.WriteLine($"Processing {deviceId} on Thread {Thread.CurrentThread.ManagedThreadId}");
|
||||
});
|
||||
|
||||
await AmmerterCreatePublishTask(timeDensity, meterInfos);
|
||||
// 处理数据
|
||||
await DeviceGroupBalanceControl.ProcessGenericListAsync(
|
||||
items: meterInfos,
|
||||
deviceIdSelector: data => data.FocusAddress,
|
||||
processor: (data, threadId) =>
|
||||
{
|
||||
_= AmmerterCreatePublishTask(timeDensity, data);
|
||||
}
|
||||
);
|
||||
}
|
||||
else if (meteryType == MeterTypeEnum.WaterMeter.ToString())
|
||||
{
|
||||
@ -567,7 +572,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading
|
||||
//根据分组创建线程批处理集中器
|
||||
foreach (var group in focusHashGroups)
|
||||
{
|
||||
await AmmerterCreatePublishTask(timeDensity, group.Value);
|
||||
await AmmerterCreatePublishTask2(timeDensity, group.Value);
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
@ -582,10 +587,10 @@ namespace JiShe.CollectBus.ScheduledMeterReading
|
||||
/// 电表创建发布任务
|
||||
/// </summary>
|
||||
/// <param name="timeDensity">采集频率</param>
|
||||
/// <param name="ammeterGroup">集中器号hash分组的集中器集合数据</param>
|
||||
/// <param name="ammeterInfo">集中器号hash分组的集中器集合数据</param>
|
||||
/// <returns></returns>
|
||||
private async Task AmmerterCreatePublishTask(int timeDensity
|
||||
, List<AmmeterInfo> ammeterGroup)
|
||||
, AmmeterInfo ammeterInfo)
|
||||
{
|
||||
var handlerPacketBuilder = TelemetryPacketBuilder.AFNHandlersDictionary;
|
||||
//todo 检查需要待补抄的电表的时间点信息，保存到需要待补抄的缓存中。如果此线程异常，该如何补偿？
|
||||
@ -593,28 +598,25 @@ namespace JiShe.CollectBus.ScheduledMeterReading
|
||||
var currentTime = DateTime.Now;
|
||||
var pendingCopyReadTime = currentTime.AddMinutes(timeDensity);
|
||||
//构建缓存任务key，依然 表计类型+采集频率+集中器地址，存hash类型
|
||||
var redisCacheKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}{ammeterInfo.Key}";
|
||||
|
||||
foreach (var ammeterInfo in ammeterGroup)
|
||||
{
|
||||
var redisCacheKey = $"{string.Format(RedisConst.CacheTelemetryPacketInfoKey, SystemType, ServerTagName, MeterTypeEnum.Ammeter, timeDensity)}{ammeterInfo.FocusAddress}";
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ammeterInfo.ItemCodes))
|
||||
{
|
||||
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}数据采集指令生成失败，采集项为空,-101");
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
|
||||
//载波的不处理
|
||||
if (ammeterInfo.MeteringPort == (int)MeterLinkProtocolEnum.Carrierwave)
|
||||
{
|
||||
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}数据采集指令生成失败，载波不处理,-102");
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
|
||||
if (ammeterInfo.State.Equals(2))
|
||||
{
|
||||
_logger.LogWarning($"{nameof(AmmerterCreatePublishTask)} {ammeterInfo.Name} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}状态为禁用，不处理");
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
|
||||
////排除1天未在线的集中器生成指令 或 排除集中器配置为自动上报的集中器
|
||||
@ -627,22 +629,22 @@ namespace JiShe.CollectBus.ScheduledMeterReading
|
||||
if (string.IsNullOrWhiteSpace(ammeterInfo.AreaCode))
|
||||
{
|
||||
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},集中器通信区号为空");
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(ammeterInfo.Address))
|
||||
{
|
||||
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},集中器通信地址为空");
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
if (Convert.ToInt32(ammeterInfo.Address) > 65535)
|
||||
{
|
||||
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},集中器通信地址无效,确保大于65535");
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
if (ammeterInfo.MeteringCode <= 0 || ammeterInfo.MeteringCode > 2033)
|
||||
if (ammeterInfo.MeteringCode <= 0 || ammeterInfo.MeteringCode > 33)
|
||||
{
|
||||
_logger.LogError($"{nameof(AmmerterCreatePublishTask)} 表ID:{ammeterInfo.ID},非有效测量点号({ammeterInfo.MeteringCode})");
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
|
||||
List<string> tempCodes = ammeterInfo.ItemCodes.Deserialize<List<string>>()!;
|
||||
@ -669,7 +671,7 @@ namespace JiShe.CollectBus.ScheduledMeterReading
|
||||
if (tempSubCodes == null || tempSubCodes.Count <= 0)
|
||||
{
|
||||
_logger.LogInformation($"{nameof(AmmerterCreatePublishTask)} 集中器{ammeterInfo.FocusAddress}的电表{ammeterInfo.Name}自动上报数据主动采集1类数据时数据类型为空");
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -758,7 +760,6 @@ namespace JiShe.CollectBus.ScheduledMeterReading
|
||||
}
|
||||
await FreeRedisProvider.Instance.HSetAsync(redisCacheKey, keyValuePairs);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 电表创建发布任务
|
||||
|
||||
@ -108,48 +108,104 @@ namespace JiShe.CollectBus.Common.DeviceBalanceControl
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 并行处理所有分组设备（每个分组一个处理线程）
|
||||
/// 并行处理泛型数据集（支持动态线程分配）
|
||||
/// </summary>
|
||||
public static void ProcessAllGroups<T>(Action<List<T>> processAction) where T : DeviceGroupBasicModel
|
||||
/// <typeparam name="T">已经分组的设备信息</typeparam>
|
||||
/// <param name="items">部分或者全部的已经分组的设备集合</param>
|
||||
/// <param name="deviceIdSelector">从泛型对象提取deviceId</param>
|
||||
/// <param name="processor">处理委托（参数：当前对象，线程ID）</param>
|
||||
/// <param name="maxThreads">可选线程限制</param>
|
||||
/// <returns></returns>
|
||||
/// <exception cref="InvalidOperationException"></exception>
|
||||
public static async Task ProcessGenericListAsync<T>(
|
||||
List<T> items, Func<T, string> deviceIdSelector, Action<T, int> processor, int? maxThreads = null)
|
||||
{
|
||||
var cache = _currentCache;
|
||||
if (cache == null)
|
||||
throw new InvalidOperationException("缓存未初始化");
|
||||
var cache = _currentCache ?? throw new InvalidOperationException("缓存未初始化");
|
||||
|
||||
// 使用并行选项控制并发度
|
||||
// 创建分组任务队列
|
||||
var groupQueues = new ConcurrentQueue<T>[cache.CachedGroups.Length];
|
||||
for (int i = 0; i < groupQueues.Length; i++)
|
||||
{
|
||||
groupQueues[i] = new ConcurrentQueue<T>();
|
||||
}
|
||||
|
||||
// 阶段1：分发数据到分组队列
|
||||
Parallel.ForEach(items, item =>
|
||||
{
|
||||
var deviceId = deviceIdSelector(item);
|
||||
if (cache.BalancedMapping.TryGetValue(deviceId, out int groupId))
|
||||
{
|
||||
groupQueues[groupId].Enqueue(item);
|
||||
}
|
||||
});
|
||||
|
||||
if ((maxThreads.HasValue && maxThreads.Value > cache.CachedGroups.Length) || maxThreads.HasValue == false)
|
||||
{
|
||||
maxThreads = cache.CachedGroups.Length;
|
||||
}
|
||||
|
||||
// 阶段2：并行处理队列
|
||||
var options = new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = cache.CachedGroups.Length // 严格匹配分组数量
|
||||
MaxDegreeOfParallelism = maxThreads.Value,
|
||||
};
|
||||
|
||||
await Task.Run(() =>
|
||||
{
|
||||
Parallel.For(0, cache.CachedGroups.Length, options, groupId =>
|
||||
{
|
||||
// 获取当前分组的只读副本
|
||||
var groupDevices = GetGroupSnapshot(cache, groupId);
|
||||
|
||||
processAction(groupDevices);
|
||||
|
||||
//foreach (var deviceId in groupDevices)
|
||||
//{
|
||||
// //执行处理操作
|
||||
// processAction(deviceId);
|
||||
|
||||
// // 可添加取消检测
|
||||
// // if (token.IsCancellationRequested) break;
|
||||
//}
|
||||
var queue = groupQueues[groupId];
|
||||
while (queue.TryDequeue(out T item))
|
||||
{
|
||||
processor(item, Thread.CurrentThread.ManagedThreadId);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 获取分组数据快照（线程安全）
|
||||
/// 并行处理所有分组设备（每个分组一个处理线程）
|
||||
/// </summary>
|
||||
private static IReadOnlyList<string> GetGroupSnapshot(CacheState cache, int groupId)
|
||||
{
|
||||
lock (cache.CachedGroups[groupId])
|
||||
{
|
||||
return cache.CachedGroups[groupId].ToList(); // 创建内存快照
|
||||
}
|
||||
}
|
||||
//public static void ProcessAllGroups<T>(Action<List<T>> processAction) where T : DeviceGroupBasicModel
|
||||
//{
|
||||
// var cache = _currentCache;
|
||||
// if (cache == null)
|
||||
// throw new InvalidOperationException("缓存未初始化");
|
||||
|
||||
// // 使用并行选项控制并发度
|
||||
// var options = new ParallelOptions
|
||||
// {
|
||||
// MaxDegreeOfParallelism = cache.CachedGroups.Length // 严格匹配分组数量
|
||||
// };
|
||||
|
||||
// Parallel.For(0, cache.CachedGroups.Length, options, groupId =>
|
||||
// {
|
||||
// // 获取当前分组的只读副本
|
||||
// var groupDevices = GetGroupSnapshot(cache, groupId);
|
||||
|
||||
// processAction(groupDevices);
|
||||
|
||||
// //foreach (var deviceId in groupDevices)
|
||||
// //{
|
||||
// // //执行处理操作
|
||||
// // processAction(deviceId);
|
||||
|
||||
// // // 可添加取消检测
|
||||
// // // if (token.IsCancellationRequested) break;
|
||||
// //}
|
||||
// });
|
||||
//}
|
||||
|
||||
///// <summary>
|
||||
///// 获取分组数据快照（线程安全）
|
||||
///// </summary>
|
||||
//public static IReadOnlyList<string> GetGroupSnapshot(CacheState cache, int groupId)
|
||||
//{
|
||||
// lock (cache.CachedGroups[groupId])
|
||||
// {
|
||||
// return cache.CachedGroups[groupId].ToList(); // 创建内存快照
|
||||
// }
|
||||
//}
|
||||
|
||||
/// <summary>
|
||||
/// 通过 deviceId 获取所在的分组集合
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user