using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Volo.Abp.DependencyInjection;
namespace JiShe.CollectBus.Common.DeviceBalanceControl
{
///
/// 设备组负载控制
///
public class DeviceGroupBalanceControl
{
private static readonly object _syncRoot = new object();
private static volatile CacheState _currentCache;
///
/// 使用ConcurrentDictionary保证线程安全的设备分组映射
///
private sealed class CacheState
{
public readonly ConcurrentDictionary BalancedMapping;
public readonly List[] CachedGroups;
public CacheState(int groupCount)
{
BalancedMapping = new ConcurrentDictionary();
CachedGroups = new List[groupCount];
for (int i = 0; i < groupCount; i++)
{
CachedGroups[i] = new List();
}
}
}
///
/// 初始化或增量更新缓存
///
public static void InitializeCache(List deviceList, int groupCount = 30)
{
if (deviceList == null || deviceList.Count <= 0)
{
throw new ArgumentException($"{nameof(InitializeCache)} 设备分组初始化失败,设备数据为空");
}
if (groupCount > 60 || groupCount <= 0)
{
groupCount = 60;
}
lock (_syncRoot)
{
// 首次初始化
if (_currentCache == null)
{
var newCache = new CacheState(groupCount);
UpdateCacheWithDevices(newCache, deviceList, groupCount);
_currentCache = newCache;
}
// 后续增量更新
else
{
if (_currentCache.CachedGroups.Length != groupCount)
{
throw new ArgumentException($"{nameof(InitializeCache)} 设备分组初始化完成以后,分组数量不能更改");
}
var clonedCache = CloneExistingCache();
UpdateCacheWithDevices(clonedCache, deviceList, groupCount);
_currentCache = clonedCache;
}
}
}
///
/// 带锁的缓存克隆(写入时复制)
///
private static CacheState CloneExistingCache()
{
var oldCache = _currentCache;
var newCache = new CacheState(oldCache.CachedGroups.Length);
// 复制已有映射
foreach (var kvp in oldCache.BalancedMapping)
{
newCache.BalancedMapping.TryAdd(kvp.Key, kvp.Value);
}
// 复制分组数据
for (int i = 0; i < oldCache.CachedGroups.Length; i++)
{
newCache.CachedGroups[i].AddRange(oldCache.CachedGroups[i]);
}
return newCache;
}
///
/// 更新设备到缓存
///
private static void UpdateCacheWithDevices(CacheState cache, List deviceList, int groupCount)
{
foreach (var deviceId in deviceList)
{
// 原子操作:如果设备不存在则计算分组
cache.BalancedMapping.GetOrAdd(deviceId, id =>
{
int groupId = GetGroupId(id, groupCount);
lock (cache.CachedGroups[groupId])
{
cache.CachedGroups[groupId].Add(id);
}
return groupId;
});
}
}
///
/// 并行处理泛型数据集(支持动态线程分配)
///
/// 已经分组的设备信息
/// 部分或者全部的已经分组的设备集合
/// 从泛型对象提取deviceId
/// 处理委托(参数:当前对象,线程ID)
/// 可选线程限制
///
///
public static async Task ProcessGenericListAsync(
List items, Func deviceIdSelector, Action processor, int? maxThreads = null)
{
var cache = _currentCache ?? throw new InvalidOperationException("缓存未初始化");
// 创建分组任务队列
var groupQueues = new ConcurrentQueue[cache.CachedGroups.Length];
for (int i = 0; i < groupQueues.Length; i++)
{
groupQueues[i] = new ConcurrentQueue();
}
// 阶段1:分发数据到分组队列
Parallel.ForEach(items, item =>
{
var deviceId = deviceIdSelector(item);
if (cache.BalancedMapping.TryGetValue(deviceId, out int groupId))
{
groupQueues[groupId].Enqueue(item);
}
});
if ((maxThreads.HasValue && maxThreads.Value > cache.CachedGroups.Length) || maxThreads.HasValue == false)
{
maxThreads = cache.CachedGroups.Length;
}
// 阶段2:并行处理队列
var options = new ParallelOptions
{
MaxDegreeOfParallelism = maxThreads.Value,
};
await Task.Run(() =>
{
Parallel.For(0, cache.CachedGroups.Length, options, async groupId =>
{
var queue = groupQueues[groupId];
while (queue.TryDequeue(out T item))
{
processor(item, groupId);
}
});
});
}
///
/// 智能节流处理(CPU友好型)
///
/// 已经分组的设备信息
/// 部分或者全部的已经分组的设备集合
/// 从泛型对象提取deviceId
/// 处理委托(参数:当前对象,分组ID)
/// 可选最佳并发度
///
///
public static async Task ProcessWithThrottleAsync(
List items,
Func deviceIdSelector,
Action processor,
int? maxConcurrency = null)
{
var cache = _currentCache ?? throw new InvalidOperationException("缓存未初始化");
// 自动计算最佳并发度
int recommendedThreads = CalculateOptimalThreadCount();
if ((maxConcurrency.HasValue && maxConcurrency.Value > cache.CachedGroups.Length) || maxConcurrency.HasValue == false)
{
maxConcurrency = cache.CachedGroups.Length;
}
int actualThreads = maxConcurrency ?? recommendedThreads;
// 创建节流器
using var throttler = new SemaphoreSlim(initialCount: actualThreads);
// 使用LongRunning避免线程池饥饿
var tasks = items.Select(async item =>
{
await throttler.WaitAsync();
try
{
var deviceId = deviceIdSelector(item);
if (cache.BalancedMapping.TryGetValue(deviceId, out int groupId))
{
// 分组级处理(保持顺序性)
await ProcessItemAsync(item, processor, groupId);
}
}
finally
{
throttler.Release();
}
});
await Task.WhenAll(tasks);
}
///
/// 自动计算最优线程数
///
public static int CalculateOptimalThreadCount()
{
int coreCount = Environment.ProcessorCount;
return Math.Min(
coreCount * 2, // 超线程优化
_currentCache?.CachedGroups.Length ?? 60
);
}
///
/// 分组异步处理(带节流)
///
private static async Task ProcessItemAsync(T item, Action processor, int groupId)
{
// 使用内存缓存降低CPU负载
await Task.Yield(); // 立即释放当前线程
// 分组处理上下文
var context = ExecutionContext.Capture();
ThreadPool.QueueUserWorkItem(_ =>
{
ExecutionContext.Run(context!, state =>
{
processor(item,groupId);
}, null);
});
}
///
/// 通过 deviceId 获取所在的分组集合
///
public static List GetGroup(string deviceId)
{
var cache = _currentCache;
if (cache == null)
throw new InvalidOperationException("缓存未初始化");
return cache.CachedGroups[cache.BalancedMapping[deviceId]];
}
///
/// 通过 deviceId 获取分组Id
///
public static int GetDeviceGroupId(string deviceId)
{
var cache = _currentCache;
if (cache == null)
throw new InvalidOperationException("缓存未初始化");
return cache.BalancedMapping[deviceId];
}
///
/// 创建均衡映射表
///
/// 数据集合
/// 分组数量
/// 允许的最大偏差百分比
///
public static Dictionary CreateBalancedMapping(List deviceList, int groupCount, int maxDeviation = 5)
{
var mapping = new Dictionary();
int targetPerGroup = deviceList.Count / groupCount;
int maxAllowed = (int)(targetPerGroup * (1 + maxDeviation / 100.0));
// 初始化分组计数器
int[] groupCounters = new int[groupCount];
foreach (var deviceId in deviceList)
{
int preferredGroup = GetGroupId(deviceId, groupCount);
// 如果首选分组未满,直接分配
if (groupCounters[preferredGroup] < maxAllowed)
{
mapping[deviceId] = preferredGroup;
groupCounters[preferredGroup]++;
}
else
{
// 寻找当前最空闲的分组
int fallbackGroup = Array.IndexOf(groupCounters, groupCounters.Min());
mapping[deviceId] = fallbackGroup;
groupCounters[fallbackGroup]++;
}
}
return mapping;
}
///
/// 分析分组分布
///
///
///
///
public static Dictionary AnalyzeDistribution(List deviceList, int groupCount)
{
Dictionary distribution = new Dictionary();
foreach (var deviceId in deviceList)
{
int groupId = GetGroupId(deviceId, groupCount);
distribution[groupId] = distribution.TryGetValue(groupId, out var count) ? count + 1 : 1;
}
return distribution;
}
///
/// 获取设备ID对应的分组ID
///
///
///
///
public static int GetGroupId(string deviceId, int groupCount)
{
int hash = Fnv1aHash(deviceId);
// 双重取模确保分布均匀
return (hash % groupCount + groupCount) % groupCount;
}
///
/// FNV-1a哈希算法
///
///
///
public static int Fnv1aHash(string input)
{
const uint fnvPrime = 16777619;
const uint fnvOffsetBasis = 2166136261;
uint hash = fnvOffsetBasis;
foreach (char c in input)
{
hash ^= (byte)c;
hash *= fnvPrime;
}
return (int)hash;
}
///
/// CRC16算法实现
///
///
///
public static ushort CRC16Hash(byte[] bytes)
{
ushort crc = 0xFFFF;
for (int i = 0; i < bytes.Length; i++)
{
crc ^= bytes[i];
for (int j = 0; j < 8; j++)
{
if ((crc & 0x0001) == 1)
{
crc = (ushort)((crc >> 1) ^ 0xA001);
}
else
{
crc >>= 1;
}
}
}
return crc;
}
///
/// 打印分组统计数据
///
public static void PrintDistributionStats()
{
var cache = _currentCache;
if (cache == null)
{
Console.WriteLine("缓存未初始化");
return;
}
var stats = cache.CachedGroups
.Select((group, idx) => new { GroupId = idx, Count = group.Count })
.OrderBy(x => x.GroupId);
Console.WriteLine("分组数据量统计:");
foreach (var stat in stats)
{
Console.WriteLine($"Group {stat.GroupId}: {stat.Count} 条数据");
}
Console.WriteLine($"总共: {stats.Sum(d=>d.Count)} 条数据");
}
}
}