修改代码
This commit is contained in:
parent
b5f1f1f50b
commit
57123d653c
@ -21,7 +21,6 @@ using System.Collections.Generic;
|
|||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using static FreeSql.Internal.GlobalFilter;
|
|
||||||
|
|
||||||
namespace JiShe.CollectBus.ScheduledMeterReading
|
namespace JiShe.CollectBus.ScheduledMeterReading
|
||||||
{
|
{
|
||||||
|
|||||||
@ -17,6 +17,11 @@ namespace JiShe.CollectBus.Common.Models
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public List<T> Items { get; set; }
|
public List<T> Items { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 总条数
|
||||||
|
/// </summary>
|
||||||
|
public long TotalCount { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// 是否有下一页
|
/// 是否有下一页
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@ -25,7 +30,7 @@ namespace JiShe.CollectBus.Common.Models
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// 下一页的分页索引
|
/// 下一页的分页索引
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long? NextScore { get; set; }
|
public decimal? NextScore { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// 下一页的分页索引
|
/// 下一页的分页索引
|
||||||
|
|||||||
@ -31,5 +31,20 @@ namespace JiShe.CollectBus.Common.Models
|
|||||||
/// 数据集合
|
/// 数据集合
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IEnumerable<T> Items { get; set; }
|
public IEnumerable<T> Items { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 是否有下一页
|
||||||
|
/// </summary>
|
||||||
|
public bool HasNext { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 下一页的分页索引
|
||||||
|
/// </summary>
|
||||||
|
public decimal? NextScore { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 下一页的分页索引
|
||||||
|
/// </summary>
|
||||||
|
public string NextMember { get; set; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -22,8 +22,8 @@ namespace JiShe.CollectBus.Common.Models
|
|||||||
public int MeterId { get; set; }
|
public int MeterId { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// 唯一标识
|
/// 唯一标识,是redis ZSet和Set memberid
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public virtual string UniqueId => $"{FocusId}:{MeterId}";
|
public virtual string MemberID => $"{FocusId}:{MeterId}";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,12 +2,13 @@
|
|||||||
using JiShe.CollectBus.Common.Helpers;
|
using JiShe.CollectBus.Common.Helpers;
|
||||||
using JiShe.CollectBus.Common.Models;
|
using JiShe.CollectBus.Common.Models;
|
||||||
using JiShe.CollectBus.Common.Extensions;
|
using JiShe.CollectBus.Common.Extensions;
|
||||||
using JiShe.CollectBus.FreeRedisProvider.Options;
|
using JiShe.CollectBus.FreeRedisProvider.Options;
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using Volo.Abp.DependencyInjection;
|
using Volo.Abp.DependencyInjection;
|
||||||
using static System.Runtime.InteropServices.JavaScript.JSType;
|
using static System.Runtime.InteropServices.JavaScript.JSType;
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
|
||||||
namespace JiShe.CollectBus.FreeRedisProvider
|
namespace JiShe.CollectBus.FreeRedisProvider
|
||||||
{
|
{
|
||||||
@ -70,12 +71,12 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
{
|
{
|
||||||
throw new ArgumentException($"{nameof(AddMeterCacheData)} 参数异常,-101");
|
throw new ArgumentException($"{nameof(AddMeterCacheData)} 参数异常,-101");
|
||||||
}
|
}
|
||||||
|
|
||||||
// 计算组合score(分类ID + 时间戳)
|
// 计算组合score(分类ID + 时间戳)
|
||||||
var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow;
|
var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow;
|
||||||
|
|
||||||
long scoreValue = ((long)data.FocusId << 32) | (uint)actualTimestamp.Ticks;
|
long scoreValue = ((long)data.FocusId << 32) | (uint)actualTimestamp.Ticks;
|
||||||
|
|
||||||
//全局索引写入
|
//全局索引写入
|
||||||
long globalScore = actualTimestamp.ToUnixTimeMilliseconds();
|
long globalScore = actualTimestamp.ToUnixTimeMilliseconds();
|
||||||
|
|
||||||
@ -83,16 +84,16 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
using (var trans = Instance.Multi())
|
using (var trans = Instance.Multi())
|
||||||
{
|
{
|
||||||
// 主数据存储Hash
|
// 主数据存储Hash
|
||||||
trans.HSet(redisCacheKey, data.UniqueId, data.Serialize());
|
trans.HSet(redisCacheKey, data.MemberID, data.Serialize());
|
||||||
|
|
||||||
// 分类索引
|
// 分类索引
|
||||||
trans.SAdd(redisCacheFocusIndexKey, data.UniqueId);
|
trans.SAdd(redisCacheFocusIndexKey, data.MemberID);
|
||||||
|
|
||||||
// 排序索引使用ZSET
|
// 排序索引使用ZSET
|
||||||
trans.ZAdd(redisCacheScoresIndexKey, scoreValue, data.UniqueId);
|
trans.ZAdd(redisCacheScoresIndexKey, scoreValue, data.MemberID);
|
||||||
|
|
||||||
//全局索引
|
//全局索引
|
||||||
trans.ZAdd(redisCacheGlobalIndexKey, globalScore, data.UniqueId);
|
trans.ZAdd(redisCacheGlobalIndexKey, globalScore, data.MemberID);
|
||||||
|
|
||||||
var results = trans.Exec();
|
var results = trans.Exec();
|
||||||
|
|
||||||
@ -122,6 +123,16 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
IEnumerable<T> items,
|
IEnumerable<T> items,
|
||||||
DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel
|
DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel
|
||||||
{
|
{
|
||||||
|
if (items == null
|
||||||
|
|| items.Count() <=0
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheFocusIndexKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheScoresIndexKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey))
|
||||||
|
{
|
||||||
|
throw new ArgumentException($"{nameof(BatchAddMeterData)} 参数异常,-101");
|
||||||
|
}
|
||||||
|
|
||||||
const int BATCH_SIZE = 1000; // 每批1000条
|
const int BATCH_SIZE = 1000; // 每批1000条
|
||||||
var semaphore = new SemaphoreSlim(Environment.ProcessorCount * 2);
|
var semaphore = new SemaphoreSlim(Environment.ProcessorCount * 2);
|
||||||
|
|
||||||
@ -144,16 +155,16 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
long globalScore = actualTimestamp.ToUnixTimeMilliseconds();
|
long globalScore = actualTimestamp.ToUnixTimeMilliseconds();
|
||||||
|
|
||||||
// 主数据存储Hash
|
// 主数据存储Hash
|
||||||
pipe.HSet(redisCacheKey, item.UniqueId, item.Serialize());
|
pipe.HSet(redisCacheKey, item.MemberID, item.Serialize());
|
||||||
|
|
||||||
// 分类索引
|
// 分类索引
|
||||||
pipe.SAdd(redisCacheFocusIndexKey, item.UniqueId);
|
pipe.SAdd(redisCacheFocusIndexKey, item.MemberID);
|
||||||
|
|
||||||
// 排序索引使用ZSET
|
// 排序索引使用ZSET
|
||||||
pipe.ZAdd(redisCacheScoresIndexKey, scoreValue, item.UniqueId);
|
pipe.ZAdd(redisCacheScoresIndexKey, scoreValue, item.MemberID);
|
||||||
|
|
||||||
//全局索引
|
//全局索引
|
||||||
pipe.ZAdd(redisCacheGlobalIndexKey, globalScore, item.UniqueId);
|
pipe.ZAdd(redisCacheGlobalIndexKey, globalScore, item.MemberID);
|
||||||
}
|
}
|
||||||
pipe.EndPipe();
|
pipe.EndPipe();
|
||||||
}
|
}
|
||||||
@ -164,29 +175,108 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
await Task.CompletedTask;
|
await Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task UpdateMeterData<T>(
|
/// <summary>
|
||||||
|
/// 删除指定redis缓存key的缓存数据
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="T"></typeparam>
|
||||||
|
/// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
|
||||||
|
/// <param name="redisCacheFocusIndexKey">集中器索引Set缓存Key</param>
|
||||||
|
/// <param name="redisCacheScoresIndexKey">集中器排序索引ZSET缓存Key</param>
|
||||||
|
/// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
|
||||||
|
/// <param name="data">表计信息</param>
|
||||||
|
/// <returns></returns>
|
||||||
|
public async Task RemoveMeterData<T>(
|
||||||
string redisCacheKey,
|
string redisCacheKey,
|
||||||
string oldCategoryIndexKey,
|
string redisCacheFocusIndexKey,
|
||||||
string newCategoryIndexKey,
|
string redisCacheScoresIndexKey,
|
||||||
string memberId, // 唯一标识(格式:"分类ID:GUID")
|
string redisCacheGlobalIndexKey,
|
||||||
T newData,
|
T data) where T : DeviceCacheBasicModel
|
||||||
int? newCategoryId = null,
|
|
||||||
DateTimeOffset? newTimestamp = null)
|
|
||||||
{
|
{
|
||||||
// 参数校验
|
|
||||||
if (string.IsNullOrWhiteSpace(memberId))
|
if (data == null
|
||||||
throw new ArgumentException("Invalid member ID");
|
|| string.IsNullOrWhiteSpace(redisCacheKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheFocusIndexKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheScoresIndexKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey))
|
||||||
|
{
|
||||||
|
throw new ArgumentException($"{nameof(RemoveMeterData)} 参数异常,-101");
|
||||||
|
}
|
||||||
|
|
||||||
|
const string luaScript = @"
|
||||||
|
local mainKey = KEYS[1]
|
||||||
|
local focusIndexKey = KEYS[2]
|
||||||
|
local scoresIndexKey = KEYS[3]
|
||||||
|
local globalIndexKey = KEYS[4]
|
||||||
|
local member = ARGV[1]
|
||||||
|
|
||||||
|
local deleted = 0
|
||||||
|
if redis.call('HDEL', mainKey, member) > 0 then
|
||||||
|
deleted = 1
|
||||||
|
end
|
||||||
|
|
||||||
|
redis.call('SREM', focusIndexKey, member)
|
||||||
|
redis.call('ZREM', scoresIndexKey, member)
|
||||||
|
redis.call('ZREM', globalIndexKey, member)
|
||||||
|
return deleted
|
||||||
|
";
|
||||||
|
|
||||||
|
var keys = new[]
|
||||||
|
{
|
||||||
|
redisCacheKey,
|
||||||
|
redisCacheFocusIndexKey,
|
||||||
|
redisCacheScoresIndexKey,
|
||||||
|
redisCacheGlobalIndexKey
|
||||||
|
};
|
||||||
|
|
||||||
|
var result = await Instance.EvalAsync(luaScript, keys, new[] { data.MemberID });
|
||||||
|
|
||||||
|
if ((int)result == 0)
|
||||||
|
throw new KeyNotFoundException("指定数据不存在");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 修改表计缓存信息
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="T"></typeparam>
|
||||||
|
/// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
|
||||||
|
/// <param name="oldRedisCacheFocusIndexKey">旧集中器索引Set缓存Key</param>
|
||||||
|
/// <param name="newRedisCacheFocusIndexKey">新集中器索引Set缓存Key</param>
|
||||||
|
/// <param name="redisCacheScoresIndexKey">集中器排序索引ZSET缓存Key</param>
|
||||||
|
/// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
|
||||||
|
/// <param name="newData">表计信息</param>
|
||||||
|
/// <param name="newTimestamp">可选时间戳</param>
|
||||||
|
/// <returns></returns>
|
||||||
|
public async Task UpdateMeterData<T>(
|
||||||
|
string redisCacheKey,
|
||||||
|
string oldRedisCacheFocusIndexKey,
|
||||||
|
string newRedisCacheFocusIndexKey,
|
||||||
|
string redisCacheScoresIndexKey,
|
||||||
|
string redisCacheGlobalIndexKey,
|
||||||
|
T newData,
|
||||||
|
DateTimeOffset? newTimestamp = null) where T : DeviceCacheBasicModel
|
||||||
|
{
|
||||||
|
if (newData == null
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(oldRedisCacheFocusIndexKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(newRedisCacheFocusIndexKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheScoresIndexKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey))
|
||||||
|
{
|
||||||
|
throw new ArgumentException($"{nameof(UpdateMeterData)} 参数异常,-101");
|
||||||
|
}
|
||||||
|
|
||||||
var luaScript = @"
|
var luaScript = @"
|
||||||
local mainKey = KEYS[1]
|
local mainKey = KEYS[1]
|
||||||
local scoreKey = KEYS[2]
|
local oldFocusIndexKey = KEYS[2]
|
||||||
local oldIndex = KEYS[3]
|
local newFocusIndexKey = KEYS[3]
|
||||||
local newIndex = KEYS[4]
|
local scoresIndexKey = KEYS[4]
|
||||||
|
local globalIndexKey = KEYS[5]
|
||||||
local member = ARGV[1]
|
local member = ARGV[1]
|
||||||
local newData = ARGV[2]
|
local newData = ARGV[2]
|
||||||
local newScore = ARGV[3]
|
local newScore = ARGV[3]
|
||||||
|
local newGlobalScore = ARGV[4]
|
||||||
|
|
||||||
-- 校验旧数据是否存在
|
-- 校验存在性
|
||||||
if redis.call('HEXISTS', mainKey, member) == 0 then
|
if redis.call('HEXISTS', mainKey, member) == 0 then
|
||||||
return 0
|
return 0
|
||||||
end
|
end
|
||||||
@ -194,70 +284,67 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
-- 更新主数据
|
-- 更新主数据
|
||||||
redis.call('HSET', mainKey, member, newData)
|
redis.call('HSET', mainKey, member, newData)
|
||||||
|
|
||||||
-- 处理分类变更
|
-- 处理变更
|
||||||
if newScore ~= '' then
|
if newScore ~= '' then
|
||||||
-- 删除旧索引
|
-- 删除旧索引
|
||||||
redis.call('SREM', oldIndex, member)
|
redis.call('SREM', oldFocusIndexKey, member)
|
||||||
-- 更新排序分数
|
redis.call('ZREM', scoresIndexKey, member)
|
||||||
redis.call('ZADD', scoreKey, newScore, member)
|
|
||||||
-- 添加新索引
|
-- 添加新索引
|
||||||
redis.call('SADD', newIndex, member)
|
redis.call('SADD', newFocusIndexKey, member)
|
||||||
|
redis.call('ZADD', scoresIndexKey, newScore, member)
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 更新全局索引
|
||||||
|
if newGlobalScore ~= '' then
|
||||||
|
-- 删除旧索引
|
||||||
|
redis.call('ZREM', globalIndexKey, member)
|
||||||
|
|
||||||
|
-- 添加新索引
|
||||||
|
redis.call('ZADD', globalIndexKey, newGlobalScore, member)
|
||||||
end
|
end
|
||||||
|
|
||||||
return 1
|
return 1
|
||||||
";
|
";
|
||||||
|
|
||||||
// 计算新score(当分类或时间变化时)
|
var actualTimestamp = newTimestamp ?? DateTimeOffset.UtcNow;
|
||||||
long? newScoreValue = null;
|
var newGlobalScore = actualTimestamp.ToUnixTimeMilliseconds();
|
||||||
if (newCategoryId.HasValue || newTimestamp.HasValue)
|
var newScoreValue = ((long)newData.FocusId << 32) | (uint)actualTimestamp.Ticks;
|
||||||
{
|
|
||||||
var parts = memberId.Split(':');
|
|
||||||
var oldCategoryId = int.Parse(parts[0]);
|
|
||||||
|
|
||||||
var actualCategoryId = newCategoryId ?? oldCategoryId;
|
|
||||||
var actualTimestamp = newTimestamp ?? DateTimeOffset.UtcNow;
|
|
||||||
|
|
||||||
newScoreValue = ((long)actualCategoryId << 32) | (uint)actualTimestamp.Ticks;
|
|
||||||
}
|
|
||||||
|
|
||||||
var result = await Instance.EvalAsync(luaScript,
|
var result = await Instance.EvalAsync(luaScript,
|
||||||
new[]
|
new[]
|
||||||
{
|
{
|
||||||
redisCacheKey,
|
redisCacheKey,
|
||||||
$"{redisCacheKey}_scores",
|
oldRedisCacheFocusIndexKey,
|
||||||
oldCategoryIndexKey,
|
newRedisCacheFocusIndexKey,
|
||||||
newCategoryIndexKey
|
redisCacheScoresIndexKey,
|
||||||
|
redisCacheGlobalIndexKey
|
||||||
},
|
},
|
||||||
new[]
|
new object[]
|
||||||
{
|
{
|
||||||
memberId,
|
newData.MemberID,
|
||||||
newData.Serialize(),
|
newData.Serialize(),
|
||||||
newScoreValue?.ToString() ?? ""
|
newScoreValue.ToString() ?? "",
|
||||||
|
newGlobalScore.ToString() ?? ""
|
||||||
});
|
});
|
||||||
|
|
||||||
// 如果时间戳变化则更新全局索引
|
|
||||||
if (newTimestamp.HasValue)
|
|
||||||
{
|
|
||||||
long newGlobalScore = newTimestamp.Value.ToUnixTimeMilliseconds();
|
|
||||||
await Instance.ZAddAsync("global_data_all", newGlobalScore, memberId);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((int)result == 0)
|
if ((int)result == 0)
|
||||||
throw new KeyNotFoundException("指定数据不存在");
|
{
|
||||||
|
throw new KeyNotFoundException($"{nameof(UpdateMeterData)}指定Key{redisCacheKey}的数据不存在");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task<BusPagedResult<T>> SingleGetMeterPagedData<T>(
|
||||||
public async Task<BusPagedResult<T>> GetMeterZSetPagedData<T>(
|
|
||||||
string redisCacheKey,
|
string redisCacheKey,
|
||||||
string redisCacheIndexKey,
|
string redisCacheScoresIndexKey,
|
||||||
int categoryId,
|
int focusId,
|
||||||
int pageSize = 10,
|
int pageSize = 10,
|
||||||
int pageIndex = 1,
|
int pageIndex = 1,
|
||||||
bool descending = true)
|
bool descending = true)
|
||||||
{
|
{
|
||||||
// 计算score范围
|
// 计算score范围
|
||||||
long minScore = (long)categoryId << 32;
|
long minScore = (long)focusId << 32;
|
||||||
long maxScore = ((long)categoryId + 1) << 32;
|
long maxScore = ((long)focusId + 1) << 32;
|
||||||
|
|
||||||
// 分页参数计算
|
// 分页参数计算
|
||||||
int start = (pageIndex - 1) * pageSize;
|
int start = (pageIndex - 1) * pageSize;
|
||||||
@ -265,13 +352,13 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
// 获取排序后的member列表
|
// 获取排序后的member列表
|
||||||
var members = descending
|
var members = descending
|
||||||
? await Instance.ZRevRangeByScoreAsync(
|
? await Instance.ZRevRangeByScoreAsync(
|
||||||
$"{redisCacheKey}_scores",
|
redisCacheScoresIndexKey,
|
||||||
maxScore,
|
maxScore,
|
||||||
minScore,
|
minScore,
|
||||||
start,
|
start,
|
||||||
pageSize)
|
pageSize)
|
||||||
: await Instance.ZRangeByScoreAsync(
|
: await Instance.ZRangeByScoreAsync(
|
||||||
$"{redisCacheKey}_scores",
|
redisCacheScoresIndexKey,
|
||||||
minScore,
|
minScore,
|
||||||
maxScore,
|
maxScore,
|
||||||
start,
|
start,
|
||||||
@ -284,7 +371,7 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
|
|
||||||
// 总数统计优化
|
// 总数统计优化
|
||||||
var total = await Instance.ZCountAsync(
|
var total = await Instance.ZCountAsync(
|
||||||
$"{redisCacheKey}_scores",
|
redisCacheScoresIndexKey,
|
||||||
minScore,
|
minScore,
|
||||||
maxScore);
|
maxScore);
|
||||||
|
|
||||||
@ -298,105 +385,290 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public async Task RemoveMeterZSetData<T>(
|
public async Task<BusPagedResult<T>> GetFocusPagedData<T>(
|
||||||
string redisCacheKey,
|
|
||||||
string redisCacheIndexKey,
|
|
||||||
string uniqueId) // 改为基于唯一标识删除
|
|
||||||
{
|
|
||||||
// 原子操作
|
|
||||||
var luaScript = @"
|
|
||||||
local mainKey = KEYS[1]
|
|
||||||
local scoreKey = KEYS[2]
|
|
||||||
local indexKey = KEYS[3]
|
|
||||||
local member = ARGV[1]
|
|
||||||
|
|
||||||
redis.call('HDEL', mainKey, member)
|
|
||||||
redis.call('ZREM', scoreKey, member)
|
|
||||||
redis.call('SREM', indexKey, member)
|
|
||||||
return 1
|
|
||||||
";
|
|
||||||
|
|
||||||
var keys = new[]
|
|
||||||
{
|
|
||||||
redisCacheKey,
|
|
||||||
$"{redisCacheKey}_scores",
|
|
||||||
redisCacheIndexKey
|
|
||||||
};
|
|
||||||
|
|
||||||
var result = await Instance.EvalAsync(luaScript,
|
|
||||||
keys,
|
|
||||||
new[] { uniqueId });
|
|
||||||
|
|
||||||
if ((int)result != 1)
|
|
||||||
throw new Exception("删除操作失败");
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<BusCacheGlobalPagedResult<T>> GetGlobalPagedData<T>(
|
|
||||||
string redisCacheKey,
|
string redisCacheKey,
|
||||||
|
string redisCacheScoresIndexKey,
|
||||||
|
int focusId,
|
||||||
int pageSize = 10,
|
int pageSize = 10,
|
||||||
long? lastScore = null,
|
long? lastScore = null,
|
||||||
string lastMember = null,
|
string lastMember = null,
|
||||||
bool descending = true)
|
bool descending = true) where T : DeviceCacheBasicModel
|
||||||
{
|
{
|
||||||
const string zsetKey = "global_data_all";
|
// 计算分数范围
|
||||||
|
long minScore = (long)focusId << 32;
|
||||||
// 分页参数处理
|
long maxScore = ((long)focusId + 1) << 32;
|
||||||
var (startScore, excludeMember) = descending
|
|
||||||
? (lastScore ?? long.MaxValue, lastMember)
|
|
||||||
: (lastScore ?? 0, lastMember);
|
|
||||||
|
|
||||||
// 获取成员列表
|
// 获取成员列表
|
||||||
string[] members;
|
var members = await GetSortedMembers(
|
||||||
if (descending)
|
redisCacheScoresIndexKey,
|
||||||
|
minScore,
|
||||||
|
maxScore,
|
||||||
|
pageSize,
|
||||||
|
lastScore,
|
||||||
|
lastMember,
|
||||||
|
descending);
|
||||||
|
|
||||||
|
// 批量获取数据
|
||||||
|
var dataDict = await Instance.HMGetAsync<T>(redisCacheKey, members.CurrentItems);
|
||||||
|
|
||||||
|
return new BusPagedResult<T>
|
||||||
{
|
{
|
||||||
members = await Instance.ZRevRangeByScoreAsync(
|
Items = dataDict,
|
||||||
|
TotalCount = await GetTotalCount(redisCacheScoresIndexKey, minScore, maxScore),
|
||||||
|
HasNext = members.HasNext,
|
||||||
|
NextScore = members.NextScore,
|
||||||
|
NextMember = members.NextMember
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<(string[] CurrentItems, bool HasNext, decimal? NextScore, string NextMember)>
|
||||||
|
GetSortedMembers(
|
||||||
|
string zsetKey,
|
||||||
|
long minScore,
|
||||||
|
long maxScore,
|
||||||
|
int pageSize,
|
||||||
|
long? lastScore,
|
||||||
|
string lastMember,
|
||||||
|
bool descending)
|
||||||
|
{
|
||||||
|
var querySize = pageSize + 1;
|
||||||
|
var (startScore, exclude) = descending
|
||||||
|
? (lastScore ?? maxScore, lastMember)
|
||||||
|
: (lastScore ?? minScore, lastMember);
|
||||||
|
|
||||||
|
var members = descending
|
||||||
|
? await Instance.ZRevRangeByScoreAsync(
|
||||||
zsetKey,
|
zsetKey,
|
||||||
max: startScore,
|
max: startScore,
|
||||||
min: 0,
|
min: minScore,
|
||||||
offset: 0,
|
offset: 0,
|
||||||
count: pageSize + 1);
|
count: querySize)
|
||||||
}
|
: await Instance.ZRangeByScoreAsync(
|
||||||
else
|
|
||||||
{
|
|
||||||
members = await Instance.ZRangeByScoreAsync(
|
|
||||||
zsetKey,
|
zsetKey,
|
||||||
min: startScore,
|
min: startScore,
|
||||||
max: long.MaxValue,
|
max: maxScore,
|
||||||
offset: 0,
|
offset: 0,
|
||||||
count: pageSize + 1);
|
count: querySize);
|
||||||
|
|
||||||
|
var hasNext = members.Length > pageSize;
|
||||||
|
var currentItems = members.Take(pageSize).ToArray();
|
||||||
|
|
||||||
|
var nextCursor = currentItems.Any()
|
||||||
|
? await GetNextCursor(zsetKey, currentItems.Last(), descending)
|
||||||
|
: (null, null);
|
||||||
|
|
||||||
|
return (currentItems, hasNext, nextCursor.score, nextCursor.member);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<long> GetTotalCount(string zsetKey, long min, long max)
|
||||||
|
{
|
||||||
|
// 缓存计数优化
|
||||||
|
var cacheKey = $"{zsetKey}_count_{min}_{max}";
|
||||||
|
var cached = await Instance.GetAsync<long?>(cacheKey);
|
||||||
|
|
||||||
|
if (cached.HasValue)
|
||||||
|
return cached.Value;
|
||||||
|
|
||||||
|
var count = await Instance.ZCountAsync(zsetKey, min, max);
|
||||||
|
await Instance.SetExAsync(cacheKey, 60, count); // 缓存60秒
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public async Task<Dictionary<int, BusPagedResult<T>>> BatchGetMeterPagedData<T>(
|
||||||
|
string redisCacheKey,
|
||||||
|
string redisCacheScoresIndexKey,
|
||||||
|
IEnumerable<int> focusIds,
|
||||||
|
int pageSizePerFocus = 10) where T : DeviceCacheBasicModel
|
||||||
|
{
|
||||||
|
var results = new ConcurrentDictionary<int, BusPagedResult<T>>();
|
||||||
|
var parallelOptions = new ParallelOptions
|
||||||
|
{
|
||||||
|
MaxDegreeOfParallelism = Environment.ProcessorCount * 2
|
||||||
|
};
|
||||||
|
|
||||||
|
await Parallel.ForEachAsync(focusIds, parallelOptions, async (focusId, _) =>
|
||||||
|
{
|
||||||
|
var data = await SingleGetMeterPagedData<T>(
|
||||||
|
redisCacheKey,
|
||||||
|
redisCacheScoresIndexKey,
|
||||||
|
focusId,
|
||||||
|
pageSizePerFocus);
|
||||||
|
|
||||||
|
results.TryAdd(focusId, data);
|
||||||
|
});
|
||||||
|
|
||||||
|
return new Dictionary<int, BusPagedResult<T>>(results);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 通过全局索引分页查询表计缓存数据
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="T"></typeparam>
|
||||||
|
/// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
|
||||||
|
/// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
|
||||||
|
/// <param name="pageSize">分页尺寸</param>
|
||||||
|
/// <param name="lastScore">最后一个索引</param>
|
||||||
|
/// <param name="lastMember">最后一个唯一标识</param>
|
||||||
|
/// <param name="descending">排序方式</param>
|
||||||
|
/// <returns></returns>
|
||||||
|
public async Task<BusCacheGlobalPagedResult<T>> GetGlobalPagedData<T>(
|
||||||
|
string redisCacheKey,
|
||||||
|
string redisCacheGlobalIndexKey,
|
||||||
|
int pageSize = 10,
|
||||||
|
decimal? lastScore = null,
|
||||||
|
string lastMember = null,
|
||||||
|
bool descending = true)
|
||||||
|
where T : DeviceCacheBasicModel
|
||||||
|
{
|
||||||
|
// 参数校验增强
|
||||||
|
if (string.IsNullOrWhiteSpace(redisCacheKey) || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey))
|
||||||
|
{
|
||||||
|
throw new ArgumentException($"{nameof(GetGlobalPagedData)} 参数异常,-101");
|
||||||
}
|
}
|
||||||
|
|
||||||
// 处理分页结果
|
if (pageSize < 1 || pageSize > 1000)
|
||||||
bool hasNext = members.Length > pageSize;
|
{
|
||||||
var actualMembers = members.Take(pageSize).ToArray();
|
throw new ArgumentException($"{nameof(GetGlobalPagedData)} 分页大小应在1-1000之间,-102");
|
||||||
|
}
|
||||||
|
|
||||||
// 批量获取数据(优化版本)
|
// 分页参数解析
|
||||||
var dataTasks = actualMembers
|
var (startScore, excludeMember) = descending
|
||||||
.Select(m => Instance.HGetAsync<T>(redisCacheKey, m))
|
? (lastScore ?? decimal.MaxValue, lastMember)
|
||||||
.ToArray();
|
: (lastScore ?? 0, lastMember);
|
||||||
await Task.WhenAll(dataTasks);
|
|
||||||
|
// 游标分页查询
|
||||||
|
var (members, hasNext) = await GetPagedMembers(
|
||||||
|
redisCacheGlobalIndexKey,
|
||||||
|
pageSize,
|
||||||
|
startScore,
|
||||||
|
excludeMember,
|
||||||
|
descending);
|
||||||
|
|
||||||
|
// 批量获取数据(优化内存分配)
|
||||||
|
var dataDict = await BatchGetData<T>(redisCacheKey, members);
|
||||||
|
|
||||||
// 获取下一页游标
|
// 获取下一页游标
|
||||||
(long? nextScore, string nextMember) = actualMembers.Any()
|
var nextCursor = members.Any()
|
||||||
? await GetNextCursor(zsetKey, actualMembers.Last(), descending)
|
? await GetNextCursor(redisCacheGlobalIndexKey, members.Last(), descending)
|
||||||
: (null, null);
|
: (null, null);
|
||||||
|
|
||||||
return new BusCacheGlobalPagedResult<T>
|
return new BusCacheGlobalPagedResult<T>
|
||||||
{
|
{
|
||||||
Items = dataTasks.Select(t => t.Result).ToList(),
|
Items = members.Select(m => dataDict.TryGetValue(m, out var v) ? v : default)
|
||||||
|
.Where(x => x != null).ToList(),
|
||||||
HasNext = hasNext,
|
HasNext = hasNext,
|
||||||
NextScore = nextScore,
|
NextScore = nextCursor.score,
|
||||||
NextMember = nextMember
|
NextMember = nextCursor.member
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<(long? score, string member)> GetNextCursor(
|
/// <summary>
|
||||||
string zsetKey,
|
/// 游标分页查询
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="redisCacheGlobalIndexKey"></param>
|
||||||
|
/// <param name="pageSize">分页数量</param>
|
||||||
|
/// <param name="startScore">开始索引</param>
|
||||||
|
/// <param name="excludeMember">开始唯一标识</param>
|
||||||
|
/// <param name="descending">排序方式</param>
|
||||||
|
/// <returns></returns>
|
||||||
|
private async Task<(List<string> Members, bool HasNext)> GetPagedMembers(
|
||||||
|
string redisCacheGlobalIndexKey,
|
||||||
|
int pageSize,
|
||||||
|
decimal? startScore,
|
||||||
|
string excludeMember,
|
||||||
|
bool descending)
|
||||||
|
{
|
||||||
|
const int bufferSize = 50; // 预读缓冲区大小
|
||||||
|
|
||||||
|
// 使用流式分页(避免OFFSET性能问题)
|
||||||
|
var members = new List<string>(pageSize + 1);
|
||||||
|
decimal? currentScore = startScore;
|
||||||
|
string lastMember = excludeMember;
|
||||||
|
|
||||||
|
while (members.Count < pageSize + 1 && currentScore.HasValue)
|
||||||
|
{
|
||||||
|
var querySize = Math.Min(bufferSize, pageSize + 1 - members.Count);
|
||||||
|
|
||||||
|
var batch = descending
|
||||||
|
? await Instance.ZRevRangeByScoreAsync(
|
||||||
|
redisCacheGlobalIndexKey,
|
||||||
|
max: currentScore.Value,
|
||||||
|
min: 0,
|
||||||
|
offset: 0,
|
||||||
|
count: querySize
|
||||||
|
)
|
||||||
|
: await Instance.ZRangeByScoreAsync(
|
||||||
|
redisCacheGlobalIndexKey,
|
||||||
|
min: currentScore.Value,
|
||||||
|
max: long.MaxValue,
|
||||||
|
offset: 0,
|
||||||
|
count: querySize);
|
||||||
|
|
||||||
|
if (!batch.Any()) break;
|
||||||
|
|
||||||
|
members.AddRange(batch);
|
||||||
|
lastMember = batch.LastOrDefault();
|
||||||
|
currentScore = await Instance.ZScoreAsync(redisCacheGlobalIndexKey, lastMember);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
members.Take(pageSize).ToList(),
|
||||||
|
members.Count > pageSize
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 批量获取指定分页的数据
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="T"></typeparam>
|
||||||
|
/// <param name="hashKey"></param>
|
||||||
|
/// <param name="members"></param>
|
||||||
|
/// <returns></returns>
|
||||||
|
private async Task<Dictionary<string, T>> BatchGetData<T>(
|
||||||
|
string hashKey,
|
||||||
|
IEnumerable<string> members)
|
||||||
|
where T : DeviceCacheBasicModel
|
||||||
|
{
|
||||||
|
const int batchSize = 100;
|
||||||
|
var result = new Dictionary<string, T>();
|
||||||
|
|
||||||
|
foreach (var batch in members.Batch(batchSize))
|
||||||
|
{
|
||||||
|
var batchArray = batch.ToArray();
|
||||||
|
var values = await Instance.HMGetAsync<T>(hashKey, batchArray);
|
||||||
|
|
||||||
|
for (int i = 0; i < batchArray.Length; i++)
|
||||||
|
{
|
||||||
|
if (EqualityComparer<T>.Default.Equals(values[i], default)) continue;
|
||||||
|
result[batchArray[i]] = values[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 获取下一页游标
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="redisCacheGlobalIndexKey">全局索引Key</param>
|
||||||
|
/// <param name="lastMember">最后一个唯一标识</param>
|
||||||
|
/// <param name="descending">排序方式</param>
|
||||||
|
/// <returns></returns>
|
||||||
|
private async Task<(decimal? score, string member)> GetNextCursor(
|
||||||
|
string redisCacheGlobalIndexKey,
|
||||||
string lastMember,
|
string lastMember,
|
||||||
bool descending)
|
bool descending)
|
||||||
{
|
{
|
||||||
var score = await Instance.ZScoreAsync(zsetKey, lastMember);
|
if (string.IsNullOrWhiteSpace(lastMember))
|
||||||
return (score.HasValue ? (long)score.Value : null, lastMember);
|
{
|
||||||
|
return (null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
var score = await Instance.ZScoreAsync(redisCacheGlobalIndexKey, lastMember);
|
||||||
|
return score.HasValue
|
||||||
|
? (Convert.ToInt64(score.Value), lastMember)
|
||||||
|
: (null, null);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -48,6 +48,66 @@ namespace JiShe.CollectBus.FreeRedisProvider
|
|||||||
string redisCacheGlobalIndexKey,
|
string redisCacheGlobalIndexKey,
|
||||||
IEnumerable<T> items,
|
IEnumerable<T> items,
|
||||||
DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel;
|
DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 删除指定redis缓存key的缓存数据
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="T"></typeparam>
|
||||||
|
/// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
|
||||||
|
/// <param name="redisCacheFocusIndexKey">集中器索引Set缓存Key</param>
|
||||||
|
/// <param name="redisCacheScoresIndexKey">集中器排序索引ZSET缓存Key</param>
|
||||||
|
/// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
|
||||||
|
/// <param name="data">表计信息</param>
|
||||||
|
/// <returns></returns>
|
||||||
|
Task RemoveMeterData<T>(
|
||||||
|
string redisCacheKey,
|
||||||
|
string redisCacheFocusIndexKey,
|
||||||
|
string redisCacheScoresIndexKey,
|
||||||
|
string redisCacheGlobalIndexKey,
|
||||||
|
T data) where T : DeviceCacheBasicModel;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 修改表计缓存信息
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="T"></typeparam>
|
||||||
|
/// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
|
||||||
|
/// <param name="oldRedisCacheFocusIndexKey">旧集中器索引Set缓存Key</param>
|
||||||
|
/// <param name="newRedisCacheFocusIndexKey">新集中器索引Set缓存Key</param>
|
||||||
|
/// <param name="redisCacheScoresIndexKey">集中器排序索引ZSET缓存Key</param>
|
||||||
|
/// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
|
||||||
|
/// <param name="newData">表计信息</param>
|
||||||
|
/// <param name="newTimestamp">可选时间戳</param>
|
||||||
|
/// <returns></returns>
|
||||||
|
Task UpdateMeterData<T>(
|
||||||
|
string redisCacheKey,
|
||||||
|
string oldRedisCacheFocusIndexKey,
|
||||||
|
string newRedisCacheFocusIndexKey,
|
||||||
|
string redisCacheScoresIndexKey,
|
||||||
|
string redisCacheGlobalIndexKey,
|
||||||
|
T newData,
|
||||||
|
DateTimeOffset? newTimestamp = null) where T : DeviceCacheBasicModel;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 通过全局索引分页查询表计缓存数据
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="T"></typeparam>
|
||||||
|
/// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
|
||||||
|
/// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
|
||||||
|
/// <param name="pageSize">分页尺寸</param>
|
||||||
|
/// <param name="lastScore">最后一个索引</param>
|
||||||
|
/// <param name="lastMember">最后一个唯一标识</param>
|
||||||
|
/// <param name="descending">排序方式</param>
|
||||||
|
/// <returns></returns>
|
||||||
|
Task<BusCacheGlobalPagedResult<T>> GetGlobalPagedData<T>(
|
||||||
|
string redisCacheKey,
|
||||||
|
string redisCacheGlobalIndexKey,
|
||||||
|
int pageSize = 10,
|
||||||
|
decimal? lastScore = null,
|
||||||
|
string lastMember = null,
|
||||||
|
bool descending = true)
|
||||||
|
where T : DeviceCacheBasicModel;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user