提交代码

src/JiShe.CollectBus.Application/CollectBusApplicationModule.cs

@@ -22,6 +22,7 @@ using Volo.Abp.BackgroundWorkers;
 using Volo.Abp.BackgroundWorkers.Hangfire;
 using Volo.Abp.EventBus;
 using Volo.Abp.Modularity;
+using Microsoft.Extensions.Options;
 
 namespace JiShe.CollectBus;
 
@@ -69,13 +70,14 @@ public class CollectBusApplicationModule : AbpModule
         //初始化主题信息
         var kafkaAdminClient = context.ServiceProvider.GetRequiredService<IAdminClientService>();
         var configuration = context.ServiceProvider.GetRequiredService<IConfiguration>();
+        var kafkaOptions = context.ServiceProvider.GetRequiredService<IOptions<KafkaOptionConfig>>();
 
         List<string> topics = ProtocolConstExtensions.GetAllTopicNamesByIssued();
         topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived());
 
         foreach (var item in topics)
         {
-            await kafkaAdminClient.CreateTopicAsync(item, configuration.GetValue<int>(CommonConst.NumPartitions), configuration.GetValue<short>(CommonConst.KafkaReplicationFactor));
+            await kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor);
         }
     }
 

src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs

@@ -389,79 +389,79 @@ namespace JiShe.CollectBus.RedisDataCache
             pageSize = Math.Clamp(pageSize, 1, 10000);
 
             const string luaScript = @"
-        local command = ARGV[1]
+            local command = ARGV[1]
-        local range_start = ARGV[2]
+            local range_start = ARGV[2]
-        local range_end = ARGV[3]
+            local range_end = ARGV[3]
-        local limit = tonumber(ARGV[4])
+            local limit = tonumber(ARGV[4])
-        local last_score = ARGV[5]
+            local last_score = ARGV[5]
-        local last_member = ARGV[6]
+            local last_member = ARGV[6]
 
-        -- 获取扩展数据（3倍分页大小）
+            -- 获取扩展数据（5倍分页大小）
-        local members
+            local members
-        if command == 'ZRANGEBYSCORE' then
+            if command == 'ZRANGEBYSCORE' then
-            members = redis.call('ZRANGEBYSCORE', KEYS[1], range_start, range_end, 
+                members = redis.call('ZRANGEBYSCORE', KEYS[1], range_start, range_end, 
-                'WITHSCORES', 'LIMIT', 0, limit * 5)
+                    'WITHSCORES', 'LIMIT', 0, limit * 5)
-        else
+            else
-            members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, 
+                members = redis.call('ZREVRANGEBYSCORE', KEYS[1], range_start, range_end, 
-                'WITHSCORES', 'LIMIT', 0, limit * 5)
+                    'WITHSCORES', 'LIMIT', 0, limit * 5)
-        end
+            end
 
-        -- 精确分页过滤
+            -- 精确分页过滤
-        local filtered = {}
+            local filtered = {}
-        local count = 0
+            local count = 0
-        local start_index = 1
+            local start_index = 1
 
-        -- 存在锚点时寻找起始位置
+            -- 存在锚点时寻找起始位置
-        if last_score ~= '' and last_member ~= '' then
+            if last_score ~= '' and last_member ~= '' then
-            for i=1,#members,2 do
+                for i=1,#members,2 do
-                local score = members[i+1]
+                    local score = members[i+1]
-                local member = members[i]
+                    local member = members[i]
                 
-                if command == 'ZRANGEBYSCORE' then
+                    if command == 'ZRANGEBYSCORE' then
-                    if tonumber(score) > tonumber(last_score) then
+                        if tonumber(score) > tonumber(last_score) then
-                        start_index = i
-                        break
-                    elseif tonumber(score) == tonumber(last_score) then
-                        if member > last_member then
                             start_index = i
                             break
+                        elseif tonumber(score) == tonumber(last_score) then
+                            if member > last_member then
+                                start_index = i
+                                break
+                            end
                         end
-                    end
+                    else
-                else
+                        if tonumber(score) < tonumber(last_score) then
-                    if tonumber(score) < tonumber(last_score) then
-                        start_index = i
-                        break
-                    elseif tonumber(score) == tonumber(last_score) then
-                        if member < last_member then
                             start_index = i
                             break
+                        elseif tonumber(score) == tonumber(last_score) then
+                            if member < last_member then
+                                start_index = i
+                                break
+                            end
                         end
                     end
                 end
             end
-        end
 
-        -- 收集有效数据
+            -- 收集有效数据
-        for i=start_index,#members,2 do
+            for i=start_index,#members,2 do
-            if count >= limit then break end
+                if count >= limit then break end
-            table.insert(filtered, members[i])
+                table.insert(filtered, members[i])
-            table.insert(filtered, members[i+1])
+                table.insert(filtered, members[i+1])
-            count = count + 1
+                count = count + 1
-        end
+            end
 
-        -- 提取有效数据
+            -- 提取有效数据
-        local result_members = {}
+            local result_members = {}
-        local result_scores = {}
+            local result_scores = {}
-        for i=1,#filtered,2 do
+            for i=1,#filtered,2 do
-            table.insert(result_members, filtered[i])
+                table.insert(result_members, filtered[i])
-            table.insert(result_scores, filtered[i+1])
+                table.insert(result_scores, filtered[i+1])
-        end
+            end
 
-        if #result_members == 0 then return {0,{},{},{}} end
+            if #result_members == 0 then return {0,{},{},{}} end
 
-        -- 获取Hash数据
+            -- 获取Hash数据
-        local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members))
+            local hash_data = redis.call('HMGET', KEYS[2], unpack(result_members))
-        return {#result_members, result_members, result_scores, hash_data}";
+            return {#result_members, result_members, result_scores, hash_data}";
 
             // 构造查询范围（包含等于）
             string rangeStart, rangeEnd;
@@ -541,66 +541,6 @@ namespace JiShe.CollectBus.RedisDataCache
             }
         }
           
-        /// <summary>
-        /// 并行分页导出方法（百万级数据支持）
-        /// </summary>
-        public async Task<List<T>> FullExportParallel<T>(
-            string hashKey,
-            string zsetKey,
-            int parallelDegree = 10,
-            int pageSize = 5000) where T : DeviceCacheBasicModel
-        {
-            var result = new ConcurrentBag<T>();
-            var totalCount = await GetTotalCount(zsetKey);
-            var totalPages = (int)Math.Ceiling(totalCount / (double)pageSize);
-
-            var semaphore = new SemaphoreSlim(parallelDegree);
-            var tasks = new List<Task>();
-
-            decimal? lastScore = null;
-            string lastMember = null;
-            var isDescending = true;
-
-            for (int page = 0; page < totalPages; page++)
-            {
-                await semaphore.WaitAsync();
-
-                tasks.Add(Task.Run(async () =>
-                {
-                    try
-                    {
-                        var pageResult = await GetAllPagedData<T>(
-                            hashKey,
-                            zsetKey,
-                            pageSize,
-                            lastScore,
-                            lastMember,
-                            isDescending);
-
-                        foreach (var item in pageResult.Items)
-                        {
-                            result.Add(item);
-                        }
-
-                        // 更新分页锚点
-                        if (pageResult.HasNext)
-                        {
-                            lastScore = pageResult.NextScore;
-                            lastMember = pageResult.NextMember;
-                        }
-                    }
-                    finally
-                    {
-                        semaphore.Release();
-                    }
-                }));
-            }
-
-            await Task.WhenAll(tasks);
-            return result.ToList();
-        }
-
-
         /// <summary>
         /// 通过ZSET索引获取数据，支持10万级别数据处理，控制在13秒以内。
         /// </summary>

src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs

@@ -22,7 +22,7 @@ namespace JiShe.CollectBus.Kafka
         /// <summary>
         /// kafka主题副本数量
         /// </summary>
-        public int KafkaReplicationFactor { get; set; }
+        public short KafkaReplicationFactor { get; set; }
 
         /// <summary>
         /// kafka主题分区数量