提交代码

src/JiShe.CollectBus.Application/CollectBusApplicationModule.cs

@@ -22,6 +22,7 @@ using Volo.Abp.BackgroundWorkers;
 using Volo.Abp.BackgroundWorkers.Hangfire;
 using Volo.Abp.EventBus;
 using Volo.Abp.Modularity;
+using Microsoft.Extensions.Options;
 
 namespace JiShe.CollectBus;
 
@@ -69,13 +70,14 @@ public class CollectBusApplicationModule : AbpModule
         //初始化主题信息
         var kafkaAdminClient = context.ServiceProvider.GetRequiredService<IAdminClientService>();
         var configuration = context.ServiceProvider.GetRequiredService<IConfiguration>();
+        var kafkaOptions = context.ServiceProvider.GetRequiredService<IOptions<KafkaOptionConfig>>();
 
         List<string> topics = ProtocolConstExtensions.GetAllTopicNamesByIssued();
         topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived());
 
         foreach (var item in topics)
         {
-            await kafkaAdminClient.CreateTopicAsync(item, configuration.GetValue<int>(CommonConst.NumPartitions), configuration.GetValue<short>(CommonConst.KafkaReplicationFactor));
+            await kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor);
         }
     }
 

src/JiShe.CollectBus.Application/RedisDataCache/RedisDataCacheService.cs

@@ -396,7 +396,7 @@ namespace JiShe.CollectBus.RedisDataCache
             local last_score = ARGV[5]
             local last_member = ARGV[6]
 
-        -- 获取扩展数据（3倍分页大小）
+            -- 获取扩展数据（5倍分页大小）
             local members
             if command == 'ZRANGEBYSCORE' then
                 members = redis.call('ZRANGEBYSCORE', KEYS[1], range_start, range_end, 
@@ -541,66 +541,6 @@ namespace JiShe.CollectBus.RedisDataCache
             }
         }
           
-        /// <summary>
-        /// 并行分页导出方法（百万级数据支持）
-        /// </summary>
-        public async Task<List<T>> FullExportParallel<T>(
-            string hashKey,
-            string zsetKey,
-            int parallelDegree = 10,
-            int pageSize = 5000) where T : DeviceCacheBasicModel
-        {
-            var result = new ConcurrentBag<T>();
-            var totalCount = await GetTotalCount(zsetKey);
-            var totalPages = (int)Math.Ceiling(totalCount / (double)pageSize);
-
-            var semaphore = new SemaphoreSlim(parallelDegree);
-            var tasks = new List<Task>();
-
-            decimal? lastScore = null;
-            string lastMember = null;
-            var isDescending = true;
-
-            for (int page = 0; page < totalPages; page++)
-            {
-                await semaphore.WaitAsync();
-
-                tasks.Add(Task.Run(async () =>
-                {
-                    try
-                    {
-                        var pageResult = await GetAllPagedData<T>(
-                            hashKey,
-                            zsetKey,
-                            pageSize,
-                            lastScore,
-                            lastMember,
-                            isDescending);
-
-                        foreach (var item in pageResult.Items)
-                        {
-                            result.Add(item);
-                        }
-
-                        // 更新分页锚点
-                        if (pageResult.HasNext)
-                        {
-                            lastScore = pageResult.NextScore;
-                            lastMember = pageResult.NextMember;
-                        }
-                    }
-                    finally
-                    {
-                        semaphore.Release();
-                    }
-                }));
-            }
-
-            await Task.WhenAll(tasks);
-            return result.ToList();
-        }
-
-
         /// <summary>
         /// 通过ZSET索引获取数据，支持10万级别数据处理，控制在13秒以内。
         /// </summary>

src/JiShe.CollectBus.KafkaProducer/KafkaOptionConfig.cs

@@ -22,7 +22,7 @@ namespace JiShe.CollectBus.Kafka
         /// <summary>
         /// kafka主题副本数量
         /// </summary>
-        public int KafkaReplicationFactor { get; set; }
+        public short KafkaReplicationFactor { get; set; }
 
         /// <summary>
         /// kafka主题分区数量