调整kafka约束

.gitignore

@@ -401,3 +401,4 @@ FodyWeavers.xsd
 # ABP Studio
 **/.abpstudio/
 /src/JiShe.CollectBus.Host/Plugins/*.dll
+JiShe.CollectBus.Kafka.Test

src/JiShe.CollectBus.KafkaProducer/Attributes/KafkaSubscribeAttribute.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.Kafka.Attributes
+{
+    [AttributeUsage(AttributeTargets.Method)]
+    public class KafkaSubscribeAttribute : Attribute
+    {
+        /// <summary>
+        /// 订阅的主题
+        /// </summary>
+        public string[] Topics { get; set; }
+
+        /// <summary>
+        /// 分区
+        /// </summary>
+        public int Partition { get; set; } = -1;
+
+        /// <summary>
+        /// 消费者组
+        /// </summary>
+        public string GroupId { get; set; } = "default";
+
+        public KafkaSubscribeAttribute(string[] topics)
+        {
+            this.Topics = topics;
+        }
+
+        public KafkaSubscribeAttribute(string[] topics, string groupId)
+        {
+            this.Topics = topics;
+            this.GroupId = groupId;
+        }
+
+        public KafkaSubscribeAttribute(string[] topics, int partition)
+        {
+            this.Topics = topics;
+            this.Partition = partition;
+        }
+
+        public KafkaSubscribeAttribute(string[] topics, int partition, string groupId)
+        {
+            this.Topics = topics;
+            this.Partition = partition;
+            this.GroupId = groupId;
+        }
+    }
+}

src/JiShe.CollectBus.KafkaProducer/CollectBusKafkaModule.cs

@@ -1,9 +1,14 @@
 using Confluent.Kafka;
 using JiShe.CollectBus.Kafka.Consumer;
 using JiShe.CollectBus.Kafka.Producer;
+using Microsoft.AspNetCore.Builder;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using System.Reflection;
+using Volo.Abp;
+using Volo.Abp.DependencyInjection;
 using Volo.Abp.Modularity;
+using static Confluent.Kafka.ConfigPropertyNames;
 
 namespace JiShe.CollectBus.Kafka
 {
@@ -16,5 +21,12 @@ namespace JiShe.CollectBus.Kafka
             // 注册Consumer
             context.Services.AddTransient(typeof(IConsumerService<,>), typeof(ConsumerService<,>));
         }
+
+        public override void OnApplicationInitialization(ApplicationInitializationContext context)
+        {
+            var app = context.GetApplicationBuilder();
+            app.UseKafkaSubscribers(Assembly.Load("JiShe.CollectBus.Application"));
+
+        }
     }
 }

src/JiShe.CollectBus.KafkaProducer/Consumer/ConsumerService.cs

@@ -69,6 +69,34 @@ namespace JiShe.CollectBus.Kafka.Consumer
             }
         }
 
+        /// <summary>
+        /// 订阅多个topic
+        /// </summary>
+        /// <param name="topics"></param>
+        /// <param name="messageHandler"></param>
+        /// <returns></returns>
+        public async Task SubscribeAsync(string[] topics, Func<TKey, TValue, Task> messageHandler)
+        {
+            _cancellationTokenSource = new CancellationTokenSource();
+            Instance.Subscribe(topics);
+
+            try
+            {
+                while (!_cancellationTokenSource.Token.IsCancellationRequested)
+                {
+                    var result = Instance.Consume(_cancellationTokenSource.Token);
+                    if (result != null)
+                    {
+                        await messageHandler(result.Message.Key, result.Message.Value);
+                    }
+                }
+            }
+            catch (OperationCanceledException)
+            {
+                Instance.Close();
+            }
+        }
+
         public void Unsubscribe()
         {
             _cancellationTokenSource?.Cancel();

src/JiShe.CollectBus.KafkaProducer/Consumer/IConsumerService.cs

@@ -11,5 +11,13 @@ namespace JiShe.CollectBus.Kafka.Consumer
         Task SubscribeAsync(string topic, Func<TKey, TValue, Task> messageHandler);
         void Unsubscribe();
         void Dispose();
+
+        /// <summary>
+        /// 订阅多个topic
+        /// </summary>
+        /// <param name="topics"></param>
+        /// <param name="messageHandler"></param>
+        /// <returns></returns>
+        Task SubscribeAsync(string[] topics, Func<TKey, TValue, Task> messageHandler);
     }
 }

src/JiShe.CollectBus.KafkaProducer/IKafkaSubscribe.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.Kafka
+{
+    public interface IKafkaSubscribe
+    {
+    }
+}

src/JiShe.CollectBus.KafkaProducer/JiShe.CollectBus.Kafka.csproj

@@ -8,7 +8,8 @@
 
   <ItemGroup>
     <PackageReference Include="Confluent.Kafka" Version="2.9.0" />
-    <PackageReference Include="Volo.Abp" Version="8.3.3" />
+    <PackageReference Include="Volo.Abp.AspNetCore" Version="8.3.3" />
+    <PackageReference Include="Volo.Abp.Core" Version="8.3.3" />
   </ItemGroup>
 
 </Project>

src/JiShe.CollectBus.KafkaProducer/KafkaSubcribesExtensions.cs

@@ -0,0 +1,119 @@
+using Confluent.Kafka;
+using JiShe.CollectBus.Kafka.Attributes;
+using JiShe.CollectBus.Kafka.Consumer;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace JiShe.CollectBus.Kafka
+{
+    public static class KafkaSubcribesExtensions
+    {
+        /// <summary>
+        /// 添加Kafka订阅
+        /// </summary>
+        /// <param name="app"></param>
+        /// <param name="assembly"></param>
+        public static void UseKafkaSubscribers(this IApplicationBuilder app, Assembly assembly)
+        {
+            var subscribeTypes = assembly.GetTypes()
+                .Where(t => typeof(IKafkaSubscribe).IsAssignableFrom(t))
+                .ToList();
+
+            if (subscribeTypes.Count == 0) return;
+
+            var provider = app.ApplicationServices;
+            var lifetime = provider.GetRequiredService<IHostApplicationLifetime>();
+
+            lifetime.ApplicationStarted.Register(() =>
+            {
+                foreach (var subscribeType in subscribeTypes)
+                {
+                    var subscribes = provider.GetServices(subscribeType).ToList();
+                    subscribes.ForEach(subscribe => { 
+                    
+                        if(subscribe is IKafkaSubscribe)
+                        {
+                            BuildKafkaSubscriber(subscribe, provider);
+                        }
+                    });
+                }
+            });
+        }
+
+        /// <summary>
+        /// 构建Kafka订阅
+        /// </summary>
+        /// <param name="subscribe"></param>
+        /// <param name="provider"></param>
+        private static void BuildKafkaSubscriber(object subscribe, IServiceProvider provider)
+        {
+            var methods = subscribe.GetType().GetMethods();
+            foreach (var method in methods)
+            {
+                var attr = method.GetCustomAttribute<KafkaSubscribeAttribute>();
+                if (attr == null) continue;
+
+                // 启动后台消费线程
+                Task.Run(() => StartConsumerAsync(provider, attr, method, subscribe));
+            }
+        }
+
+        /// <summary>
+        /// 启动后台消费线程
+        /// </summary>
+        /// <param name="config"></param>
+        /// <param name="attr"></param>
+        /// <param name="method"></param>
+        /// <param name="consumerInstance"></param>
+        /// <returns></returns>
+        private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr,MethodInfo method, object subscribe)
+        {
+            var consumerService = provider.GetRequiredService<IConsumerService<Ignore, string>>();
+            await consumerService.SubscribeAsync(attr.Topics, async (key, message) =>
+            {
+                try
+                {
+                    if (string.IsNullOrEmpty(message))
+                        await Task.CompletedTask ;
+
+                    // 处理消息
+                    await ProcessMessageAsync(message, method, subscribe);
+                }
+                catch (ConsumeException ex)
+                {
+                    // 处理消费错误
+                    throw;
+                }
+            });
+        }
+
+        private static async Task ProcessMessageAsync(string message, MethodInfo method, object subscribe)
+        {
+            var parameters = method.GetParameters();
+            if (parameters.Length != 1) return;
+
+            var paramType = parameters[0].ParameterType;
+            var messageObj = paramType == typeof(string)
+                ? message
+                : JsonConvert.DeserializeObject(message, paramType);
+
+            if (method.ReturnType == typeof(Task))
+            {
+                await (Task)method.Invoke(subscribe, new[] { messageObj })!;
+            }
+            else
+            {
+                method.Invoke(subscribe, new[] { messageObj });
+            }
+        }
+
+    }
+}

src/JiShe.CollectBus.KafkaProducer/Producer/ProducerService.cs

@@ -37,7 +37,7 @@ namespace JiShe.CollectBus.Kafka.Producer
                 CompressionType = CompressionType.Lz4, // 配置使用压缩算法LZ4，其他：gzip/snappy/zstd
                 BatchSize = 32768, // 修改批次大小为32K
                 LingerMs = 20, // 修改等待时间为20ms
-                Acks = Acks.All, // 表明只有所有副本Broker都收到消息才算提交成功
+                Acks = Acks.All, // 表明只有所有副本Broker都收到消息才算提交成功, 可以 Acks.Leader
                 MessageSendMaxRetries = 50, // 消息发送失败最大重试50次
             };