admin/JiShe.CollectBus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

dev #2

Merged
admin merged 176 commits from dev into master 2025-04-18 01:31:49 +00:00
Conversation 0 Commits 176 Files Changed 427 +54 -54
3 changed files with 54 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit ca1e4e28e5 - Show all commits

22
modules/JiShe.CollectBus.Kafka/Consumer/ConsumerService.cs
View File

@ -114,21 +114,18 @@ namespace JiShe.CollectBus.Kafka.Consumer
consumer!.Subscribe(topics); consumer!.Subscribe(topics);
_ = Task.Run(async () => await Task.Run(async () =>
{ {
while (!cts.IsCancellationRequested) while (!cts.IsCancellationRequested)
{ {
try try
{ {
//_logger.LogInformation($"Kafka消费: {string.Join("", topics)} 开始拉取消息....");
var result = consumer.Consume(cts.Token); var result = consumer.Consume(cts.Token);
if (result == null || result.Message==null || result.Message.Value == null) if (result == null || result.Message==null || result.Message.Value == null)
{
#if DEBUG
_logger.LogWarning($"Kafka消费: {result?.Topic} 分区 {result?.Partition} 值为NULL");
#endif
//consumer.Commit(result); // 手动提交
continue; continue;
}
if (result.IsPartitionEOF) if (result.IsPartitionEOF)
{ {
_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition); _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);
@ -188,22 +185,17 @@ namespace JiShe.CollectBus.Kafka.Consumer
consumer!.Subscribe(topics); consumer!.Subscribe(topics);
_ = Task.Run(async () => await Task.Run(async () =>
{ {
while (!cts.IsCancellationRequested) while (!cts.IsCancellationRequested)
{ {
try try
{ {
//_logger.LogInformation($"Kafka消费: {string.Join("", topics)} 开始拉取消息....");
var result = consumer.Consume(cts.Token); var result = consumer.Consume(cts.Token);
if (result == null || result.Message==null || result.Message.Value == null) if (result == null || result.Message==null || result.Message.Value == null)
{
#if DEBUG
_logger.LogWarning($"Kafka消费: {result?.Topic} 分区 {result?.Partition} 值为NULL");
#endif
//consumer.Commit(result); // 手动提交
consumer.StoreOffset(result);
continue; continue;
}
if (result.IsPartitionEOF) if (result.IsPartitionEOF)
{ {
_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition); _logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);

78
modules/JiShe.CollectBus.Kafka/KafkaSubcribesExtensions.cs
View File

@ -12,6 +12,7 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Newtonsoft.Json; using Newtonsoft.Json;
using System.Collections.Generic;
using System.Reflection; using System.Reflection;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -40,7 +41,7 @@ namespace JiShe.CollectBus.Kafka
kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor).ConfigureAwait(false).GetAwaiter().GetResult(); kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor).ConfigureAwait(false).GetAwaiter().GetResult();
} }
lifetime.ApplicationStarted.Register(() => lifetime.ApplicationStarted.Register(async() =>
{ {
var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>(); var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>();
int threadCount = 0; int threadCount = 0;
@ -65,6 +66,7 @@ namespace JiShe.CollectBus.Kafka
!type.IsAbstract && !type.IsInterface).ToList(); ; !type.IsAbstract && !type.IsInterface).ToList(); ;
if (subscribeTypes.Count == 0) if (subscribeTypes.Count == 0)
continue; continue;
foreach (var subscribeType in subscribeTypes) foreach (var subscribeType in subscribeTypes)
{ {
var subscribes = provider.GetServices(subscribeType).ToList(); var subscribes = provider.GetServices(subscribeType).ToList();
@ -80,7 +82,6 @@ namespace JiShe.CollectBus.Kafka
} }
} }
logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程"); logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程");
}); });
} }
@ -100,7 +101,7 @@ namespace JiShe.CollectBus.Kafka
kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor).ConfigureAwait(false).GetAwaiter().GetResult(); kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor).ConfigureAwait(false).GetAwaiter().GetResult();
} }
lifetime.ApplicationStarted.Register(() => lifetime.ApplicationStarted.Register(async () =>
{ {
var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>(); var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>();
int threadCount = 0; int threadCount = 0;
@ -140,6 +141,7 @@ namespace JiShe.CollectBus.Kafka
.ToArray(); .ToArray();
//var configuration = provider.GetRequiredService<IConfiguration>(); //var configuration = provider.GetRequiredService<IConfiguration>();
int threadCount = 0; int threadCount = 0;
List<Task> tasks = new List<Task>();
foreach (var sub in subscribedMethods) foreach (var sub in subscribedMethods)
{ {
int partitionCount = 3;// kafkaOptionConfig.NumPartitions; int partitionCount = 3;// kafkaOptionConfig.NumPartitions;
@ -156,7 +158,7 @@ namespace JiShe.CollectBus.Kafka
threadCount++; threadCount++;
} }
} }
return Tuple.Create(threadCount, subscribedMethods.Length); return await Task.FromResult(Tuple.Create(threadCount, subscribedMethods.Length));
} }
/// <summary> /// <summary>
@ -169,42 +171,48 @@ namespace JiShe.CollectBus.Kafka
/// <returns></returns> /// <returns></returns>
private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr,MethodInfo method, object subscribe, ILogger<CollectBusKafkaModule> logger) private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr,MethodInfo method, object subscribe, ILogger<CollectBusKafkaModule> logger)
{ {
var consumerService = provider.GetRequiredService<IConsumerService>(); await Task.Run(async () =>
{
var consumerService = provider.GetRequiredService<IConsumerService>();
if (attr.EnableBatch) if (attr.EnableBatch)
{
await consumerService.SubscribeBatchAsync<object>(attr.Topic, async (message) =>
{ {
try await consumerService.SubscribeBatchAsync<object>(attr.Topic, async (message) =>
{ {
// 处理消息 try
return await ProcessMessageAsync(message.ToList(), method, subscribe); {
} logger.LogInformation($"kafka批量消费消息:{message}");
catch (ConsumeException ex) // 处理消息
{ return await ProcessMessageAsync(message.ToList(), method, subscribe);
// 处理消费错误 }
logger.LogError($"kafka批量消费异常:{ex.Message}"); catch (ConsumeException ex)
} {
return await Task.FromResult(false); // 处理消费错误
}, attr.GroupId, attr.BatchSize,attr.BatchTimeout); logger.LogError($"kafka批量消费异常:{ex.Message}");
} }
else return await Task.FromResult(false);
{ }, attr.GroupId, attr.BatchSize, attr.BatchTimeout);
await consumerService.SubscribeAsync<object>(attr.Topic, async (message) => }
else
{ {
try await consumerService.SubscribeAsync<object>(attr.Topic, async (message) =>
{ {
// 处理消息 try
return await ProcessMessageAsync(new List<object>() { message }, method, subscribe); {
} logger.LogInformation($"kafka消费消息:{message}");
catch (ConsumeException ex) // 处理消息
{ return await ProcessMessageAsync(new List<object>() { message }, method, subscribe);
// 处理消费错误 }
logger.LogError($"kafka消费异常:{ex.Message}"); catch (ConsumeException ex)
} {
return await Task.FromResult(false); // 处理消费错误
}, attr.GroupId); logger.LogError($"kafka消费异常:{ex.Message}");
} }
return await Task.FromResult(false);
}, attr.GroupId);
}
});
} }

8
shared/JiShe.CollectBus.Common/Consts/ProtocolConst.cs
View File

@ -12,11 +12,11 @@ namespace JiShe.CollectBus.Common.Consts
/// <summary> /// <summary>
/// 心跳下行消息主题 /// 心跳下行消息主题
/// </summary> /// </summary>
public const string SubscriberHeartbeatIssuedEventName = "issued.heartbeat.event2"; public const string SubscriberHeartbeatIssuedEventName = "issued.heartbeat.event";
/// <summary> /// <summary>
/// 登录下行消息主题 /// 登录下行消息主题
/// </summary> /// </summary>
public const string SubscriberLoginIssuedEventName = "issued.login.event2"; public const string SubscriberLoginIssuedEventName = "issued.login.event";
/// <summary> /// <summary>
/// 上行消息主题,测试使用 /// 上行消息主题,测试使用
@ -26,11 +26,11 @@ namespace JiShe.CollectBus.Common.Consts
/// <summary> /// <summary>
/// 心跳上行消息主题 /// 心跳上行消息主题
/// </summary> /// </summary>
public const string SubscriberHeartbeatReceivedEventName = "received.heartbeat.event2"; public const string SubscriberHeartbeatReceivedEventName = "received.heartbeat.event";
/// <summary> /// <summary>
/// 登录上行消息主题 /// 登录上行消息主题
/// </summary> /// </summary>
public const string SubscriberLoginReceivedEventName = "received.login.event2"; public const string SubscriberLoginReceivedEventName = "received.login.event";
#region 电表消息主题 #region 电表消息主题
/// <summary> /// <summary>