Compare commits

..

251 Commits

Author SHA1 Message Date
ea81151ca9 Merge pull request 'dev' (#4) from dev into master
Reviewed-on: daizan/JiShe.CollectBus#4
2025-05-20 05:39:25 +00:00
ChenYi
87dd6be681 降低增量源码生成器依赖库版本号为4.11.0 2025-05-20 11:56:37 +08:00
cli
d958c32054 update 2025-05-19 15:53:49 +08:00
cli
4ad705d645 修改临时dockerfile 2025-05-19 15:46:12 +08:00
cli
2d936e10ca 修改代码 2025-05-19 15:45:02 +08:00
cli
6124131313 添加临时dockerfile 2025-05-19 15:07:52 +08:00
cli
c3f940fd7b dockerfile 2025-05-19 10:13:33 +08:00
cli
d66d9940bf 修改dockerfile 2025-05-19 10:12:31 +08:00
cli
252e93bbfc 添加主程序解决方案文件 2025-05-19 10:08:35 +08:00
75b6da16a0 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-05-19 08:51:40 +08:00
ba92a3d604 打开注释的订阅信息,优化电表信息未缓存不保存数据,仅保存日志 2025-05-19 08:51:10 +08:00
cli
97bdcb238a Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-05-19 08:38:13 +08:00
cli
15308fe2d7 添加正式环境配置文件 2025-05-19 08:38:09 +08:00
070173af4f 增加TCP 客户端主动关闭监听 2025-05-18 17:18:31 +08:00
ChenYi
9603b02532 优化报文数据构建 2025-05-18 17:14:30 +08:00
ChenYi
921973e5d4 优化IoTDB数据驱动 2025-05-18 16:04:23 +08:00
ChenYi
17df6af4f0 优化结构 2025-05-18 14:10:39 +08:00
ChenYi
17e36f5a56 迁移服务搭建 2025-05-18 13:36:11 +08:00
3bb691b312 调整判断 2025-05-16 14:57:48 +08:00
63a6ace884 修复透明转发回来匹配表信息问题 2025-05-16 13:47:54 +08:00
99e19428a1 解决冲突 2025-05-16 09:25:38 +08:00
ChenYi
d87f7a2c41 解决IoTDB驱动单测点数据模型元组可空类型元素映射失败的问题。 2025-05-16 09:22:54 +08:00
ChenYi
6d95430b63 修复IoTDB驱动decimal类型解析赋值异常的问题 2025-05-15 23:59:10 +08:00
a772396bba 提交代码 2025-05-15 16:24:05 +08:00
900075adf8 Merge branch 'zhy_feat_dev_v10' into dev 2025-05-15 15:26:00 +08:00
b5af5c9cce 暂存代码 2025-05-15 15:18:56 +08:00
52207a92fa Merge branch 'zhy_feat_dev_v10' into dev 2025-05-15 14:35:12 +08:00
88c4a883a6 修改代码 2025-05-15 14:34:44 +08:00
ChenYi
1b5b43fde6 修改代码 2025-05-15 14:05:18 +08:00
ChenYi
fcf4b0c852 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-05-15 11:23:12 +08:00
ChenYi
e0c7ceab32 三相表断闸合闸 2025-05-15 11:23:08 +08:00
84bb1a9563 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-05-15 09:53:43 +08:00
3e10a31840 调整事件类型定义 2025-05-15 09:51:50 +08:00
ChenYi
e02a0953e9 完善3761下行透明转发处理逻辑 2025-05-14 23:28:44 +08:00
ChenYi
be3cd5d3e7 优化异常捕捉,修复IoTDBEntity基类属性异常的问题 2025-05-14 17:41:39 +08:00
ChenYi
ba0af3a12a 合并代码 2025-05-14 15:21:59 +08:00
ChenYi
ee0b48afbd 修改代码 2025-05-14 15:21:33 +08:00
483590534a 调整代码 2025-05-14 15:20:26 +08:00
ChenYi
94ff58dd12 完善Kafka主题消息消费订阅 2025-05-14 14:40:34 +08:00
ChenYi
9df3f652f9 合并代码 2025-05-14 11:34:18 +08:00
ChenYi
f73254e04f 优化IoTDB查询条件处理,使用委托进行实现值类型转换拼接 2025-05-14 11:33:31 +08:00
d99737c0e9 Merge branch 'zhy_feat_dev_v10' into dev 2025-05-14 08:32:28 +08:00
a62284005a 调整代码 2025-05-14 08:32:04 +08:00
f11857367a Merge branch 'zhy_feat_dev_v10' into dev 2025-05-13 17:49:56 +08:00
2e1115c655 优化取设备信息统一取redis 2025-05-13 17:49:12 +08:00
cli
2e58aecc2a Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-05-13 17:02:09 +08:00
cli
5f7ba73575 添加生产环境配置文件 2025-05-13 17:02:03 +08:00
ChenYi
35c483d3e3 合并代码 2025-05-13 14:52:05 +08:00
ChenYi
c4d4337673 修改设备缓存信息Redis存储结构 2025-05-13 14:51:38 +08:00
cli
f1f5e82171 合并 2025-05-13 08:31:01 +08:00
ChenYi
13fd36c647 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-05-12 23:18:26 +08:00
ChenYi
705a5cbbf7 新增设备缓存信息Redis数据结构 2025-05-12 23:18:02 +08:00
5b4673adef 优化iotdb 存储 2025-05-12 17:24:47 +08:00
fed56f33ff Merge branch 'dev' into zhy_feat_dev_v10 2025-05-12 17:09:52 +08:00
ChenYi
fa84f42ca2 修复IoTDB可空类型转换赋值失败的问题 2025-05-12 17:08:09 +08:00
ff52fdc46a 优化存储 2025-05-12 16:48:45 +08:00
a8da939ba4 Merge branch 'dev' into zhy_feat_dev_v10 2025-05-12 16:10:44 +08:00
dac7865938 优化保存IOTDB日志做DataType区分 2025-05-12 16:09:53 +08:00
ChenYi
1c8d672170 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-05-12 15:14:44 +08:00
ChenYi
eaf6f22bdc 修复IoTDB查询映射类型异常的问题 2025-05-12 15:14:19 +08:00
3ca548ef96 Merge branch 'zhy_feat_cev_v8' into dev 2025-05-12 14:03:03 +08:00
529623a232 解析所有日志保存到iotdb 2025-05-12 14:02:22 +08:00
ChenYi
80b8942d9d 优化代码 2025-05-12 11:20:50 +08:00
ChenYi
e35172d69f 优化代码 2025-05-12 10:14:08 +08:00
陈益
cf9bf6c210 完善Redis缓存封装 2025-05-11 21:53:55 +08:00
acd0a5c155 Merge branch 'dev' into zhy_feat_cev_v8 2025-05-11 10:43:54 +08:00
57148f3189 代码暂存 2025-05-11 10:42:55 +08:00
ChenYi
f6130e1d0b 优化IoTDB驱动,实现30个字段实体的270W任务数据开发环境处理平均耗时在70秒 2025-05-09 17:54:52 +08:00
cli
72b44632cd 合并 2025-05-09 14:00:30 +08:00
cli
ae2c0b1839 暂存 2025-05-09 14:00:05 +08:00
ChenYi
541c118dbb 修复IoTDB驱动适配增量源码生成器以后,单侧点模式失效的问题 2025-05-08 22:44:01 +08:00
8d1fc46e8f 修改代码 2025-05-08 17:33:07 +08:00
ChenYi
bdfc8eabae 合并代码 2025-05-08 17:22:06 +08:00
ChenYi
fa593de754 完善IoTDB驱动,适配增量源码生成器 2025-05-08 17:21:20 +08:00
1a9d3924b0 优化终端日历时钟解析 2025-05-08 17:20:07 +08:00
75efbb3a36 调试增加否认帧打印 2025-05-08 15:51:25 +08:00
3488b0012a 解析错误数据保存NULL 2025-05-08 15:12:37 +08:00
ChenYi
c03207aa21 优化增量源码生成器和IoTDB驱动 2025-05-08 14:42:13 +08:00
91386007ae 测试下发更改发为字节 2025-05-08 11:58:26 +08:00
245c91c9e3 添加测试下发 2025-05-08 11:25:08 +08:00
ChenYi
db7384ae74 完善增量源码生成器 2025-05-08 11:17:43 +08:00
ChenYi
edecbc386e 完善数据通道数据处理,解决数据丢失的问题。 2025-05-08 10:28:23 +08:00
a5ed0a37a0 修改代码 2025-05-08 09:29:31 +08:00
7a18dc4df9 统一字段名称 2025-05-08 09:25:41 +08:00
0de5177472 统一解析字段名称 2025-05-08 09:25:17 +08:00
ChenYi
ac226110cd 合并代码 2025-05-08 08:48:09 +08:00
ChenYi
f71ce3bacb 完善增量源码生成器 2025-05-08 08:43:37 +08:00
ChenYi
c47ee94469 修改代码 2025-05-07 17:27:37 +08:00
ChenYi
ff517664fe 复杂类型增量源生成器 2025-05-07 17:20:10 +08:00
0328aa08d9 提交代码 2025-05-07 17:10:02 +08:00
ChenYi
6b012d9303 完善复杂类型增量源码生成器 2025-05-07 16:37:26 +08:00
cdcf078e5a 优化消费流程 2025-05-07 15:17:54 +08:00
1ae875606f 优化代码 2025-05-07 11:55:26 +08:00
34d4bc0b8f 优化376.1协议插件解析 2025-05-07 11:53:50 +08:00
ChenYi
bca7202558 优化复杂类型增量源生成器 2025-05-07 10:15:45 +08:00
1718a0afa7 调整代码 2025-05-07 08:37:57 +08:00
ChenYi
53e6bb252a 修复15分钟任务Kafka主题异常的问题,新增增量源码工厂类 2025-05-06 23:46:12 +08:00
be4c25519d 优化解析,移除不必要的DTO 2025-05-06 19:25:15 +08:00
1e85f894a8 优化kafka发布消息 2025-05-06 18:41:23 +08:00
c4335aa3b7 修改代码 2025-05-06 15:28:00 +08:00
f96eb27353 修改代码 2025-05-06 15:21:31 +08:00
09de209a7c Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-05-06 14:35:17 +08:00
ChenYi
01264dd3ec 优化数据通道读取速度,并推送Kafka 2025-05-06 14:33:49 +08:00
94f36cfa62 调整透明转发解析 2025-05-06 14:33:12 +08:00
53fa6f503c Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-30 17:26:27 +08:00
9ccd016bc0 Merge branch 'zhy_feat_cev_v7' into dev 2025-04-30 17:26:04 +08:00
e1e6398348 统一时标类型 2025-04-30 17:25:35 +08:00
ChenYi
0d2bfdc33c 优化数据通道管理 2025-04-30 17:11:09 +08:00
ChenYi
fd9e3e5af3 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-30 15:58:08 +08:00
ChenYi
ede7c8f1bb 实现IoTDB平均每分钟写入120万数据。 2025-04-30 15:57:14 +08:00
318dd2dcbe Merge branch 'zhy_feat_cev_v7' into dev 2025-04-30 15:18:05 +08:00
92b9cb37c1 修改代码 2025-04-30 15:17:51 +08:00
4bde9be137 Merge branch 'zhy_feat_cev_v7' into dev 2025-04-30 15:14:46 +08:00
a70cf4b7ce kafka优化异常重试 2025-04-30 15:14:23 +08:00
ChenYi
5ba1325204 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-30 12:37:01 +08:00
ChenYi
f63cf8abe8 任务数据构建改为通道实现 2025-04-30 12:36:54 +08:00
a97421a00e kafka 订阅验证分区数 2025-04-30 10:35:26 +08:00
0c51c0d854 Merge branch 'zhy_feat_cev_v7' into dev 2025-04-30 09:51:37 +08:00
4e30cc13f1 修复kafka基准测试 2025-04-30 09:50:50 +08:00
ChenYi
8bfc6fa6df 新增数据通道管理服务 2025-04-29 23:55:53 +08:00
ChenYi
5890b16570 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-29 23:48:57 +08:00
ChenYi
52a7da69e9 优化任务数据创建逻辑,改为上一个任务时间为待采集点 2025-04-29 23:48:47 +08:00
b0bbf67f87 Merge branch 'zhy_feat_cev_v7' into dev 2025-04-29 11:44:50 +08:00
f020d456d5 优化返回消息体都增加消息ID 2025-04-29 11:43:16 +08:00
ChenYi
d97e38439a 修复问题 2025-04-29 10:12:09 +08:00
ChenYi
cc8465ec77 合并 2025-04-29 10:03:59 +08:00
ChenYi
f457e140f6 日志添加类型 2025-04-29 10:02:10 +08:00
981d948255 修改代码 2025-04-29 09:42:45 +08:00
0937490f05 代码合并 2025-04-29 09:19:46 +08:00
4f1814b8df 协议解析优化
策略模式回调采用Action方式灵活处理
2025-04-29 09:16:48 +08:00
ChenYi
083a28f7af 调整源码生成器结构 2025-04-29 09:14:58 +08:00
陈益
5c19d02fdd 解决源码生成失败的问题 2025-04-28 22:35:19 +08:00
ChenYi
0b2f2447c5 修改代码 2025-04-28 17:45:00 +08:00
ChenYi
1c1ad56408 修改代码 2025-04-28 17:01:28 +08:00
ChenYi
d18f60f9a7 解决IoTDB写入速度过慢的问题 2025-04-28 16:37:31 +08:00
ChenYi
021153a319 源码生成器应用 2025-04-28 14:07:51 +08:00
ChenYi
cab0e1ed08 增量源码生成 2025-04-28 09:53:45 +08:00
ChenYi
c5c428f551 Merge branch 'feature_定时抄读_18_CY' into dev 2025-04-27 17:27:45 +08:00
ChenYi
8eb99b683a 日冻结,月冻结数据处理 2025-04-27 17:27:04 +08:00
31edca94fb Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-27 17:24:01 +08:00
5b1c393cc0 调整代码 2025-04-27 17:23:26 +08:00
ChenYi
8c6fe78739 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-27 16:32:18 +08:00
ChenYi
33c8fa6209 完善默认端到云协议插件处理,改用配置项 2025-04-27 16:32:03 +08:00
cli
64ca50e39e 修改 2025-04-27 16:24:51 +08:00
cli
f5ac33e28b 修好正式环境连接字符 2025-04-27 16:03:49 +08:00
ChenYi
966172ea0f 修复依赖引入 2025-04-27 15:55:40 +08:00
cli
f9bf6f5263 修改解决方案文件 2025-04-27 15:53:33 +08:00
a3a4952147 代码合并 2025-04-27 15:49:23 +08:00
2ee205d00b 代码暂存 2025-04-27 15:44:54 +08:00
ChenYi
13f055a3e5 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-27 15:44:17 +08:00
ChenYi
e7ff709a68 修改代码 2025-04-27 15:44:13 +08:00
ChenYi
8f64c3aa5b Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-27 15:43:16 +08:00
ChenYi
83cd2f6fc6 合并代码 2025-04-27 15:36:43 +08:00
ChenYi
d27351cb4c 合并代码 2025-04-27 15:32:44 +08:00
ChenYi
b1ccb874a7 升级依赖 2025-04-27 15:30:45 +08:00
cli
13d16903b7 更新配置文件 2025-04-27 14:24:06 +08:00
cli
b34b71e0c5 增加独立sln文件 2025-04-27 14:15:08 +08:00
cli
6f098b4598 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-27 14:10:14 +08:00
cli
496ae0c797 添加打包脚本和dockerfile 2025-04-27 14:09:58 +08:00
0610a80221 Merge pull request 'dev' (#3) from dev into master
Reviewed-on: daizan/JiShe.CollectBus#3
2025-04-27 03:42:50 +00:00
aeb60acd9a Merge branch 'master' into dev 2025-04-27 03:41:34 +00:00
cli
edf92c8940 合并 2025-04-27 11:05:06 +08:00
cli
d37e17d912 临时提交 2025-04-27 11:04:31 +08:00
ChenYi
63acc1c263 修复问题 2025-04-27 10:33:33 +08:00
eb26d89488 合并代码 2025-04-27 10:20:22 +08:00
9b1d5680d6 代码合并 2025-04-27 10:05:12 +08:00
ChenYi
653c9b1fc8 合并代码 2025-04-27 09:53:29 +08:00
713782e60f 暂定透明转发接受匹配规则 2025-04-27 09:49:17 +08:00
cli
e866758782 暂存 2025-04-27 09:47:08 +08:00
b374692c02 优化匹配 2025-04-27 09:44:08 +08:00
ChenYi
55f4debc5e 完成定时抄读集中器版本、定时抄读SIM卡信息 2025-04-27 09:40:31 +08:00
35febf6a99 优化解析保存到IotDB 2025-04-27 09:31:12 +08:00
cli
355aad3ee5 规整命名空间 2025-04-27 09:16:37 +08:00
b80d894f6c 添加常量零序列电流 2025-04-27 08:43:41 +08:00
4c13eb6ee4 Merge branch 'zhy_feat_cev_v5' into dev 2025-04-26 19:07:22 +08:00
314d96dd59 添加接受OCH、ODH、10H等数据解析 2025-04-26 19:04:50 +08:00
cli
946dffbaa7 合并 2025-04-25 17:42:46 +08:00
cli
5adabb3b7e 修改代码 2025-04-25 17:41:43 +08:00
ChenYi
a2cc947d7d Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-25 17:19:03 +08:00
ChenYi
27104ada3a 协议池应用04 2025-04-25 17:18:59 +08:00
32617a9b83 Merge branch 'zhy_feat_cev_v5' into dev 2025-04-25 15:22:20 +08:00
ChenYi
2dbf4ded4a 调整日志 2025-04-25 15:21:46 +08:00
825e7d5f18 调整代码结构 2025-04-25 15:21:43 +08:00
ChenYi
1b7b8a91a5 调整代码 2025-04-25 14:58:51 +08:00
ChenYi
d60589a180 修改代码 2025-04-25 14:42:11 +08:00
ChenYi
7b933f27f1 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-25 14:37:39 +08:00
ChenYi
51b7147a38 调整框架 2025-04-25 14:37:35 +08:00
c3273960c6 调整代码结构 2025-04-25 14:35:59 +08:00
a5cd6b86f2 调整项目结构 2025-04-25 14:23:06 +08:00
f09c50b833 调整协议实体类结构 2025-04-25 13:51:22 +08:00
81369a744e Merge branch 'zhy_feat_cev_v4' into dev 2025-04-25 13:43:28 +08:00
2e7d76d8aa 暂存 2025-04-25 13:42:52 +08:00
ChenYi
9104cda58a 结构调整 2025-04-25 13:42:08 +08:00
ChenYi
710d2a60c4 调整协议池项目组结构 2025-04-25 12:01:15 +08:00
3490fc22f1 解决冲突 2025-04-25 09:31:16 +08:00
16dab4ba29 调整结构 2025-04-25 09:28:56 +08:00
ChenYi
e8a1a7d23e 优化服务应用配置,将其单独管理。 2025-04-25 09:28:20 +08:00
ChenYi
f2be3a5516 合并代码 2025-04-25 08:33:13 +08:00
陈益
e1853b2655 协议池应用02 2025-04-24 23:39:39 +08:00
cli
bf4f631a8c Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-24 21:01:05 +08:00
cli
dbe31c87da 实现重载 2025-04-24 21:01:01 +08:00
264eec540a 修改代码 2025-04-24 19:40:51 +08:00
0ce146e7be 报文处理 2025-04-24 19:35:40 +08:00
cc35ad2664 添加报文解析 2025-04-24 19:31:28 +08:00
ChenYi
dec99af6dd 协议池的应用处理01 2025-04-24 17:48:20 +08:00
cli
820773bac2 修复编译报错 2025-04-24 00:37:00 +08:00
cli
7adee9f257 合并 2025-04-24 00:34:30 +08:00
cli
d4ae0d2a1d 协议卸载与重载 2025-04-24 00:34:00 +08:00
陈益
57021e2b9c 修改代码 2025-04-23 23:42:35 +08:00
ChenYi
310ae97a6b 优化报文存储记录 2025-04-23 17:49:13 +08:00
ChenYi
41d71a5fba 合并代码 2025-04-23 16:19:06 +08:00
ChenYi
4de3fb7db2 完成定时阀控 2025-04-23 16:17:29 +08:00
fff1ba1a7b Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-23 15:11:27 +08:00
8da5fee0a2 提交 2025-04-23 15:10:20 +08:00
cli
039a1a8b89 合并 2025-04-23 14:46:56 +08:00
cli
7dd26dc89b 添加健康检查 2025-04-23 14:46:19 +08:00
9c24f53eb6 Merge branch 'dev' of https://310.jisheyun.com/daizan/JiShe.CollectBus into dev 2025-04-23 14:00:10 +08:00
a340225cda 优化kafka消费者被自动踢出消费组增加自动重试订阅 2025-04-23 13:59:15 +08:00
ChenYi
42a446c24d 完善IoTDB存储路径约束 2025-04-23 11:13:59 +08:00
ChenYi
17bb8a5692 合并代码 2025-04-23 09:45:31 +08:00
ChenYi
9c6e8c0de8 合并代码 2025-04-23 09:45:21 +08:00
ChenYi
d91fc5b35f 完善水表处理 2025-04-23 09:42:09 +08:00
9cb8688425 修改代码 2025-04-22 23:47:59 +08:00
c5364f4a95 优化kafka消费者实例复用
优化登录心跳逻辑
2025-04-22 23:45:08 +08:00
ChenYi
60aad0032b 解决IoTDB无法重复调用查询接口的问题 2025-04-22 23:44:37 +08:00
ChenYi
26f6796409 解决IoTDB数据查询映射异常的问题 2025-04-22 22:11:55 +08:00
cli
ed750f66b6 合并 2025-04-22 21:06:56 +08:00
cli
3db0db19ea 合并 2025-04-22 21:06:35 +08:00
cli
0efb3dec00 添加协议文件监控程序,实现协议文件CRUD的监控,同时记录修改事件,并支持自动重启和健康检查 2025-04-22 21:01:28 +08:00
4543299dde 修改代码 2025-04-22 18:10:18 +08:00
633df21f43 新增报文解析采用策略模式执行解析 2025-04-22 18:07:47 +08:00
ChenYi
5647385582 完善时间控制 2025-04-22 17:58:14 +08:00
5ca4cbad13 Merge branch 'zhy_feat_cev_v3' into dev 2025-04-22 16:49:45 +08:00
7c55d96b7c 修改代码 2025-04-22 16:48:53 +08:00
ChenYi
ea6dc9f39f 修复主题名称 2025-04-22 16:48:43 +08:00
ChenYi
c29ec906b1 合并代码 2025-04-22 16:46:11 +08:00
ChenYi
f4016ce3ad 解决IoTDB纳秒时间戳保存的问题,完善IoTDB的查询封装 2025-04-22 16:44:47 +08:00
b1a5d29fa7 修改代码 2025-04-22 10:05:38 +08:00
37781e32e1 调整代码 2025-04-22 09:40:09 +08:00
2c202081f9 解决冲突 2025-04-22 09:37:49 +08:00
ChenYi
a6d970af19 优化IoTDB数据类型映射,实现微秒和纳秒的扩展封装 2025-04-21 22:57:49 +08:00
ChenYi
309f5c37d2 解决单侧点表模型路径处理问题,并添加路径Attribute,以支持固定存储路径场景 2025-04-21 14:57:12 +08:00
ChenYi
ca1a23fd33 优化IoTDB驱动 2025-04-21 14:20:49 +08:00
ChenYi
3c6a04f947 合并代码 2025-04-21 10:52:28 +08:00
ChenYi
38283db1b2 测试IoTDB实体约束 2025-04-21 10:50:36 +08:00
cli
02f2a2cafc 规范代码 2025-04-21 10:17:40 +08:00
cli
01aeaebb0a 合并 2025-04-21 09:56:22 +08:00
cli
640fcb754d 添加拦截器 2025-04-21 09:54:34 +08:00
ChenYi
cb6cfea4dd 合并代码 2025-04-21 09:48:12 +08:00
ChenYi
cc59b32742 完善IoTDB的数据实体约束 2025-04-21 09:45:30 +08:00
a7602cde86 Merge pull request 'dev' (#2) from dev into master
Reviewed-on: daizan/JiShe.CollectBus#2
2025-04-18 01:31:49 +00:00
404 changed files with 31489 additions and 5762 deletions

30
.dockerignore Normal file
View File

@ -0,0 +1,30 @@
**/.classpath
**/.dockerignore
**/.env
**/.git
**/.gitignore
**/.project
**/.settings
**/.toolstarget
**/.vs
**/.vscode
**/*.*proj.user
**/*.dbmdl
**/*.jfm
**/azds.yaml
**/bin
**/charts
**/docker-compose*
**/Dockerfile*
**/node_modules
**/npm-debug.log
**/obj
**/secrets.dev.yaml
**/values.dev.yaml
LICENSE
README.md
!**/.gitignore
!.git/HEAD
!.git/config
!.git/packed-refs
!.git/refs/heads/**

View File

@ -0,0 +1,20 @@
@ECHO off
cls
ECHO Deleting all BIN and OBJ folders...
ECHO.
FOR /d /r . %%d in (bin,obj) DO (
IF EXIST "%%d" (
ECHO %%d | FIND /I "\node_modules\" > Nul && (
ECHO.Skipping: %%d
) || (
ECHO.Deleting: %%d
rd /s/q "%%d"
)
)
)
ECHO.
ECHO.BIN and OBJ folders have been successfully deleted. Press any key to exit.
pause > nul

52
Dockerfile Normal file
View File

@ -0,0 +1,52 @@
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
WORKDIR /app
EXPOSE 80
EXPOSE 443
ENV TZ=Asia/Shanghai
ENV ASPNETCORE_ENVIRONMENT=Production
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
# WORKDIR /src
COPY ["JiShe.CollectBus.Main.sln", "."]
COPY ["common.props", "."]
COPY ["NuGet.Config", "."]
COPY ["web/", "web/"]
COPY ["modules/", "modules/"]
COPY ["services/", "services/"]
COPY ["shared/", "shared/"]
COPY ["protocols/", "protocols/"]
# 恢复项目依赖
RUN dotnet restore "JiShe.CollectBus.Main.sln"
# 构建项目
WORKDIR "/web/JiShe.CollectBus.Host"
RUN dotnet build "JiShe.CollectBus.Host.csproj" -c Release -o /app/build
# 发布项目
FROM build AS publish
RUN dotnet publish "JiShe.CollectBus.Host.csproj" -c Release -o /app/publish /p:UseAppHost=false
# 创建最终镜像
FROM base AS final
WORKDIR /app
# 创建Plugins目录
RUN mkdir -p /app/Plugins
# 复制发布内容
COPY --from=publish /app/publish .
# 健康检查
HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
CMD curl -f http://localhost:80/health || exit 1
# 设置入口点
ENTRYPOINT ["dotnet", "JiShe.CollectBus.Host.dll"]
# 启动命令
# 可选:添加命令行参数
# CMD ["--urls", "http://+:80"]

View File

@ -0,0 +1,25 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.9.34728.123
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.PluginFileWatcher", "external\JiShe.CollectBus.PluginFileWatcher\JiShe.CollectBus.PluginFileWatcher.csproj", "{0F67A493-A4DF-550E-AB4D-95F55144C706}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{0F67A493-A4DF-550E-AB4D-95F55144C706}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0F67A493-A4DF-550E-AB4D-95F55144C706}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0F67A493-A4DF-550E-AB4D-95F55144C706}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0F67A493-A4DF-550E-AB4D-95F55144C706}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {4324B3B4-B60B-4E3C-91D8-59576B4E26DD}
EndGlobalSection
EndGlobal

188
JiShe.CollectBus.Main.sln Normal file
View File

@ -0,0 +1,188 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.9.34728.123
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Domain.Shared", "shared\JiShe.CollectBus.Domain.Shared\JiShe.CollectBus.Domain.Shared.csproj", "{D64C1577-4929-4B60-939E-96DE1534891A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Domain", "services\JiShe.CollectBus.Domain\JiShe.CollectBus.Domain.csproj", "{F2840BC7-0188-4606-9126-DADD0F5ABF7A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Application.Contracts", "services\JiShe.CollectBus.Application.Contracts\JiShe.CollectBus.Application.Contracts.csproj", "{BD65D04F-08D5-40C1-8C24-77CA0BACB877}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Application", "services\JiShe.CollectBus.Application\JiShe.CollectBus.Application.csproj", "{78040F9E-3501-4A40-82DF-00A597710F35}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.MongoDB", "modules\JiShe.CollectBus.MongoDB\JiShe.CollectBus.MongoDB.csproj", "{F1C58097-4C08-4D88-8976-6B3389391481}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.HttpApi", "web\JiShe.CollectBus.HttpApi\JiShe.CollectBus.HttpApi.csproj", "{077AA5F8-8B61-420C-A6B5-0150A66FDB34}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Host", "web\JiShe.CollectBus.Host\JiShe.CollectBus.Host.csproj", "{35829A15-4127-4F69-8BDE-9405DEAACA9A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Common", "shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj", "{AD2F1928-4411-4511-B564-5FB996EC08B9}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.DbMigrator", "services\JiShe.CollectBus.DbMigrator\JiShe.CollectBus.DbMigrator.csproj", "{8BA01C3D-297D-42DF-BD63-EF07202A0A67}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.FreeSql", "modules\JiShe.CollectBus.FreeSql\JiShe.CollectBus.FreeSql.csproj", "{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.FreeRedis", "modules\JiShe.CollectBus.FreeRedis\JiShe.CollectBus.FreeRedis.csproj", "{C06C4082-638F-2996-5FED-7784475766C1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Kafka", "modules\JiShe.CollectBus.Kafka\JiShe.CollectBus.Kafka.csproj", "{F0288175-F0EC-48BD-945F-CF1512850943}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.IoTDB", "modules\JiShe.CollectBus.IoTDB\JiShe.CollectBus.IoTDB.csproj", "{A3F3C092-0A25-450B-BF6A-5983163CBEF5}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Protocol.Test", "protocols\JiShe.CollectBus.Protocol.Test\JiShe.CollectBus.Protocol.Test.csproj", "{A377955E-7EA1-6F29-8CF7-774569E93925}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Cassandra", "modules\JiShe.CollectBus.Cassandra\JiShe.CollectBus.Cassandra.csproj", "{443B4549-0AC0-4493-8F3E-49C83225DD76}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "1.Web", "1.Web", "{A02F7D8A-04DC-44D6-94D4-3F65712D6B94}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "4.Modules", "4.Modules", "{2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "5.Protocols", "5.Protocols", "{3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "2.Services", "2.Services", "{BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "3.Shared", "3.Shared", "{EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T1882018", "protocols\JiShe.CollectBus.Protocol.T1882018\JiShe.CollectBus.Protocol.T1882018.csproj", "{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T37612012", "protocols\JiShe.CollectBus.Protocol.T37612012\JiShe.CollectBus.Protocol.T37612012.csproj", "{8A61DF78-069B-40B5-8811-614E2960443E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol", "protocols\JiShe.CollectBus.Protocol\JiShe.CollectBus.Protocol.csproj", "{E27377CC-E2D3-4237-060F-96EA214D3129}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T6452007", "protocols\JiShe.CollectBus.Protocol.T6452007\JiShe.CollectBus.Protocol.T6452007.csproj", "{75B7D419-C261-577D-58D6-AA3ACED9129F}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "0.Docs", "0.Docs", "{D8346C4C-55B8-43E8-A6B8-E59D56FE6D92}"
ProjectSection(SolutionItems) = preProject
.gitignore = .gitignore
Dockerfile = Dockerfile
NuGet.Config = NuGet.Config
PackageAndPublish.bat = PackageAndPublish.bat
readme.md = readme.md
Temp.Dockerfile = Temp.Dockerfile
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Analyzers.Shared", "shared\JiShe.CollectBus.Analyzers.Shared\JiShe.CollectBus.Analyzers.Shared.csproj", "{DD68F314-BC66-5601-B094-B1A7BE93F4E0}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Analyzers", "modules\JiShe.CollectBus.Analyzers\JiShe.CollectBus.Analyzers.csproj", "{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{D64C1577-4929-4B60-939E-96DE1534891A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D64C1577-4929-4B60-939E-96DE1534891A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D64C1577-4929-4B60-939E-96DE1534891A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D64C1577-4929-4B60-939E-96DE1534891A}.Release|Any CPU.Build.0 = Release|Any CPU
{F2840BC7-0188-4606-9126-DADD0F5ABF7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F2840BC7-0188-4606-9126-DADD0F5ABF7A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F2840BC7-0188-4606-9126-DADD0F5ABF7A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F2840BC7-0188-4606-9126-DADD0F5ABF7A}.Release|Any CPU.Build.0 = Release|Any CPU
{BD65D04F-08D5-40C1-8C24-77CA0BACB877}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BD65D04F-08D5-40C1-8C24-77CA0BACB877}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BD65D04F-08D5-40C1-8C24-77CA0BACB877}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BD65D04F-08D5-40C1-8C24-77CA0BACB877}.Release|Any CPU.Build.0 = Release|Any CPU
{78040F9E-3501-4A40-82DF-00A597710F35}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{78040F9E-3501-4A40-82DF-00A597710F35}.Debug|Any CPU.Build.0 = Debug|Any CPU
{78040F9E-3501-4A40-82DF-00A597710F35}.Release|Any CPU.ActiveCfg = Release|Any CPU
{78040F9E-3501-4A40-82DF-00A597710F35}.Release|Any CPU.Build.0 = Release|Any CPU
{F1C58097-4C08-4D88-8976-6B3389391481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F1C58097-4C08-4D88-8976-6B3389391481}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F1C58097-4C08-4D88-8976-6B3389391481}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F1C58097-4C08-4D88-8976-6B3389391481}.Release|Any CPU.Build.0 = Release|Any CPU
{077AA5F8-8B61-420C-A6B5-0150A66FDB34}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{077AA5F8-8B61-420C-A6B5-0150A66FDB34}.Debug|Any CPU.Build.0 = Debug|Any CPU
{077AA5F8-8B61-420C-A6B5-0150A66FDB34}.Release|Any CPU.ActiveCfg = Release|Any CPU
{077AA5F8-8B61-420C-A6B5-0150A66FDB34}.Release|Any CPU.Build.0 = Release|Any CPU
{35829A15-4127-4F69-8BDE-9405DEAACA9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{35829A15-4127-4F69-8BDE-9405DEAACA9A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{35829A15-4127-4F69-8BDE-9405DEAACA9A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{35829A15-4127-4F69-8BDE-9405DEAACA9A}.Release|Any CPU.Build.0 = Release|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Release|Any CPU.Build.0 = Release|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Release|Any CPU.Build.0 = Release|Any CPU
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}.Release|Any CPU.Build.0 = Release|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Release|Any CPU.Build.0 = Release|Any CPU
{F0288175-F0EC-48BD-945F-CF1512850943}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F0288175-F0EC-48BD-945F-CF1512850943}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F0288175-F0EC-48BD-945F-CF1512850943}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F0288175-F0EC-48BD-945F-CF1512850943}.Release|Any CPU.Build.0 = Release|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Release|Any CPU.Build.0 = Release|Any CPU
{A377955E-7EA1-6F29-8CF7-774569E93925}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A377955E-7EA1-6F29-8CF7-774569E93925}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A377955E-7EA1-6F29-8CF7-774569E93925}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A377955E-7EA1-6F29-8CF7-774569E93925}.Release|Any CPU.Build.0 = Release|Any CPU
{443B4549-0AC0-4493-8F3E-49C83225DD76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{443B4549-0AC0-4493-8F3E-49C83225DD76}.Debug|Any CPU.Build.0 = Debug|Any CPU
{443B4549-0AC0-4493-8F3E-49C83225DD76}.Release|Any CPU.ActiveCfg = Release|Any CPU
{443B4549-0AC0-4493-8F3E-49C83225DD76}.Release|Any CPU.Build.0 = Release|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Release|Any CPU.Build.0 = Release|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Release|Any CPU.Build.0 = Release|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Release|Any CPU.Build.0 = Release|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Release|Any CPU.Build.0 = Release|Any CPU
{DD68F314-BC66-5601-B094-B1A7BE93F4E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DD68F314-BC66-5601-B094-B1A7BE93F4E0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DD68F314-BC66-5601-B094-B1A7BE93F4E0}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DD68F314-BC66-5601-B094-B1A7BE93F4E0}.Release|Any CPU.Build.0 = Release|Any CPU
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{D64C1577-4929-4B60-939E-96DE1534891A} = {EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}
{F2840BC7-0188-4606-9126-DADD0F5ABF7A} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{BD65D04F-08D5-40C1-8C24-77CA0BACB877} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{78040F9E-3501-4A40-82DF-00A597710F35} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{F1C58097-4C08-4D88-8976-6B3389391481} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{077AA5F8-8B61-420C-A6B5-0150A66FDB34} = {A02F7D8A-04DC-44D6-94D4-3F65712D6B94}
{35829A15-4127-4F69-8BDE-9405DEAACA9A} = {A02F7D8A-04DC-44D6-94D4-3F65712D6B94}
{AD2F1928-4411-4511-B564-5FB996EC08B9} = {EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}
{8BA01C3D-297D-42DF-BD63-EF07202A0A67} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{C06C4082-638F-2996-5FED-7784475766C1} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{F0288175-F0EC-48BD-945F-CF1512850943} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{A3F3C092-0A25-450B-BF6A-5983163CBEF5} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{A377955E-7EA1-6F29-8CF7-774569E93925} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{443B4549-0AC0-4493-8F3E-49C83225DD76} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{8A61DF78-069B-40B5-8811-614E2960443E} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{E27377CC-E2D3-4237-060F-96EA214D3129} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{75B7D419-C261-577D-58D6-AA3ACED9129F} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{DD68F314-BC66-5601-B094-B1A7BE93F4E0} = {EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {4324B3B4-B60B-4E3C-91D8-59576B4E26DD}
EndGlobalSection
EndGlobal

View File

@ -0,0 +1,179 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.9.34728.123
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Domain.Shared", "shared\JiShe.CollectBus.Domain.Shared\JiShe.CollectBus.Domain.Shared.csproj", "{D64C1577-4929-4B60-939E-96DE1534891A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Domain", "services\JiShe.CollectBus.Domain\JiShe.CollectBus.Domain.csproj", "{F2840BC7-0188-4606-9126-DADD0F5ABF7A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Application.Contracts", "services\JiShe.CollectBus.Application.Contracts\JiShe.CollectBus.Application.Contracts.csproj", "{BD65D04F-08D5-40C1-8C24-77CA0BACB877}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Application", "services\JiShe.CollectBus.Application\JiShe.CollectBus.Application.csproj", "{78040F9E-3501-4A40-82DF-00A597710F35}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.MongoDB", "modules\JiShe.CollectBus.MongoDB\JiShe.CollectBus.MongoDB.csproj", "{F1C58097-4C08-4D88-8976-6B3389391481}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.HttpApi", "web\JiShe.CollectBus.HttpApi\JiShe.CollectBus.HttpApi.csproj", "{077AA5F8-8B61-420C-A6B5-0150A66FDB34}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Host", "web\JiShe.CollectBus.Host\JiShe.CollectBus.Host.csproj", "{35829A15-4127-4F69-8BDE-9405DEAACA9A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Common", "shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj", "{AD2F1928-4411-4511-B564-5FB996EC08B9}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.DbMigrator", "services\JiShe.CollectBus.DbMigrator\JiShe.CollectBus.DbMigrator.csproj", "{8BA01C3D-297D-42DF-BD63-EF07202A0A67}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.FreeSql", "modules\JiShe.CollectBus.FreeSql\JiShe.CollectBus.FreeSql.csproj", "{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.FreeRedis", "modules\JiShe.CollectBus.FreeRedis\JiShe.CollectBus.FreeRedis.csproj", "{C06C4082-638F-2996-5FED-7784475766C1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Kafka", "modules\JiShe.CollectBus.Kafka\JiShe.CollectBus.Kafka.csproj", "{F0288175-F0EC-48BD-945F-CF1512850943}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.IoTDB", "modules\JiShe.CollectBus.IoTDB\JiShe.CollectBus.IoTDB.csproj", "{A3F3C092-0A25-450B-BF6A-5983163CBEF5}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Protocol.Test", "protocols\JiShe.CollectBus.Protocol.Test\JiShe.CollectBus.Protocol.Test.csproj", "{A377955E-7EA1-6F29-8CF7-774569E93925}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Cassandra", "modules\JiShe.CollectBus.Cassandra\JiShe.CollectBus.Cassandra.csproj", "{443B4549-0AC0-4493-8F3E-49C83225DD76}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "1.Web", "1.Web", "{A02F7D8A-04DC-44D6-94D4-3F65712D6B94}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "4.Modules", "4.Modules", "{2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "6.Protocols", "6.Protocols", "{3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "2.Services", "2.Services", "{BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "3.Shared", "3.Shared", "{EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Kafka.Test", "modules\JiShe.CollectBus.Kafka.Test\JiShe.CollectBus.Kafka.Test.csproj", "{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T1882018", "protocols\JiShe.CollectBus.Protocol.T1882018\JiShe.CollectBus.Protocol.T1882018.csproj", "{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T37612012", "protocols\JiShe.CollectBus.Protocol.T37612012\JiShe.CollectBus.Protocol.T37612012.csproj", "{8A61DF78-069B-40B5-8811-614E2960443E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol", "protocols\JiShe.CollectBus.Protocol\JiShe.CollectBus.Protocol.csproj", "{E27377CC-E2D3-4237-060F-96EA214D3129}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T6452007", "protocols\JiShe.CollectBus.Protocol.T6452007\JiShe.CollectBus.Protocol.T6452007.csproj", "{75B7D419-C261-577D-58D6-AA3ACED9129F}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "0.Docs", "0.Docs", "{D8346C4C-55B8-43E8-A6B8-E59D56FE6D92}"
ProjectSection(SolutionItems) = preProject
.gitignore = .gitignore
Dockerfile = Dockerfile
NuGet.Config = NuGet.Config
readme.md = readme.md
EndProjectSection
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{D64C1577-4929-4B60-939E-96DE1534891A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D64C1577-4929-4B60-939E-96DE1534891A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D64C1577-4929-4B60-939E-96DE1534891A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D64C1577-4929-4B60-939E-96DE1534891A}.Release|Any CPU.Build.0 = Release|Any CPU
{F2840BC7-0188-4606-9126-DADD0F5ABF7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F2840BC7-0188-4606-9126-DADD0F5ABF7A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F2840BC7-0188-4606-9126-DADD0F5ABF7A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F2840BC7-0188-4606-9126-DADD0F5ABF7A}.Release|Any CPU.Build.0 = Release|Any CPU
{BD65D04F-08D5-40C1-8C24-77CA0BACB877}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BD65D04F-08D5-40C1-8C24-77CA0BACB877}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BD65D04F-08D5-40C1-8C24-77CA0BACB877}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BD65D04F-08D5-40C1-8C24-77CA0BACB877}.Release|Any CPU.Build.0 = Release|Any CPU
{78040F9E-3501-4A40-82DF-00A597710F35}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{78040F9E-3501-4A40-82DF-00A597710F35}.Debug|Any CPU.Build.0 = Debug|Any CPU
{78040F9E-3501-4A40-82DF-00A597710F35}.Release|Any CPU.ActiveCfg = Release|Any CPU
{78040F9E-3501-4A40-82DF-00A597710F35}.Release|Any CPU.Build.0 = Release|Any CPU
{F1C58097-4C08-4D88-8976-6B3389391481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F1C58097-4C08-4D88-8976-6B3389391481}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F1C58097-4C08-4D88-8976-6B3389391481}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F1C58097-4C08-4D88-8976-6B3389391481}.Release|Any CPU.Build.0 = Release|Any CPU
{077AA5F8-8B61-420C-A6B5-0150A66FDB34}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{077AA5F8-8B61-420C-A6B5-0150A66FDB34}.Debug|Any CPU.Build.0 = Debug|Any CPU
{077AA5F8-8B61-420C-A6B5-0150A66FDB34}.Release|Any CPU.ActiveCfg = Release|Any CPU
{077AA5F8-8B61-420C-A6B5-0150A66FDB34}.Release|Any CPU.Build.0 = Release|Any CPU
{35829A15-4127-4F69-8BDE-9405DEAACA9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{35829A15-4127-4F69-8BDE-9405DEAACA9A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{35829A15-4127-4F69-8BDE-9405DEAACA9A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{35829A15-4127-4F69-8BDE-9405DEAACA9A}.Release|Any CPU.Build.0 = Release|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Release|Any CPU.Build.0 = Release|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Release|Any CPU.Build.0 = Release|Any CPU
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}.Release|Any CPU.Build.0 = Release|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C06C4082-638F-2996-5FED-7784475766C1}.Release|Any CPU.Build.0 = Release|Any CPU
{F0288175-F0EC-48BD-945F-CF1512850943}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F0288175-F0EC-48BD-945F-CF1512850943}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F0288175-F0EC-48BD-945F-CF1512850943}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F0288175-F0EC-48BD-945F-CF1512850943}.Release|Any CPU.Build.0 = Release|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A3F3C092-0A25-450B-BF6A-5983163CBEF5}.Release|Any CPU.Build.0 = Release|Any CPU
{A377955E-7EA1-6F29-8CF7-774569E93925}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A377955E-7EA1-6F29-8CF7-774569E93925}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A377955E-7EA1-6F29-8CF7-774569E93925}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A377955E-7EA1-6F29-8CF7-774569E93925}.Release|Any CPU.Build.0 = Release|Any CPU
{443B4549-0AC0-4493-8F3E-49C83225DD76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{443B4549-0AC0-4493-8F3E-49C83225DD76}.Debug|Any CPU.Build.0 = Debug|Any CPU
{443B4549-0AC0-4493-8F3E-49C83225DD76}.Release|Any CPU.ActiveCfg = Release|Any CPU
{443B4549-0AC0-4493-8F3E-49C83225DD76}.Release|Any CPU.Build.0 = Release|Any CPU
{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B}.Release|Any CPU.Build.0 = Release|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Release|Any CPU.Build.0 = Release|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Release|Any CPU.Build.0 = Release|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Release|Any CPU.Build.0 = Release|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{D64C1577-4929-4B60-939E-96DE1534891A} = {EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}
{F2840BC7-0188-4606-9126-DADD0F5ABF7A} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{BD65D04F-08D5-40C1-8C24-77CA0BACB877} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{78040F9E-3501-4A40-82DF-00A597710F35} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{F1C58097-4C08-4D88-8976-6B3389391481} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{077AA5F8-8B61-420C-A6B5-0150A66FDB34} = {A02F7D8A-04DC-44D6-94D4-3F65712D6B94}
{35829A15-4127-4F69-8BDE-9405DEAACA9A} = {A02F7D8A-04DC-44D6-94D4-3F65712D6B94}
{AD2F1928-4411-4511-B564-5FB996EC08B9} = {EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}
{8BA01C3D-297D-42DF-BD63-EF07202A0A67} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{C06C4082-638F-2996-5FED-7784475766C1} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{F0288175-F0EC-48BD-945F-CF1512850943} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{A3F3C092-0A25-450B-BF6A-5983163CBEF5} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{A377955E-7EA1-6F29-8CF7-774569E93925} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{443B4549-0AC0-4493-8F3E-49C83225DD76} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{8A61DF78-069B-40B5-8811-614E2960443E} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{E27377CC-E2D3-4237-060F-96EA214D3129} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{75B7D419-C261-577D-58D6-AA3ACED9129F} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {4324B3B4-B60B-4E3C-91D8-59576B4E26DD}
EndGlobalSection
EndGlobal

View File

@ -19,10 +19,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Host", "we
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Common", "shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj", "{AD2F1928-4411-4511-B564-5FB996EC08B9}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Protocol", "protocols\JiShe.CollectBus.Protocol\JiShe.CollectBus.Protocol.csproj", "{C62EFF95-5C32-435F-BD78-6977E828F894}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.Protocol.Contracts", "protocols\JiShe.CollectBus.Protocol.Contracts\JiShe.CollectBus.Protocol.Contracts.csproj", "{38C1808B-009A-418B-B17B-AB3626341B5D}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.DbMigrator", "services\JiShe.CollectBus.DbMigrator\JiShe.CollectBus.DbMigrator.csproj", "{8BA01C3D-297D-42DF-BD63-EF07202A0A67}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "JiShe.CollectBus.FreeSql", "modules\JiShe.CollectBus.FreeSql\JiShe.CollectBus.FreeSql.csproj", "{FE0457D9-4038-4A17-8808-DCAD06CFC0A0}"
@ -41,14 +37,43 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "1.Web", "1.Web", "{A02F7D8A
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "4.Modules", "4.Modules", "{2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "3.Protocols", "3.Protocols", "{3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}"
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "5.Protocols", "5.Protocols", "{3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "2.Services", "2.Services", "{BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "5.Shared", "5.Shared", "{EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}"
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "3.Shared", "3.Shared", "{EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Kafka.Test", "modules\JiShe.CollectBus.Kafka.Test\JiShe.CollectBus.Kafka.Test.csproj", "{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T1882018", "protocols\JiShe.CollectBus.Protocol.T1882018\JiShe.CollectBus.Protocol.T1882018.csproj", "{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T37612012", "protocols\JiShe.CollectBus.Protocol.T37612012\JiShe.CollectBus.Protocol.T37612012.csproj", "{8A61DF78-069B-40B5-8811-614E2960443E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol", "protocols\JiShe.CollectBus.Protocol\JiShe.CollectBus.Protocol.csproj", "{E27377CC-E2D3-4237-060F-96EA214D3129}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Protocol.T6452007", "protocols\JiShe.CollectBus.Protocol.T6452007\JiShe.CollectBus.Protocol.T6452007.csproj", "{75B7D419-C261-577D-58D6-AA3ACED9129F}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "0.Docs", "0.Docs", "{D8346C4C-55B8-43E8-A6B8-E59D56FE6D92}"
ProjectSection(SolutionItems) = preProject
.gitignore = .gitignore
Dockerfile = Dockerfile
NuGet.Config = NuGet.Config
PackageAndPublish.bat = PackageAndPublish.bat
readme.md = readme.md
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Analyzers.Shared", "shared\JiShe.CollectBus.Analyzers.Shared\JiShe.CollectBus.Analyzers.Shared.csproj", "{DD68F314-BC66-5601-B094-B1A7BE93F4E0}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Analyzers", "modules\JiShe.CollectBus.Analyzers\JiShe.CollectBus.Analyzers.csproj", "{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Migration.Application.Contracts", "services\JiShe.CollectBus.Migration.Application.Contracts\JiShe.CollectBus.Migration.Application.Contracts.csproj", "{E01625B5-B5B7-7A4B-468F-EC12C1BDBA2A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Migration.Application", "services\JiShe.CollectBus.Migration.Application\JiShe.CollectBus.Migration.Application.csproj", "{B955C5DA-3C20-35D2-0770-8FE473C41C44}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Migration.Host", "web\JiShe.CollectBus.Migration.Host\JiShe.CollectBus.Migration.Host.csproj", "{995D3D91-7221-D4A3-A7B2-FEC202328A18}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JiShe.CollectBus.Migration.HttpApi", "web\JiShe.CollectBus.Migration.HttpApi\JiShe.CollectBus.Migration.HttpApi.csproj", "{8A113DE5-7D50-6E6B-739F-B6EEAD5E13B4}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -87,14 +112,6 @@ Global
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{AD2F1928-4411-4511-B564-5FB996EC08B9}.Release|Any CPU.Build.0 = Release|Any CPU
{C62EFF95-5C32-435F-BD78-6977E828F894}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C62EFF95-5C32-435F-BD78-6977E828F894}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C62EFF95-5C32-435F-BD78-6977E828F894}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C62EFF95-5C32-435F-BD78-6977E828F894}.Release|Any CPU.Build.0 = Release|Any CPU
{38C1808B-009A-418B-B17B-AB3626341B5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{38C1808B-009A-418B-B17B-AB3626341B5D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{38C1808B-009A-418B-B17B-AB3626341B5D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{38C1808B-009A-418B-B17B-AB3626341B5D}.Release|Any CPU.Build.0 = Release|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8BA01C3D-297D-42DF-BD63-EF07202A0A67}.Release|Any CPU.ActiveCfg = Release|Any CPU
@ -127,6 +144,46 @@ Global
{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B}.Release|Any CPU.Build.0 = Release|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F}.Release|Any CPU.Build.0 = Release|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8A61DF78-069B-40B5-8811-614E2960443E}.Release|Any CPU.Build.0 = Release|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E27377CC-E2D3-4237-060F-96EA214D3129}.Release|Any CPU.Build.0 = Release|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{75B7D419-C261-577D-58D6-AA3ACED9129F}.Release|Any CPU.Build.0 = Release|Any CPU
{DD68F314-BC66-5601-B094-B1A7BE93F4E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DD68F314-BC66-5601-B094-B1A7BE93F4E0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DD68F314-BC66-5601-B094-B1A7BE93F4E0}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DD68F314-BC66-5601-B094-B1A7BE93F4E0}.Release|Any CPU.Build.0 = Release|Any CPU
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A}.Release|Any CPU.Build.0 = Release|Any CPU
{E01625B5-B5B7-7A4B-468F-EC12C1BDBA2A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E01625B5-B5B7-7A4B-468F-EC12C1BDBA2A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E01625B5-B5B7-7A4B-468F-EC12C1BDBA2A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E01625B5-B5B7-7A4B-468F-EC12C1BDBA2A}.Release|Any CPU.Build.0 = Release|Any CPU
{B955C5DA-3C20-35D2-0770-8FE473C41C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B955C5DA-3C20-35D2-0770-8FE473C41C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B955C5DA-3C20-35D2-0770-8FE473C41C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B955C5DA-3C20-35D2-0770-8FE473C41C44}.Release|Any CPU.Build.0 = Release|Any CPU
{995D3D91-7221-D4A3-A7B2-FEC202328A18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{995D3D91-7221-D4A3-A7B2-FEC202328A18}.Debug|Any CPU.Build.0 = Debug|Any CPU
{995D3D91-7221-D4A3-A7B2-FEC202328A18}.Release|Any CPU.ActiveCfg = Release|Any CPU
{995D3D91-7221-D4A3-A7B2-FEC202328A18}.Release|Any CPU.Build.0 = Release|Any CPU
{8A113DE5-7D50-6E6B-739F-B6EEAD5E13B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8A113DE5-7D50-6E6B-739F-B6EEAD5E13B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8A113DE5-7D50-6E6B-739F-B6EEAD5E13B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8A113DE5-7D50-6E6B-739F-B6EEAD5E13B4}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -140,8 +197,6 @@ Global
{077AA5F8-8B61-420C-A6B5-0150A66FDB34} = {A02F7D8A-04DC-44D6-94D4-3F65712D6B94}
{35829A15-4127-4F69-8BDE-9405DEAACA9A} = {A02F7D8A-04DC-44D6-94D4-3F65712D6B94}
{AD2F1928-4411-4511-B564-5FB996EC08B9} = {EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}
{C62EFF95-5C32-435F-BD78-6977E828F894} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{38C1808B-009A-418B-B17B-AB3626341B5D} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{8BA01C3D-297D-42DF-BD63-EF07202A0A67} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{FE0457D9-4038-4A17-8808-DCAD06CFC0A0} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{C06C4082-638F-2996-5FED-7784475766C1} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
@ -150,6 +205,16 @@ Global
{A377955E-7EA1-6F29-8CF7-774569E93925} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{443B4549-0AC0-4493-8F3E-49C83225DD76} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{6D6A2A58-7406-9C8C-7B23-3E442CCE3E6B} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{430D298B-377E-49B8-83AA-ADC7C0EBDB0F} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{8A61DF78-069B-40B5-8811-614E2960443E} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{E27377CC-E2D3-4237-060F-96EA214D3129} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{75B7D419-C261-577D-58D6-AA3ACED9129F} = {3C3F9DB2-EC97-4464-B49F-BF1A0C2B46DC}
{DD68F314-BC66-5601-B094-B1A7BE93F4E0} = {EBF7C01F-9B4F-48E6-8418-2CBFDA51EB0B}
{EB97C7BB-1E4A-CBA4-04C1-22DBF48A253A} = {2E0FE301-34C3-4561-9CAE-C7A9E65AEE59}
{E01625B5-B5B7-7A4B-468F-EC12C1BDBA2A} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{B955C5DA-3C20-35D2-0770-8FE473C41C44} = {BA4DA3E7-9AD0-47AD-A0E6-A0BB6700DA23}
{995D3D91-7221-D4A3-A7B2-FEC202328A18} = {A02F7D8A-04DC-44D6-94D4-3F65712D6B94}
{8A113DE5-7D50-6E6B-739F-B6EEAD5E13B4} = {A02F7D8A-04DC-44D6-94D4-3F65712D6B94}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {4324B3B4-B60B-4E3C-91D8-59576B4E26DD}

7
NuGet.Config Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<add key="nuget" value="https://api.nuget.org/v3/index.json" />
</packageSources>
</configuration>

76
PackageAndPublish.bat Normal file
View File

@ -0,0 +1,76 @@
@echo off
setlocal enabledelayedexpansion
set VERSION=1.0.0
set CONFIGURATION=Release
set OUTPUT_DIR=%~dp0\nupkgs
set API_KEY=your-nuget-api-key
set SOURCE=https://api.nuget.org/v3/index.json
REM 创建输出目录
if not exist "%OUTPUT_DIR%" mkdir "%OUTPUT_DIR%"
REM 清理
echo 清理解决方案...
dotnet clean JiShe.CollectBus.sln -c %CONFIGURATION%
REM 删除之前的包
echo 删除之前的包...
if exist "%OUTPUT_DIR%\*.nupkg" del /q "%OUTPUT_DIR%\*.nupkg"
REM 打包项目
echo 开始打包项目...
REM 打包 Protocol 项目
echo 打包 Protocol 项目...
call :
protocols\JiShe.CollectBus.Protocol\JiShe.CollectBus.Protocol.csproj
call :PackProject protocols\JiShe.CollectBus.Protocol.Contracts\JiShe.CollectBus.Protocol.Contracts.csproj
call :PackProject protocols\JiShe.CollectBus.Protocol.T37612012\JiShe.CollectBus.Protocol.T37612012.csproj
REM 打包 Modules 项目
echo 打包 Modules 项目...
call :PackProject modules\JiShe.CollectBus.FreeRedis\JiShe.CollectBus.FreeRedis.csproj
call :PackProject modules\JiShe.CollectBus.Kafka\JiShe.CollectBus.Kafka.csproj
call :PackProject modules\JiShe.CollectBus.IoTDB\JiShe.CollectBus.IoTDB.csproj
call :PackProject modules\JiShe.CollectBus.MongoDB\JiShe.CollectBus.MongoDB.csproj
call :PackProject modules\JiShe.CollectBus.FreeSql\JiShe.CollectBus.FreeSql.csproj
call :PackProject modules\JiShe.CollectBus.Cassandra\JiShe.CollectBus.Cassandra.csproj
REM 打包 Shared 项目
echo 打包 Shared 项目...
call :PackProject shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj
call :PackProject shared\JiShe.CollectBus.Domain.Shared\JiShe.CollectBus.Domain.Shared.csproj
REM 打包 Services 项目
@REM echo 打包 Services 项目...
@REM call :PackProject services\JiShe.CollectBus.Domain\JiShe.CollectBus.Domain.csproj
@REM call :PackProject services\JiShe.CollectBus.Application.Contracts\JiShe.CollectBus.Application.Contracts.csproj
@REM call :PackProject services\JiShe.CollectBus.Application\JiShe.CollectBus.Application.csproj
@REM call :PackProject services\JiShe.CollectBus.EntityFrameworkCore\JiShe.CollectBus.EntityFrameworkCore.csproj
echo.
echo 是否要发布包到 NuGet? (Y/N)
set /p PUBLISH_CHOICE=
if /i "%PUBLISH_CHOICE%"=="Y" (
echo 开始发布包...
for %%f in ("%OUTPUT_DIR%\*.nupkg") do (
echo 发布: %%f
dotnet nuget push "%%f" --api-key %API_KEY% --source %SOURCE% --skip-duplicate
)
echo 所有包已发布完成!
) else (
echo 跳过发布操作。所有包都在 %OUTPUT_DIR% 目录中。
)
goto :eof
:PackProject
if exist "%~1" (
echo 打包: %~1
dotnet pack "%~1" -c %CONFIGURATION% --include-symbols -p:SymbolPackageFormat=snupkg -p:PackageVersion=%VERSION% -o "%OUTPUT_DIR%"
) else (
echo 警告: 项目不存在 - %~1
)
goto :eof

52
Temp.Dockerfile Normal file
View File

@ -0,0 +1,52 @@
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
WORKDIR /app
EXPOSE 80
EXPOSE 443
ENV TZ=Asia/Shanghai
ENV ASPNETCORE_ENVIRONMENT=Production
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
COPY ["publish/", "/app/publish"]
# # WORKDIR /src
# COPY ["JiShe.CollectBus.Main.sln", "."]
# COPY ["common.props", "."]
# COPY ["NuGet.Config", "."]
# COPY ["web/", "web/"]
# COPY ["modules/", "modules/"]
# COPY ["services/", "services/"]
# COPY ["shared/", "shared/"]
# COPY ["protocols/", "protocols/"]
# # 恢复项目依赖
# RUN dotnet restore "JiShe.CollectBus.Main.sln"
# # 构建项目
# WORKDIR "/web/JiShe.CollectBus.Host"
# RUN dotnet build "JiShe.CollectBus.Host.csproj" -c Release -o /app/build
# 发布项目
FROM build AS publish
# RUN dotnet publish "JiShe.CollectBus.Host.csproj" -c Release -o /app/publish /p:UseAppHost=false
# 创建最终镜像
FROM base AS final
WORKDIR /app
# 创建Plugins目录
RUN mkdir -p /app/Plugins
# 复制发布内容
COPY --from=publish /app/publish .
# 设置入口点
ENTRYPOINT ["dotnet", "JiShe.CollectBus.Host.dll"]
# 启动命令
# 可选:添加命令行参数
# CMD ["--urls", "http://+:80"]

View File

@ -0,0 +1,287 @@
using System;
using System.Collections.Generic;
namespace JiShe.CollectBus.PluginFileWatcher
{
/// <summary>
/// 文件监控程序的配置类
/// </summary>
public class FileMonitorConfig
{
/// <summary>
/// 基本配置
/// </summary>
public GeneralConfig General { get; set; } = new GeneralConfig();
/// <summary>
/// 文件过滤配置
/// </summary>
public FileFiltersConfig FileFilters { get; set; } = new FileFiltersConfig();
/// <summary>
/// 性能相关配置
/// </summary>
public PerformanceConfig Performance { get; set; } = new PerformanceConfig();
/// <summary>
/// 健壮性相关配置
/// </summary>
public RobustnessConfig Robustness { get; set; } = new RobustnessConfig();
/// <summary>
/// 事件存储和回放配置
/// </summary>
public EventStorageConfig EventStorage { get; set; } = new EventStorageConfig();
/// <summary>
/// 文件系统通知过滤器配置
/// </summary>
public List<string> NotifyFilters { get; set; } = new List<string>();
/// <summary>
/// 日志配置
/// </summary>
public LoggingConfig Logging { get; set; } = new LoggingConfig();
}
/// <summary>
/// 常规配置
/// </summary>
public class GeneralConfig
{
/// <summary>
/// 是否启用文件过滤
/// </summary>
public bool EnableFileFiltering { get; set; } = true;
/// <summary>
/// 内存监控间隔(分钟)
/// </summary>
public int MemoryMonitorIntervalMinutes { get; set; } = 1;
/// <summary>
/// 默认监控路径
/// </summary>
public string DefaultMonitorPath { get; set; } = string.Empty;
}
/// <summary>
/// 文件过滤配置
/// </summary>
public class FileFiltersConfig
{
/// <summary>
/// 允许监控的文件扩展名
/// </summary>
public string[] AllowedExtensions { get; set; } = new[] { ".dll" };
/// <summary>
/// 排除的目录
/// </summary>
public string[] ExcludedDirectories { get; set; } = new[] { "bin", "obj", "node_modules" };
/// <summary>
/// 是否包含子目录
/// </summary>
public bool IncludeSubdirectories { get; set; } = true;
}
/// <summary>
/// 性能相关配置
/// </summary>
public class PerformanceConfig
{
/// <summary>
/// 内存清理阈值(事件数)
/// </summary>
public int MemoryCleanupThreshold { get; set; } = 5000;
/// <summary>
/// 通道容量
/// </summary>
public int ChannelCapacity { get; set; } = 1000;
/// <summary>
/// 事件去抖时间(秒)
/// </summary>
public int EventDebounceTimeSeconds { get; set; } = 3;
/// <summary>
/// 最大字典大小
/// </summary>
public int MaxDictionarySize { get; set; } = 10000;
/// <summary>
/// 清理间隔(秒)
/// </summary>
public int CleanupIntervalSeconds { get; set; } = 5;
/// <summary>
/// 处理延迟(毫秒)
/// </summary>
public int ProcessingDelayMs { get; set; } = 5;
}
/// <summary>
/// 健壮性相关配置
/// </summary>
public class RobustnessConfig
{
/// <summary>
/// 是否启用自动恢复机制
/// </summary>
public bool EnableAutoRecovery { get; set; } = true;
/// <summary>
/// 监控器健康检查间隔(秒)
/// </summary>
public int WatcherHealthCheckIntervalSeconds { get; set; } = 30;
/// <summary>
/// 监控器无响应超时时间(秒)
/// </summary>
public int WatcherTimeoutSeconds { get; set; } = 60;
/// <summary>
/// 监控器重启尝试最大次数
/// </summary>
public int MaxRestartAttempts { get; set; } = 3;
/// <summary>
/// 重启尝试之间的延迟(秒)
/// </summary>
public int RestartDelaySeconds { get; set; } = 5;
/// <summary>
/// 是否启用文件锁检测
/// </summary>
public bool EnableFileLockDetection { get; set; } = true;
/// <summary>
/// 对锁定文件的处理策略: Skip(跳过), Retry(重试), Log(仅记录)
/// </summary>
public string LockedFileStrategy { get; set; } = "Retry";
/// <summary>
/// 文件锁定重试次数
/// </summary>
public int FileLockRetryCount { get; set; } = 3;
/// <summary>
/// 文件锁定重试间隔(毫秒)
/// </summary>
public int FileLockRetryDelayMs { get; set; } = 500;
}
/// <summary>
/// 事件存储和回放配置
/// </summary>
public class EventStorageConfig
{
/// <summary>
/// 是否启用事件存储
/// </summary>
public bool EnableEventStorage { get; set; } = true;
/// <summary>
/// 存储类型SQLite 或 File
/// </summary>
public string StorageType { get; set; } = "SQLite";
/// <summary>
/// 事件存储目录
/// </summary>
public string StorageDirectory { get; set; } = "D:/EventLogs";
/// <summary>
/// SQLite数据库文件路径
/// </summary>
public string DatabasePath { get; set; } = "D:/EventLogs/events.db";
/// <summary>
/// SQLite连接字符串
/// </summary>
public string ConnectionString { get; set; } = "Data Source=D:/EventLogs/events.db";
/// <summary>
/// 数据库命令超时(秒)
/// </summary>
public int CommandTimeout { get; set; } = 30;
/// <summary>
/// 事件日志文件名格式 (使用DateTime.ToString格式)
/// </summary>
public string LogFileNameFormat { get; set; } = "FileEvents_{0:yyyy-MM-dd}.json";
/// <summary>
/// 存储间隔(秒),多久将缓存的事件写入一次存储
/// </summary>
public int StorageIntervalSeconds { get; set; } = 60;
/// <summary>
/// 事件批量写入大小,达到此数量时立即写入存储
/// </summary>
public int BatchSize { get; set; } = 100;
/// <summary>
/// 最大保留事件记录条数
/// </summary>
public int MaxEventRecords { get; set; } = 100000;
/// <summary>
/// 数据保留天数
/// </summary>
public int DataRetentionDays { get; set; } = 30;
/// <summary>
/// 最大保留日志文件数
/// </summary>
public int MaxLogFiles { get; set; } = 30;
/// <summary>
/// 是否压缩存储的事件日志
/// </summary>
public bool CompressLogFiles { get; set; } = true;
/// <summary>
/// 是否可以回放事件
/// </summary>
public bool EnableEventReplay { get; set; } = true;
/// <summary>
/// 回放时间间隔(毫秒)
/// </summary>
public int ReplayIntervalMs { get; set; } = 100;
/// <summary>
/// 回放速度倍率大于1加速小于1减速
/// </summary>
public double ReplaySpeedFactor { get; set; } = 1.0;
}
/// <summary>
/// 日志相关配置
/// </summary>
public class LoggingConfig
{
/// <summary>
/// 日志级别Verbose、Debug、Information、Warning、Error、Fatal
/// </summary>
public string LogLevel { get; set; } = "Information";
/// <summary>
/// 是否记录文件事件处理详情
/// </summary>
public bool LogFileEventDetails { get; set; } = false;
/// <summary>
/// 日志文件保留天数
/// </summary>
public int RetainedLogDays { get; set; } = 30;
/// <summary>
/// 日志文件目录
/// </summary>
public string LogDirectory { get; set; } = "Logs";
}
}

View File

@ -0,0 +1,277 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Serilog;
using Serilog.Extensions.Logging;
using ILogger = Microsoft.Extensions.Logging.ILogger;
namespace JiShe.CollectBus.PluginFileWatcher
{
/// <summary>
/// 数据库操作工具类,用于命令行测试数据库功能
/// </summary>
public class DbUtility
{
private readonly EventDatabaseManager _dbManager;
private readonly ILogger _logger;
private readonly FileMonitorConfig _config;
/// <summary>
/// 初始化数据库工具类
/// </summary>
/// <param name="configPath">配置文件路径</param>
public DbUtility(string configPath = "appsettings.json")
{
// 从配置文件加载配置
var configuration = new ConfigurationBuilder()
.AddJsonFile(configPath, optional: false, reloadOnChange: true)
.Build();
// 初始化日志
var serilogLogger = new LoggerConfiguration()
.ReadFrom.Configuration(configuration)
.CreateLogger();
// 将Serilog适配为Microsoft.Extensions.Logging.ILogger
_logger = new SerilogLoggerFactory(serilogLogger).CreateLogger("DbUtility");
// 创建配置对象
_config = new FileMonitorConfig();
configuration.GetSection("FileMonitor").Bind(_config);
// 确保SQLite存储已启用
_config.EventStorage.EnableEventStorage = true;
_config.EventStorage.StorageType = "SQLite";
// 创建数据库管理器
_dbManager = new EventDatabaseManager(_config, _logger);
}
/// <summary>
/// 执行数据库维护操作
/// </summary>
/// <param name="args">命令行参数</param>
public async Task ExecuteAsync(string[] args)
{
if (args.Length == 0)
{
PrintUsage();
return;
}
string command = args[0].ToLower();
switch (command)
{
case "stats":
await ShowStatisticsAsync();
break;
case "cleanup":
int days = args.Length > 1 && int.TryParse(args[1], out int d) ? d : _config.EventStorage.DataRetentionDays;
await _dbManager.CleanupOldDataAsync(days);
Console.WriteLine($"已清理 {days} 天前的旧数据");
break;
case "query":
await QueryEventsAsync(args);
break;
case "test":
await GenerateTestDataAsync(args);
break;
default:
Console.WriteLine($"未知命令: {command}");
PrintUsage();
break;
}
}
/// <summary>
/// 显示帮助信息
/// </summary>
private void PrintUsage()
{
Console.WriteLine("数据库工具使用方法:");
Console.WriteLine(" stats - 显示数据库统计信息");
Console.WriteLine(" cleanup [days] - 清理指定天数之前的数据(默认使用配置值)");
Console.WriteLine(" query [limit] - 查询最近的事件默认10条");
Console.WriteLine(" query type:X ext:Y - 按类型和扩展名查询事件");
Console.WriteLine(" test [count] - 生成测试数据默认100条");
}
/// <summary>
/// 显示数据库统计信息
/// </summary>
private async Task ShowStatisticsAsync()
{
try
{
var stats = await _dbManager.GetDatabaseStatsAsync();
Console.WriteLine("数据库统计信息:");
Console.WriteLine($"事件总数: {stats.TotalEvents}");
Console.WriteLine($"最早事件时间: {stats.OldestEventTime?.ToLocalTime()}");
Console.WriteLine($"最新事件时间: {stats.NewestEventTime?.ToLocalTime()}");
Console.WriteLine("\n事件类型分布:");
if (stats.EventTypeCounts != null)
{
foreach (var item in stats.EventTypeCounts)
{
Console.WriteLine($" {item.Key}: {item.Value}");
}
}
Console.WriteLine("\n扩展名分布 (Top 10):");
if (stats.TopExtensions != null)
{
foreach (var item in stats.TopExtensions)
{
Console.WriteLine($" {item.Key}: {item.Value}");
}
}
}
catch (Exception ex)
{
Console.WriteLine($"获取统计信息出错: {ex.Message}");
_logger.LogError(ex, "获取数据库统计信息时发生错误");
}
}
/// <summary>
/// 查询事件
/// </summary>
private async Task QueryEventsAsync(string[] args)
{
try
{
var queryParams = new EventQueryParams
{
PageSize = 10,
PageIndex = 0
};
// 解析查询参数
if (args.Length > 1)
{
foreach (var arg in args.Skip(1))
{
if (int.TryParse(arg, out int limit))
{
queryParams.PageSize = limit;
}
else if (arg.StartsWith("type:", StringComparison.OrdinalIgnoreCase))
{
string typeValue = arg.Substring(5);
if (Enum.TryParse<FileEventType>(typeValue, true, out var eventType))
{
queryParams.EventType = eventType;
}
}
else if (arg.StartsWith("ext:", StringComparison.OrdinalIgnoreCase))
{
queryParams.ExtensionFilter = arg.Substring(4);
if (!queryParams.ExtensionFilter.StartsWith("."))
{
queryParams.ExtensionFilter = "." + queryParams.ExtensionFilter;
}
}
else if (arg.StartsWith("path:", StringComparison.OrdinalIgnoreCase))
{
queryParams.PathFilter = arg.Substring(5);
}
}
}
// 执行查询
var result = await _dbManager.QueryEventsAsync(queryParams);
Console.WriteLine($"查询结果 (总数: {result.TotalCount}):");
foreach (var evt in result.Events)
{
string typeStr = evt.EventType.ToString();
string timestamp = evt.Timestamp.ToLocalTime().ToString("yyyy-MM-dd HH:mm:ss");
Console.WriteLine($"[{timestamp}] {typeStr,-10} {evt.FileName} ({evt.Extension})");
}
if (result.HasMore)
{
Console.WriteLine($"... 还有更多结果,共 {result.TotalCount} 条");
}
}
catch (Exception ex)
{
Console.WriteLine($"查询事件出错: {ex.Message}");
_logger.LogError(ex, "查询事件时发生错误");
}
}
/// <summary>
/// 生成测试数据
/// </summary>
private async Task GenerateTestDataAsync(string[] args)
{
try
{
int count = args.Length > 1 && int.TryParse(args[1], out int c) ? c : 100;
var events = new List<FileEvent>();
var rnd = new Random();
string[] extensions = { ".dll", ".exe", ".txt", ".cs", ".xml", ".json", ".png", ".jpg" };
string[] directories = { "C:\\Temp", "D:\\Work", "C:\\Program Files", "D:\\Projects", "E:\\Data" };
DateTime startTime = DateTime.Now.AddHours(-24);
for (int i = 0; i < count; i++)
{
var eventType = (FileEventType)rnd.Next(0, 5);
var ext = extensions[rnd.Next(extensions.Length)];
var dir = directories[rnd.Next(directories.Length)];
var fileName = $"TestFile_{i:D5}{ext}";
var timestamp = startTime.AddMinutes(i);
var fileEvent = new FileEvent
{
Id = Guid.NewGuid(),
Timestamp = timestamp,
EventType = eventType,
FullPath = $"{dir}\\{fileName}",
FileName = fileName,
Directory = dir,
Extension = ext,
FileSize = rnd.Next(1024, 1024 * 1024)
};
// 如果是重命名事件,添加旧文件名
if (eventType == FileEventType.Renamed)
{
fileEvent.OldFileName = $"OldFile_{i:D5}{ext}";
fileEvent.OldFullPath = $"{dir}\\{fileEvent.OldFileName}";
}
// 添加一些元数据
fileEvent.Metadata["CreationTime"] = timestamp.AddMinutes(-rnd.Next(1, 60)).ToString("o");
fileEvent.Metadata["LastWriteTime"] = timestamp.ToString("o");
fileEvent.Metadata["IsReadOnly"] = (rnd.Next(10) < 2).ToString();
fileEvent.Metadata["TestData"] = $"测试数据 {i}";
events.Add(fileEvent);
}
Console.WriteLine($"正在生成 {count} 条测试数据...");
await _dbManager.SaveEventsAsync(events);
Console.WriteLine("测试数据生成完成!");
}
catch (Exception ex)
{
Console.WriteLine($"生成测试数据出错: {ex.Message}");
_logger.LogError(ex, "生成测试数据时发生错误");
}
}
}
}

View File

@ -0,0 +1,28 @@
# 请参阅 https://aka.ms/customizecontainer 以了解如何自定义调试容器,以及 Visual Studio 如何使用此 Dockerfile 生成映像以更快地进行调试。
# 此阶段用于在快速模式(默认为调试配置)下从 VS 运行时
FROM mcr.microsoft.com/dotnet/runtime:8.0 AS base
USER $APP_UID
WORKDIR /app
# 此阶段用于生成服务项目
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
ARG BUILD_CONFIGURATION=Release
WORKDIR /src
COPY ["external/JiShe.CollectBus.PluginFileWatcher/JiShe.CollectBus.PluginFileWatcher.csproj", "external/JiShe.CollectBus.PluginFileWatcher/"]
RUN dotnet restore "./external/JiShe.CollectBus.PluginFileWatcher/JiShe.CollectBus.PluginFileWatcher.csproj"
COPY . .
WORKDIR "/src/external/JiShe.CollectBus.PluginFileWatcher"
RUN dotnet build "./JiShe.CollectBus.PluginFileWatcher.csproj" -c $BUILD_CONFIGURATION -o /app/build
# 此阶段用于发布要复制到最终阶段的服务项目
FROM build AS publish
ARG BUILD_CONFIGURATION=Release
RUN dotnet publish "./JiShe.CollectBus.PluginFileWatcher.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
# 此阶段在生产中使用,或在常规模式下从 VS 运行时使用(在不使用调试配置时为默认值)
FROM base AS final
WORKDIR /app
COPY --from=publish /app/publish .
ENTRYPOINT ["dotnet", "JiShe.CollectBus.PluginFileWatcher.dll"]

View File

@ -0,0 +1,481 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging;
namespace JiShe.CollectBus.PluginFileWatcher
{
/// <summary>
/// SQLite数据库管理器用于管理文件事件的存储和检索
/// </summary>
public class EventDatabaseManager : IDisposable
{
private readonly FileMonitorConfig _config;
private readonly ILogger _logger;
private readonly string _connectionString;
private readonly string _databasePath;
private readonly int _commandTimeout;
private bool _disposed;
/// <summary>
/// 初始化数据库管理器
/// </summary>
/// <param name="config">配置对象</param>
/// <param name="logger">日志记录器</param>
public EventDatabaseManager(FileMonitorConfig config, ILogger logger)
{
_config = config ?? throw new ArgumentNullException(nameof(config));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
// 确保使用配置中的设置
_databasePath = config.EventStorage.DatabasePath;
_connectionString = config.EventStorage.ConnectionString;
_commandTimeout = config.EventStorage.CommandTimeout;
// 确保数据库目录存在
string dbDirectory = Path.GetDirectoryName(_databasePath);
if (!string.IsNullOrEmpty(dbDirectory) && !Directory.Exists(dbDirectory))
{
Directory.CreateDirectory(dbDirectory);
}
// 初始化数据库
InitializeDatabase().GetAwaiter().GetResult();
}
/// <summary>
/// 初始化数据库,确保必要的表已创建
/// </summary>
private async Task InitializeDatabase()
{
try
{
using var connection = new SqliteConnection(_connectionString);
await connection.OpenAsync();
// 启用外键约束
using (var command = connection.CreateCommand())
{
command.CommandText = "PRAGMA foreign_keys = ON;";
await command.ExecuteNonQueryAsync();
}
// 创建文件事件表
string createTableSql = @"
CREATE TABLE IF NOT EXISTS FileEvents (
Id TEXT PRIMARY KEY,
Timestamp TEXT NOT NULL,
EventType INTEGER NOT NULL,
FullPath TEXT NOT NULL,
FileName TEXT NOT NULL,
Directory TEXT NOT NULL,
Extension TEXT NOT NULL,
OldFileName TEXT,
OldFullPath TEXT,
FileSize INTEGER,
CreatedAt TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_events_timestamp ON FileEvents(Timestamp);
CREATE INDEX IF NOT EXISTS idx_events_eventtype ON FileEvents(EventType);
CREATE INDEX IF NOT EXISTS idx_events_extension ON FileEvents(Extension);";
await connection.ExecuteAsync(createTableSql, commandTimeout: _commandTimeout);
// 创建元数据表
string createMetadataTableSql = @"
CREATE TABLE IF NOT EXISTS EventMetadata (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
EventId TEXT NOT NULL,
MetadataKey TEXT NOT NULL,
MetadataValue TEXT,
FOREIGN KEY (EventId) REFERENCES FileEvents(Id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_metadata_eventid ON EventMetadata(EventId);
CREATE INDEX IF NOT EXISTS idx_metadata_key ON EventMetadata(MetadataKey);";
await connection.ExecuteAsync(createMetadataTableSql, commandTimeout: _commandTimeout);
_logger.LogInformation("数据库初始化成功");
}
catch (Exception ex)
{
_logger.LogError(ex, "初始化数据库失败");
throw;
}
}
/// <summary>
/// 保存文件事件到数据库
/// </summary>
/// <param name="events">要保存的事件列表</param>
public async Task SaveEventsAsync(List<FileEvent> events)
{
if (events == null || events.Count == 0)
return;
try
{
using var connection = new SqliteConnection(_connectionString);
await connection.OpenAsync();
// 启用外键约束
using (var command = connection.CreateCommand())
{
command.CommandText = "PRAGMA foreign_keys = ON;";
await command.ExecuteNonQueryAsync();
}
// 开始事务
using var transaction = connection.BeginTransaction();
try
{
foreach (var fileEvent in events)
{
// 插入事件数据
string insertEventSql = @"
INSERT INTO FileEvents (
Id, Timestamp, EventType, FullPath, FileName,
Directory, Extension, OldFileName, OldFullPath,
FileSize, CreatedAt
) VALUES (
@Id, @Timestamp, @EventType, @FullPath, @FileName,
@Directory, @Extension, @OldFileName, @OldFullPath,
@FileSize, @CreatedAt
)";
await connection.ExecuteAsync(insertEventSql, new
{
Id = fileEvent.Id.ToString(), // 确保ID始终以字符串形式保存
Timestamp = fileEvent.Timestamp.ToString("o"),
EventType = (int)fileEvent.EventType,
fileEvent.FullPath,
fileEvent.FileName,
fileEvent.Directory,
fileEvent.Extension,
fileEvent.OldFileName,
fileEvent.OldFullPath,
fileEvent.FileSize,
CreatedAt = DateTime.UtcNow.ToString("o")
}, transaction, _commandTimeout);
// 插入元数据
if (fileEvent.Metadata != null && fileEvent.Metadata.Count > 0)
{
string insertMetadataSql = @"
INSERT INTO EventMetadata (EventId, MetadataKey, MetadataValue)
VALUES (@EventId, @MetadataKey, @MetadataValue)";
foreach (var metadata in fileEvent.Metadata)
{
await connection.ExecuteAsync(insertMetadataSql, new
{
EventId = fileEvent.Id.ToString(), // 确保ID以相同格式保存
MetadataKey = metadata.Key,
MetadataValue = metadata.Value
}, transaction, _commandTimeout);
}
}
}
// 提交事务
transaction.Commit();
_logger.LogInformation($"已成功保存 {events.Count} 个事件到数据库");
}
catch (Exception ex)
{
// 回滚事务
transaction.Rollback();
_logger.LogError(ex, "保存事件到数据库时发生错误");
throw;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "连接数据库失败");
throw;
}
}
/// <summary>
/// 查询事件
/// </summary>
/// <param name="queryParams">查询参数</param>
/// <returns>查询结果</returns>
public async Task<EventQueryResult> QueryEventsAsync(EventQueryParams queryParams)
{
if (queryParams == null)
throw new ArgumentNullException(nameof(queryParams));
var result = new EventQueryResult
{
StartTime = queryParams.StartTime ?? DateTime.MinValue,
EndTime = queryParams.EndTime ?? DateTime.MaxValue
};
try
{
using var connection = new SqliteConnection(_connectionString);
await connection.OpenAsync();
// 启用外键约束
using (var command = connection.CreateCommand())
{
command.CommandText = "PRAGMA foreign_keys = ON;";
await command.ExecuteNonQueryAsync();
}
// 构建查询条件
var conditions = new List<string>();
var parameters = new DynamicParameters();
if (queryParams.StartTime.HasValue)
{
conditions.Add("Timestamp >= @StartTime");
parameters.Add("@StartTime", queryParams.StartTime.Value.ToString("o"));
}
if (queryParams.EndTime.HasValue)
{
conditions.Add("Timestamp <= @EndTime");
parameters.Add("@EndTime", queryParams.EndTime.Value.ToString("o"));
}
if (queryParams.EventType.HasValue)
{
conditions.Add("EventType = @EventType");
parameters.Add("@EventType", (int)queryParams.EventType.Value);
}
if (!string.IsNullOrEmpty(queryParams.PathFilter))
{
conditions.Add("FullPath LIKE @PathFilter");
parameters.Add("@PathFilter", $"%{queryParams.PathFilter}%");
}
if (!string.IsNullOrEmpty(queryParams.ExtensionFilter))
{
conditions.Add("Extension = @ExtensionFilter");
parameters.Add("@ExtensionFilter", queryParams.ExtensionFilter);
}
// 构建WHERE子句
string whereClause = conditions.Count > 0
? $"WHERE {string.Join(" AND ", conditions)}"
: string.Empty;
// 构建ORDER BY子句
string orderByClause = queryParams.AscendingOrder
? "ORDER BY Timestamp ASC"
: "ORDER BY Timestamp DESC";
// 获取总记录数
string countSql = $"SELECT COUNT(*) FROM FileEvents {whereClause}";
result.TotalCount = await connection.ExecuteScalarAsync<int>(countSql, parameters, commandTimeout: _commandTimeout);
// 应用分页
string paginationClause = $"LIMIT @PageSize OFFSET @Offset";
parameters.Add("@PageSize", queryParams.PageSize);
parameters.Add("@Offset", queryParams.PageIndex * queryParams.PageSize);
// 查询事件数据
string querySql = $@"
SELECT Id,
Timestamp,
EventType,
FullPath,
FileName,
Directory,
Extension,
OldFileName,
OldFullPath,
FileSize
FROM FileEvents
{whereClause}
{orderByClause}
{paginationClause}";
var events = await connection.QueryAsync<dynamic>(querySql, parameters, commandTimeout: _commandTimeout);
// 处理查询结果
foreach (var eventData in events)
{
var fileEvent = new FileEvent
{
Id = Guid.Parse(eventData.Id),
Timestamp = DateTime.Parse(eventData.Timestamp),
EventType = (FileEventType)eventData.EventType,
FullPath = eventData.FullPath,
FileName = eventData.FileName,
Directory = eventData.Directory,
Extension = eventData.Extension,
OldFileName = eventData.OldFileName,
OldFullPath = eventData.OldFullPath,
FileSize = eventData.FileSize
};
// 获取元数据
string metadataSql = "SELECT MetadataKey, MetadataValue FROM EventMetadata WHERE EventId = @EventId";
var metadata = await connection.QueryAsync<dynamic>(metadataSql, new { EventId = fileEvent.Id.ToString() }, commandTimeout: _commandTimeout);
foreach (var item in metadata)
{
fileEvent.Metadata[item.MetadataKey] = item.MetadataValue;
}
result.Events.Add(fileEvent);
}
result.HasMore = (queryParams.PageIndex + 1) * queryParams.PageSize < result.TotalCount;
return result;
}
catch (Exception ex)
{
_logger.LogError(ex, "查询事件时发生错误");
throw;
}
}
/// <summary>
/// 清理旧数据
/// </summary>
/// <param name="retentionDays">数据保留天数</param>
public async Task CleanupOldDataAsync(int retentionDays)
{
if (retentionDays <= 0)
return;
try
{
DateTime cutoffDate = DateTime.UtcNow.AddDays(-retentionDays);
string cutoffDateStr = cutoffDate.ToString("o");
using var connection = new SqliteConnection(_connectionString);
await connection.OpenAsync();
// 启用外键约束
using (var command = connection.CreateCommand())
{
command.CommandText = "PRAGMA foreign_keys = ON;";
await command.ExecuteNonQueryAsync();
}
// 删除旧事件(级联删除元数据)
string deleteSql = "DELETE FROM FileEvents WHERE Timestamp < @CutoffDate";
int deletedCount = await connection.ExecuteAsync(deleteSql, new { CutoffDate = cutoffDateStr }, commandTimeout: _commandTimeout);
_logger.LogInformation($"已清理 {deletedCount} 条旧事件数据({retentionDays}天前)");
}
catch (Exception ex)
{
_logger.LogError(ex, "清理旧数据时发生错误");
throw;
}
}
/// <summary>
/// 获取数据库统计信息
/// </summary>
/// <returns>数据库统计信息</returns>
public async Task<DatabaseStats> GetDatabaseStatsAsync()
{
try
{
using var connection = new SqliteConnection(_connectionString);
await connection.OpenAsync();
// 启用外键约束
using (var command = connection.CreateCommand())
{
command.CommandText = "PRAGMA foreign_keys = ON;";
await command.ExecuteNonQueryAsync();
}
var stats = new DatabaseStats();
// 获取事件总数
stats.TotalEvents = await connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM FileEvents", commandTimeout: _commandTimeout);
// 获取最早和最新事件时间
stats.OldestEventTime = await connection.ExecuteScalarAsync<DateTime?>("SELECT Timestamp FROM FileEvents ORDER BY Timestamp ASC LIMIT 1", commandTimeout: _commandTimeout);
stats.NewestEventTime = await connection.ExecuteScalarAsync<DateTime?>("SELECT Timestamp FROM FileEvents ORDER BY Timestamp DESC LIMIT 1", commandTimeout: _commandTimeout);
// 获取事件类型分布
var eventTypeCounts = await connection.QueryAsync<dynamic>("SELECT EventType, COUNT(*) AS Count FROM FileEvents GROUP BY EventType", commandTimeout: _commandTimeout);
stats.EventTypeCounts = new Dictionary<FileEventType, int>();
foreach (var item in eventTypeCounts)
{
stats.EventTypeCounts[(FileEventType)item.EventType] = item.Count;
}
// 获取扩展名分布前10个
var extensionCounts = await connection.QueryAsync<dynamic>(
"SELECT Extension, COUNT(*) AS Count FROM FileEvents GROUP BY Extension ORDER BY Count DESC LIMIT 10",
commandTimeout: _commandTimeout);
stats.TopExtensions = new Dictionary<string, int>();
foreach (var item in extensionCounts)
{
stats.TopExtensions[item.Extension] = item.Count;
}
return stats;
}
catch (Exception ex)
{
_logger.LogError(ex, "获取数据库统计信息时发生错误");
throw;
}
}
/// <summary>
/// 释放资源
/// </summary>
public void Dispose()
{
if (_disposed)
return;
_disposed = true;
}
}
/// <summary>
/// 数据库统计信息
/// </summary>
public class DatabaseStats
{
/// <summary>
/// 事件总数
/// </summary>
public int TotalEvents { get; set; }
/// <summary>
/// 最早事件时间
/// </summary>
public DateTime? OldestEventTime { get; set; }
/// <summary>
/// 最新事件时间
/// </summary>
public DateTime? NewestEventTime { get; set; }
/// <summary>
/// 事件类型计数
/// </summary>
public Dictionary<FileEventType, int> EventTypeCounts { get; set; }
/// <summary>
/// 排名前列的文件扩展名
/// </summary>
public Dictionary<string, int> TopExtensions { get; set; }
}
}

View File

@ -0,0 +1,745 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace JiShe.CollectBus.PluginFileWatcher
{
/// <summary>
/// 负责文件事件的存储、查询和回放
/// </summary>
public class EventStorage : IDisposable
{
private readonly FileMonitorConfig _config;
private readonly ILogger _logger;
private readonly ConcurrentQueue<FileEvent> _eventQueue;
private readonly Timer _storageTimer;
private readonly SemaphoreSlim _storageLock = new SemaphoreSlim(1, 1);
private readonly string _storageDirectory;
private readonly EventDatabaseManager _dbManager;
private bool _disposed;
/// <summary>
/// 创建新的事件存储管理器实例
/// </summary>
/// <param name="config">文件监控配置</param>
/// <param name="logger">日志记录器</param>
public EventStorage(FileMonitorConfig config, ILogger logger)
{
_config = config ?? throw new ArgumentNullException(nameof(config));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_eventQueue = new ConcurrentQueue<FileEvent>();
// 确保存储目录存在
_storageDirectory = !string.IsNullOrEmpty(_config.EventStorage.StorageDirectory)
? _config.EventStorage.StorageDirectory
: Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "EventLogs");
if (!Directory.Exists(_storageDirectory))
{
Directory.CreateDirectory(_storageDirectory);
}
// 创建数据库管理器如果配置为SQLite存储类型
if (config.EventStorage.EnableEventStorage &&
config.EventStorage.StorageType.Equals("SQLite", StringComparison.OrdinalIgnoreCase))
{
_dbManager = new EventDatabaseManager(config, logger);
_logger.LogInformation("已初始化SQLite事件存储");
}
// 初始化存储定时器(如果启用)
if (_config.EventStorage.EnableEventStorage)
{
var intervalMs = _config.EventStorage.StorageIntervalSeconds * 1000;
_storageTimer = new Timer(SaveEventsTimerCallback, null, intervalMs, intervalMs);
_logger.LogInformation($"事件存储已初始化,存储目录:{_storageDirectory},存储间隔:{_config.EventStorage.StorageIntervalSeconds}秒");
}
}
/// <summary>
/// 记录一个文件事件
/// </summary>
/// <param name="fileEvent">文件事件</param>
public void RecordEvent(FileEvent fileEvent)
{
if (fileEvent == null || !_config.EventStorage.EnableEventStorage) return;
_eventQueue.Enqueue(fileEvent);
_logger.LogDebug($"文件事件已加入队列:{fileEvent.EventType} - {fileEvent.FullPath}");
}
/// <summary>
/// 从FileSystemEventArgs记录事件
/// </summary>
/// <param name="e">文件系统事件参数</param>
public void RecordEvent(FileSystemEventArgs e)
{
if (e == null || !_config.EventStorage.EnableEventStorage) return;
var fileEvent = FileEvent.FromFileSystemEventArgs(e);
RecordEvent(fileEvent);
}
/// <summary>
/// 定时将事件保存到文件
/// </summary>
private async void SaveEventsTimerCallback(object state)
{
if (_disposed || _eventQueue.IsEmpty) return;
try
{
// 防止多个定时器回调同时执行
if (!await _storageLock.WaitAsync(0))
{
return;
}
try
{
// 从队列中取出事件
List<FileEvent> eventsToSave = new List<FileEvent>();
int batchSize = _config.EventStorage.BatchSize;
while (eventsToSave.Count < batchSize && !_eventQueue.IsEmpty)
{
if (_eventQueue.TryDequeue(out var fileEvent))
{
eventsToSave.Add(fileEvent);
}
}
if (eventsToSave.Count > 0)
{
await SaveEventsToFileAsync(eventsToSave);
_logger.LogInformation($"已成功保存 {eventsToSave.Count} 个事件");
}
// 如果有配置,清理旧日志文件
if (_config.EventStorage.MaxLogFiles > 0)
{
await CleanupOldLogFilesAsync();
}
// 如果有配置,清理旧数据库记录
if (_dbManager != null && _config.EventStorage.DataRetentionDays > 0)
{
await _dbManager.CleanupOldDataAsync(_config.EventStorage.DataRetentionDays);
}
}
finally
{
_storageLock.Release();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "保存事件时发生错误");
}
}
/// <summary>
/// 将事件保存到文件
/// </summary>
/// <param name="events">要保存的事件列表</param>
private async Task SaveEventsToFileAsync(List<FileEvent> events)
{
if (events == null || events.Count == 0) return;
try
{
// 根据存储类型选择保存方式
if (_config.EventStorage.StorageType.Equals("SQLite", StringComparison.OrdinalIgnoreCase) && _dbManager != null)
{
// 保存到SQLite数据库
await _dbManager.SaveEventsAsync(events);
}
else
{
// 保存到文件
string fileName = string.Format(
_config.EventStorage.LogFileNameFormat,
DateTime.Now);
string filePath = Path.Combine(_storageDirectory, fileName);
// 创建事件日志文件对象
var logFile = new EventLogFile
{
CreatedTime = DateTime.UtcNow,
Events = events
};
// 序列化为JSON
string jsonContent = JsonSerializer.Serialize(logFile, new JsonSerializerOptions
{
WriteIndented = true
});
// 是否启用压缩
if (_config.EventStorage.CompressLogFiles)
{
string gzFilePath = $"{filePath}.gz";
await CompressAndSaveStringAsync(jsonContent, gzFilePath);
_logger.LogInformation($"已将事件保存到压缩文件:{gzFilePath}");
}
else
{
await File.WriteAllTextAsync(filePath, jsonContent);
_logger.LogInformation($"已将事件保存到文件:{filePath}");
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "保存事件到文件时发生错误");
throw;
}
}
/// <summary>
/// 压缩并保存字符串到文件
/// </summary>
/// <param name="content">要保存的内容</param>
/// <param name="filePath">文件路径</param>
private static async Task CompressAndSaveStringAsync(string content, string filePath)
{
using var fileStream = new FileStream(filePath, FileMode.Create);
using var gzipStream = new GZipStream(fileStream, CompressionLevel.Optimal);
using var writer = new StreamWriter(gzipStream);
await writer.WriteAsync(content);
}
/// <summary>
/// 清理过多的日志文件
/// </summary>
private async Task CleanupOldLogFilesAsync()
{
try
{
// 检查是否需要清理
if (_config.EventStorage.MaxLogFiles <= 0) return;
var directory = new DirectoryInfo(_storageDirectory);
var logFiles = directory.GetFiles("*.*")
.Where(f => f.Name.EndsWith(".json") || f.Name.EndsWith(".gz"))
.OrderByDescending(f => f.CreationTime)
.ToArray();
// 如果文件数量超过最大值,删除最旧的文件
if (logFiles.Length > _config.EventStorage.MaxLogFiles)
{
int filesToDelete = logFiles.Length - _config.EventStorage.MaxLogFiles;
var filesToRemove = logFiles.Skip(logFiles.Length - filesToDelete).ToArray();
foreach (var file in filesToRemove)
{
try
{
file.Delete();
_logger.LogInformation($"已删除旧的事件日志文件:{file.Name}");
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"删除旧日志文件失败:{file.FullName}");
}
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "清理旧日志文件时发生错误");
}
await Task.CompletedTask;
}
/// <summary>
/// 查询历史事件
/// </summary>
/// <param name="queryParams">查询参数</param>
/// <returns>查询结果</returns>
public async Task<EventQueryResult> QueryEventsAsync(EventQueryParams queryParams)
{
if (queryParams == null) throw new ArgumentNullException(nameof(queryParams));
// 如果是SQLite存储且数据库管理器可用使用数据库查询
if (_config.EventStorage.StorageType.Equals("SQLite", StringComparison.OrdinalIgnoreCase) && _dbManager != null)
{
return await _dbManager.QueryEventsAsync(queryParams);
}
var result = new EventQueryResult
{
StartTime = queryParams.StartTime ?? DateTime.MinValue,
EndTime = queryParams.EndTime ?? DateTime.MaxValue
};
try
{
await _storageLock.WaitAsync();
try
{
// 获取所有日志文件
var directory = new DirectoryInfo(_storageDirectory);
if (!directory.Exists)
{
return result;
}
var logFiles = directory.GetFiles("*.*")
.Where(f => f.Name.EndsWith(".json") || f.Name.EndsWith(".gz"))
.OrderByDescending(f => f.CreationTime)
.ToArray();
List<FileEvent> allEvents = new List<FileEvent>();
// 加载所有日志文件中的事件
foreach (var file in logFiles)
{
try
{
var events = await LoadEventsFromFileAsync(file.FullName);
if (events != null && events.Count > 0)
{
allEvents.AddRange(events);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"从文件加载事件失败:{file.FullName}");
}
}
// 内存中队列的事件也包含在查询中
FileEvent[] queuedEvents = _eventQueue.ToArray();
allEvents.AddRange(queuedEvents);
// 应用查询过滤条件
var filteredEvents = allEvents
.Where(e => (queryParams.StartTime == null || e.Timestamp >= queryParams.StartTime) &&
(queryParams.EndTime == null || e.Timestamp <= queryParams.EndTime) &&
(queryParams.EventType == null || e.EventType == queryParams.EventType.Value) &&
(string.IsNullOrEmpty(queryParams.PathFilter) || e.FullPath.Contains(queryParams.PathFilter, StringComparison.OrdinalIgnoreCase)) &&
(string.IsNullOrEmpty(queryParams.ExtensionFilter) || e.Extension.Equals(queryParams.ExtensionFilter, StringComparison.OrdinalIgnoreCase)))
.ToList();
// 应用排序
IEnumerable<FileEvent> orderedEvents = queryParams.AscendingOrder
? filteredEvents.OrderBy(e => e.Timestamp)
: filteredEvents.OrderByDescending(e => e.Timestamp);
// 计算总数
result.TotalCount = filteredEvents.Count;
// 应用分页
int skip = queryParams.PageIndex * queryParams.PageSize;
int take = queryParams.PageSize;
result.Events = orderedEvents.Skip(skip).Take(take).ToList();
result.HasMore = (skip + take) < result.TotalCount;
return result;
}
finally
{
_storageLock.Release();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "查询事件时发生错误");
throw;
}
}
/// <summary>
/// 从文件加载事件
/// </summary>
/// <param name="filePath">文件路径</param>
/// <returns>事件列表</returns>
private async Task<List<FileEvent>> LoadEventsFromFileAsync(string filePath)
{
if (string.IsNullOrEmpty(filePath) || !File.Exists(filePath))
{
return new List<FileEvent>();
}
try
{
string jsonContent;
// 处理压缩文件
if (filePath.EndsWith(".gz"))
{
using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read);
using var gzipStream = new GZipStream(fileStream, CompressionMode.Decompress);
using var reader = new StreamReader(gzipStream);
jsonContent = await reader.ReadToEndAsync();
}
else
{
jsonContent = await File.ReadAllTextAsync(filePath);
}
var logFile = JsonSerializer.Deserialize<EventLogFile>(jsonContent);
return logFile?.Events ?? new List<FileEvent>();
}
catch (Exception ex)
{
_logger.LogError(ex, $"从文件加载事件失败:{filePath}");
return new List<FileEvent>();
}
}
/// <summary>
/// 启动事件回放会话
/// </summary>
/// <param name="queryParams">查询参数,定义要回放的事件</param>
/// <param name="replayHandler">回放处理回调</param>
/// <param name="cancellationToken">取消标记</param>
/// <returns>回放会话控制器</returns>
public async Task<EventReplaySession> StartReplayAsync(
EventQueryParams queryParams,
Func<FileEvent, Task> replayHandler,
CancellationToken cancellationToken = default)
{
if (replayHandler == null) throw new ArgumentNullException(nameof(replayHandler));
// 查询要回放的事件
var queryResult = await QueryEventsAsync(queryParams);
// 创建并启动回放会话
var session = new EventReplaySession(
queryResult.Events,
replayHandler,
_config.EventStorage.ReplayIntervalMs,
_config.EventStorage.ReplaySpeedFactor,
_logger,
cancellationToken);
await session.StartAsync();
return session;
}
/// <summary>
/// 释放资源
/// </summary>
public void Dispose()
{
if (_disposed) return;
_disposed = true;
_storageTimer?.Dispose();
_storageLock?.Dispose();
_dbManager?.Dispose();
// 尝试保存剩余事件
if (_config.EventStorage.EnableEventStorage && !_eventQueue.IsEmpty)
{
var remainingEvents = new List<FileEvent>();
while (!_eventQueue.IsEmpty && _eventQueue.TryDequeue(out var evt))
{
remainingEvents.Add(evt);
}
if (remainingEvents.Count > 0)
{
SaveEventsToFileAsync(remainingEvents).GetAwaiter().GetResult();
}
}
GC.SuppressFinalize(this);
}
}
/// <summary>
/// 事件回放会话
/// </summary>
public class EventReplaySession : IDisposable
{
private readonly List<FileEvent> _events;
private readonly Func<FileEvent, Task> _replayHandler;
private readonly int _replayIntervalMs;
private readonly double _speedFactor;
private readonly ILogger _logger;
private readonly CancellationToken _cancellationToken;
private CancellationTokenSource _linkedCts;
private Task _replayTask;
private bool _disposed;
private bool _isPaused;
private readonly SemaphoreSlim _pauseSemaphore = new SemaphoreSlim(1, 1);
/// <summary>
/// 回放进度0-100
/// </summary>
public int Progress { get; private set; }
/// <summary>
/// 当前回放的事件索引
/// </summary>
public int CurrentIndex { get; private set; }
/// <summary>
/// 事件总数
/// </summary>
public int TotalEvents => _events?.Count ?? 0;
/// <summary>
/// 回放是否已完成
/// </summary>
public bool IsCompleted { get; private set; }
/// <summary>
/// 回放是否已暂停
/// </summary>
public bool IsPaused => _isPaused;
/// <summary>
/// 回放已处理的事件数
/// </summary>
public int ProcessedEvents { get; private set; }
/// <summary>
/// 回放开始时间
/// </summary>
public DateTime StartTime { get; private set; }
/// <summary>
/// 回放结束时间(如果已完成)
/// </summary>
public DateTime? EndTime { get; private set; }
/// <summary>
/// 创建新的事件回放会话
/// </summary>
/// <param name="events">要回放的事件</param>
/// <param name="replayHandler">回放处理回调</param>
/// <param name="replayIntervalMs">回放间隔(毫秒)</param>
/// <param name="speedFactor">速度因子</param>
/// <param name="logger">日志记录器</param>
/// <param name="cancellationToken">取消标记</param>
public EventReplaySession(
List<FileEvent> events,
Func<FileEvent, Task> replayHandler,
int replayIntervalMs,
double speedFactor,
ILogger logger,
CancellationToken cancellationToken = default)
{
_events = events ?? throw new ArgumentNullException(nameof(events));
_replayHandler = replayHandler ?? throw new ArgumentNullException(nameof(replayHandler));
_replayIntervalMs = Math.Max(1, replayIntervalMs);
_speedFactor = Math.Max(0.1, speedFactor);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_cancellationToken = cancellationToken;
_linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
}
/// <summary>
/// 启动回放
/// </summary>
public async Task StartAsync()
{
if (_replayTask != null) return;
StartTime = DateTime.Now;
_replayTask = Task.Run(ReplayEventsAsync, _linkedCts.Token);
await Task.CompletedTask;
}
/// <summary>
/// 暂停回放
/// </summary>
public async Task PauseAsync()
{
if (_isPaused || IsCompleted) return;
await _pauseSemaphore.WaitAsync();
try
{
_isPaused = true;
}
finally
{
_pauseSemaphore.Release();
}
_logger.LogInformation("事件回放已暂停");
}
/// <summary>
/// 恢复回放
/// </summary>
public async Task ResumeAsync()
{
if (!_isPaused || IsCompleted) return;
await _pauseSemaphore.WaitAsync();
try
{
_isPaused = false;
// 释放信号量以允许回放任务继续
_pauseSemaphore.Release();
}
catch
{
_pauseSemaphore.Release();
throw;
}
_logger.LogInformation("事件回放已恢复");
}
/// <summary>
/// 停止回放
/// </summary>
public async Task StopAsync()
{
if (IsCompleted) return;
try
{
// 取消回放任务
_linkedCts?.Cancel();
// 如果暂停中,先恢复以允许取消
if (_isPaused)
{
await ResumeAsync();
}
// 等待任务完成
if (_replayTask != null)
{
await Task.WhenAny(_replayTask, Task.Delay(1000));
}
IsCompleted = true;
EndTime = DateTime.Now;
_logger.LogInformation("事件回放已手动停止");
}
catch (Exception ex)
{
_logger.LogError(ex, "停止事件回放时发生错误");
}
}
/// <summary>
/// 回放事件处理
/// </summary>
private async Task ReplayEventsAsync()
{
try
{
_logger.LogInformation($"开始回放{_events.Count}个事件,速度因子:{_speedFactor}");
if (_events.Count == 0)
{
IsCompleted = true;
EndTime = DateTime.Now;
return;
}
DateTime? lastEventTime = null;
for (int i = 0; i < _events.Count; i++)
{
// 检查是否取消
if (_linkedCts.Token.IsCancellationRequested)
{
_logger.LogInformation("事件回放已取消");
break;
}
// 检查暂停状态
if (_isPaused)
{
// 等待恢复信号
await _pauseSemaphore.WaitAsync(_linkedCts.Token);
_pauseSemaphore.Release();
}
var currentEvent = _events[i];
CurrentIndex = i;
// 计算等待时间(根据事件之间的实际时间差和速度因子)
if (lastEventTime.HasValue && i > 0)
{
var actualTimeDiff = currentEvent.Timestamp - lastEventTime.Value;
var waitTimeMs = (int)(actualTimeDiff.TotalMilliseconds / _speedFactor);
// 应用最小等待时间
waitTimeMs = Math.Max(_replayIntervalMs, waitTimeMs);
// 等待指定时间
await Task.Delay(waitTimeMs, _linkedCts.Token);
}
// 处理当前事件
try
{
await _replayHandler(currentEvent);
ProcessedEvents++;
// 更新进度
Progress = (int)((i + 1) * 100.0 / _events.Count);
}
catch (Exception ex)
{
_logger.LogError(ex, $"处理回放事件时发生错误:{currentEvent.EventType} - {currentEvent.FullPath}");
}
lastEventTime = currentEvent.Timestamp;
}
// 完成回放
IsCompleted = true;
Progress = 100;
EndTime = DateTime.Now;
_logger.LogInformation($"事件回放已完成,共处理{ProcessedEvents}个事件,耗时:{(EndTime.Value - StartTime).TotalSeconds:F2}秒");
}
catch (OperationCanceledException)
{
_logger.LogInformation("事件回放已取消");
IsCompleted = true;
EndTime = DateTime.Now;
}
catch (Exception ex)
{
_logger.LogError(ex, "事件回放过程中发生错误");
IsCompleted = true;
EndTime = DateTime.Now;
}
}
/// <summary>
/// 释放资源
/// </summary>
public void Dispose()
{
if (_disposed) return;
_disposed = true;
_linkedCts?.Cancel();
_linkedCts?.Dispose();
_pauseSemaphore?.Dispose();
GC.SuppressFinalize(this);
}
}
}

View File

@ -0,0 +1,254 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.Json.Serialization;
namespace JiShe.CollectBus.PluginFileWatcher
{
/// <summary>
/// 表示一个文件系统事件的数据模型,用于序列化和存储
/// </summary>
public class FileEvent
{
/// <summary>
/// 事件唯一标识
/// </summary>
public Guid Id { get; set; } = Guid.NewGuid();
/// <summary>
/// 事件发生时间
/// </summary>
public DateTime Timestamp { get; set; } = DateTime.UtcNow;
/// <summary>
/// 事件类型
/// </summary>
public FileEventType EventType { get; set; }
/// <summary>
/// 文件完整路径
/// </summary>
public string FullPath { get; set; }
/// <summary>
/// 文件名
/// </summary>
public string FileName { get; set; }
/// <summary>
/// 文件所在目录
/// </summary>
public string Directory { get; set; }
/// <summary>
/// 文件扩展名
/// </summary>
public string Extension { get; set; }
/// <summary>
/// 重命名前的旧文件名(仅在重命名事件中有效)
/// </summary>
public string OldFileName { get; set; }
/// <summary>
/// 重命名前的旧路径(仅在重命名事件中有效)
/// </summary>
public string OldFullPath { get; set; }
/// <summary>
/// 文件大小(字节),如果可获取
/// </summary>
public long? FileSize { get; set; }
/// <summary>
/// 自定义属性,可用于存储其他元数据
/// </summary>
public Dictionary<string, string> Metadata { get; set; } = new Dictionary<string, string>();
/// <summary>
/// 从FileSystemEventArgs创建FileEvent
/// </summary>
/// <param name="e">FileSystemEventArgs参数</param>
/// <returns>FileEvent对象</returns>
public static FileEvent FromFileSystemEventArgs(FileSystemEventArgs e)
{
var fileEvent = new FileEvent
{
EventType = GetEventType(e.ChangeType),
FullPath = e.FullPath,
FileName = e.Name ?? Path.GetFileName(e.FullPath),
Directory = Path.GetDirectoryName(e.FullPath),
Extension = Path.GetExtension(e.FullPath)
};
// 如果是重命名事件,添加旧文件名信息
if (e is RenamedEventArgs renamedEvent)
{
fileEvent.OldFileName = Path.GetFileName(renamedEvent.OldFullPath);
fileEvent.OldFullPath = renamedEvent.OldFullPath;
}
// 尝试获取文件大小(如果文件存在且可访问)
try
{
if (File.Exists(e.FullPath) && e.ChangeType != WatcherChangeTypes.Deleted)
{
var fileInfo = new FileInfo(e.FullPath);
fileEvent.FileSize = fileInfo.Length;
// 添加一些额外的元数据
fileEvent.Metadata["CreationTime"] = fileInfo.CreationTime.ToString("o");
fileEvent.Metadata["LastWriteTime"] = fileInfo.LastWriteTime.ToString("o");
fileEvent.Metadata["IsReadOnly"] = fileInfo.IsReadOnly.ToString();
}
}
catch
{
// 忽略任何获取文件信息时的错误
}
return fileEvent;
}
/// <summary>
/// 将WatcherChangeTypes转换为FileEventType
/// </summary>
/// <param name="changeType">WatcherChangeTypes枚举值</param>
/// <returns>对应的FileEventType</returns>
public static FileEventType GetEventType(WatcherChangeTypes changeType)
{
return changeType switch
{
WatcherChangeTypes.Created => FileEventType.Created,
WatcherChangeTypes.Deleted => FileEventType.Deleted,
WatcherChangeTypes.Changed => FileEventType.Modified,
WatcherChangeTypes.Renamed => FileEventType.Renamed,
_ => FileEventType.Other
};
}
}
/// <summary>
/// 文件事件类型
/// </summary>
public enum FileEventType
{
/// <summary>
/// 文件被创建
/// </summary>
Created,
/// <summary>
/// 文件被修改
/// </summary>
Modified,
/// <summary>
/// 文件被删除
/// </summary>
Deleted,
/// <summary>
/// 文件被重命名
/// </summary>
Renamed,
/// <summary>
/// 其他类型事件
/// </summary>
Other
}
/// <summary>
/// 表示一个事件日志文件
/// </summary>
public class EventLogFile
{
/// <summary>
/// 日志文件创建时间
/// </summary>
public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
/// <summary>
/// 日志文件包含的事件列表
/// </summary>
public List<FileEvent> Events { get; set; } = new List<FileEvent>();
}
/// <summary>
/// 事件查询结果
/// </summary>
public class EventQueryResult
{
/// <summary>
/// 查询到的事件列表
/// </summary>
public List<FileEvent> Events { get; set; } = new List<FileEvent>();
/// <summary>
/// 匹配的事件总数
/// </summary>
public int TotalCount { get; set; }
/// <summary>
/// 查询是否有更多结果
/// </summary>
public bool HasMore { get; set; }
/// <summary>
/// 查询时间范围的开始时间
/// </summary>
public DateTime StartTime { get; set; }
/// <summary>
/// 查询时间范围的结束时间
/// </summary>
public DateTime EndTime { get; set; }
}
/// <summary>
/// 事件查询参数
/// </summary>
public class EventQueryParams
{
/// <summary>
/// 查询开始时间
/// </summary>
public DateTime? StartTime { get; set; }
/// <summary>
/// 查询结束时间
/// </summary>
public DateTime? EndTime { get; set; }
/// <summary>
/// 事件类型过滤
/// </summary>
public FileEventType? EventType { get; set; }
/// <summary>
/// 文件路径过滤(支持包含关系)
/// </summary>
public string PathFilter { get; set; }
/// <summary>
/// 文件扩展名过滤
/// </summary>
public string ExtensionFilter { get; set; }
/// <summary>
/// 分页大小
/// </summary>
public int PageSize { get; set; } = 100;
/// <summary>
/// 分页索引从0开始
/// </summary>
public int PageIndex { get; set; } = 0;
/// <summary>
/// 排序方向true为升序false为降序
/// </summary>
public bool AscendingOrder { get; set; } = false;
}
}

View File

@ -0,0 +1,157 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace JiShe.CollectBus.PluginFileWatcher
{
/// <summary>
/// 文件监控相关工具类
/// </summary>
public static class FileWatcherUtils
{
/// <summary>
/// 检测文件是否被锁定
/// </summary>
/// <param name="filePath">要检查的文件路径</param>
/// <returns>如果文件被锁定则返回true否则返回false</returns>
public static bool IsFileLocked(string filePath)
{
if (string.IsNullOrEmpty(filePath) || !File.Exists(filePath))
return false;
try
{
using (FileStream stream = File.Open(filePath, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
{
// 文件可以被完全访问,没有被锁定
stream.Close();
}
return false;
}
catch (IOException)
{
// 文件被锁定或正在被其他进程使用
return true;
}
catch (Exception)
{
// 其他错误(权限不足、路径无效等)
return true;
}
}
/// <summary>
/// 尝试处理一个可能被锁定的文件
/// </summary>
/// <param name="filePath">文件路径</param>
/// <param name="action">成功解锁后要执行的操作</param>
/// <param name="config">健壮性配置</param>
/// <returns>处理结果true表示成功处理false表示处理失败</returns>
public static async Task<bool> TryHandleLockedFileAsync(string filePath, Func<Task> action, RobustnessConfig config)
{
if (!config.EnableFileLockDetection)
{
// 如果禁用了锁检测,直接执行操作
try
{
await action();
return true;
}
catch
{
return false;
}
}
// 如果文件不存在或不是锁定状态,直接执行操作
if (!File.Exists(filePath) || !IsFileLocked(filePath))
{
try
{
await action();
return true;
}
catch
{
return false;
}
}
// 文件被锁定,根据策略处理
switch (config.LockedFileStrategy.ToLower())
{
case "skip":
// 跳过这个文件
Console.WriteLine($"文件被锁定,已跳过: {filePath}");
return false;
case "retry":
// 重试几次
for (int i = 0; i < config.FileLockRetryCount; i++)
{
await Task.Delay(config.FileLockRetryDelayMs);
if (!IsFileLocked(filePath))
{
try
{
await action();
Console.WriteLine($"文件锁已释放,成功处理: {filePath}");
return true;
}
catch
{
// 继续重试
}
}
}
Console.WriteLine($"文件仍然被锁定,重试{config.FileLockRetryCount}次后放弃: {filePath}");
return false;
case "log":
default:
// 只记录不处理
Console.WriteLine($"文件被锁定,已记录: {filePath}");
return false;
}
}
/// <summary>
/// 检查文件系统监控器是否健康
/// </summary>
/// <param name="watcher">要检查的监控器</param>
/// <param name="lastEventTime">最后一次事件的时间</param>
/// <param name="config">健壮性配置</param>
/// <returns>如果监控器健康则返回true否则返回false</returns>
public static bool IsWatcherHealthy(FileSystemWatcher watcher, DateTime lastEventTime, RobustnessConfig config)
{
if (watcher == null || !watcher.EnableRaisingEvents)
return false;
// 如果配置了超时时间,检查是否超时
if (config.WatcherTimeoutSeconds > 0)
{
// 如果最后事件时间超过了超时时间,认为监控器可能已经失效
TimeSpan timeSinceLastEvent = DateTime.UtcNow - lastEventTime;
if (timeSinceLastEvent.TotalSeconds > config.WatcherTimeoutSeconds)
{
// 执行一个简单的测试:尝试改变一些属性看是否抛出异常
try
{
var currentFilter = watcher.Filter;
watcher.Filter = currentFilter;
return true; // 如果没有异常,认为监控器仍然正常
}
catch
{
return false; // 抛出异常,认为监控器已经失效
}
}
}
// 默认情况下认为监控器健康
return true;
}
}
}

View File

@ -0,0 +1,39 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
<DockerfileContext>..\..</DockerfileContext>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.1.66" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.4" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.0" />
<!--<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />-->
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
<PackageReference Include="Serilog" Version="4.2.0" />
<PackageReference Include="Serilog.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="Serilog.Settings.Configuration" Version="8.0.4" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,10 @@
{
"profiles": {
"JiShe.CollectBus.PluginFileWatcher": {
"commandName": "Project"
},
"Container (Dockerfile)": {
"commandName": "Docker"
}
}
}

View File

@ -0,0 +1,88 @@
{
"Serilog": {
"Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ],
"MinimumLevel": {
"Default": "Information",
"Override": {
"Microsoft": "Warning",
"System": "Warning"
}
},
"WriteTo": [
{
"Name": "Console",
"Args": {
"outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}] [{Level:u3}] {Message:lj}{NewLine}{Exception}"
}
},
{
"Name": "File",
"Args": {
"path": "Logs/filemonitor-.log",
"rollingInterval": "Day",
"outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}] [{Level:u3}] {Message:lj}{NewLine}{Exception}",
"retainedFileCountLimit": 31
}
}
],
"Enrich": [ "FromLogContext" ]
},
"FileMonitorConfig": {
"General": {
"EnableFileFiltering": true,
"MemoryMonitorIntervalMinutes": 1,
"DefaultMonitorPath": "D:\\MonitorFiles"
},
"FileFilters": {
"AllowedExtensions": [ ".dll" ],
"ExcludedDirectories": [ "bin", "obj", "node_modules" ],
"IncludeSubdirectories": true
},
"Performance": {
"MemoryCleanupThreshold": 5000,
"ChannelCapacity": 1000,
"EventDebounceTimeSeconds": 3,
"MaxDictionarySize": 10000,
"CleanupIntervalSeconds": 5,
"ProcessingDelayMs": 5
},
"Robustness": {
"EnableAutoRecovery": true,
"WatcherHealthCheckIntervalSeconds": 30,
"WatcherTimeoutSeconds": 60,
"MaxRestartAttempts": 3,
"RestartDelaySeconds": 5,
"EnableFileLockDetection": true,
"LockedFileStrategy": "Retry",
"FileLockRetryCount": 3,
"FileLockRetryDelayMs": 500
},
"EventStorage": {
"EnableEventStorage": true,
"StorageType": "SQLite",
"StorageDirectory": "D:/EventLogs",
"DatabasePath": "D:/EventLogs/events.db",
"ConnectionString": "Data Source=D:/EventLogs/events.db;Foreign Keys=True",
"CommandTimeout": 30,
"LogFileNameFormat": "FileEvents_{0:yyyy-MM-dd}.json",
"StorageIntervalSeconds": 60,
"BatchSize": 100,
"MaxEventRecords": 100000,
"DataRetentionDays": 30,
"MaxLogFiles": 30,
"CompressLogFiles": true,
"EnableEventReplay": true,
"ReplayIntervalMs": 100,
"ReplaySpeedFactor": 1.0
},
"NotifyFilters": [
"LastWrite",
"FileName",
"DirectoryName",
"CreationTime"
],
"Logging": {
"LogLevel": "Information"
}
}
}

View File

@ -0,0 +1,720 @@
using JiShe.CollectBus.Analyzers.Shared;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Text;
namespace JiShe.CollectBus.IncrementalGenerator
{
/// <summary>
/// 复杂类型增量源生成器
/// </summary>
[Generator(LanguageNames.CSharp)]
public class ComplexTypeSourceAnalyzers : IIncrementalGenerator
{
private const string AttributeFullName = "JiShe.CollectBus.Analyzers.Shared.SourceAnalyzersAttribute";
public void Initialize(IncrementalGeneratorInitializationContext context)
{
//Debugger.Launch();
// 步骤1筛选带有 [SourceAnalyzers] 的类
var classDeclarations = context.SyntaxProvider
.CreateSyntaxProvider(
predicate: static (s, _) => IsClassWithAttribute(s),
transform: static (ctx, _) => GetClassDeclaration(ctx))
.Where(static c => c is not null);
// 步骤2合并编译信息
var compilationAndClasses = context.CompilationProvider.Combine(classDeclarations.Collect());
context.RegisterSourceOutput(compilationAndClasses, (spc, source) =>
GenerateCode(source.Left, source.Right!, spc));
}
private static bool IsClassWithAttribute(SyntaxNode node) => node is ClassDeclarationSyntax cds && cds.AttributeLists.Count > 0;
private static ClassDeclarationSyntax GetClassDeclaration(GeneratorSyntaxContext context)
{
var classDecl = (ClassDeclarationSyntax)context.Node;
var semanticModel = context.SemanticModel;
// 获取类符号
var classSymbol = semanticModel.GetDeclaredSymbol(classDecl) as INamedTypeSymbol;
if (classSymbol == null) return null;
// 检查是否包含 SourceAnalyzers 特性
var sourceAnalyzerAttr = classSymbol.GetAttributes().FirstOrDefault(attr => attr.AttributeClass?.ToDisplayString() == AttributeFullName);
// 必须包含 EntityType 参数
if (sourceAnalyzerAttr == null ||
sourceAnalyzerAttr.ConstructorArguments.Length == 0)
{
return null;
}
return classDecl;
//var classDecl = (ClassDeclarationSyntax)context.Node;
//var attributeType = context.SemanticModel.Compilation.GetTypeByMetadataName(AttributeFullName);
//foreach (var attribute in classDecl.AttributeLists.SelectMany(al => al.Attributes))
//{
// var symbol = context.SemanticModel.GetSymbolInfo(attribute).Symbol;
// if (symbol is IMethodSymbol ctor &&
// SymbolEqualityComparer.Default.Equals(ctor.ContainingType, attributeType))
// {
// return classDecl;
// }
//}
//return null;
}
/// <summary>
/// 递归获取所有层级的属性
/// </summary>
/// <param name="classSymbol"></param>
/// <returns></returns>
private static IEnumerable<IPropertySymbol> GetAllPropertiesInHierarchy(INamedTypeSymbol classSymbol)
{
var currentSymbol = classSymbol;
while (currentSymbol != null)
{
foreach (var prop in currentSymbol.GetMembers().OfType<IPropertySymbol>())
{
yield return prop;
}
currentSymbol = currentSymbol.BaseType; // 向上遍历基类
}
}
/// <summary>
/// 生成代码
/// </summary>
/// <param name="compilation"></param>
/// <param name="classes"></param>
/// <param name="context"></param>
private static void GenerateCode(
Compilation compilation,
IEnumerable<ClassDeclarationSyntax> classes,
SourceProductionContext context)
{
var processedTypes = new HashSet<ITypeSymbol>(SymbolEqualityComparer.Default);
if (!classes.Any())
{
context.ReportDiagnostic(Diagnostic.Create(
new DiagnosticDescriptor("GEN002", "没有目标类",
"没有找到SourceAnalyzers标记的类", "Debug", DiagnosticSeverity.Warning, true),
Location.None));
}
foreach (var classDecl in classes.Distinct())
{
var model = compilation.GetSemanticModel(classDecl.SyntaxTree);
var classSymbol = model.GetDeclaredSymbol(classDecl) as INamedTypeSymbol;
if (classSymbol == null || !processedTypes.Add(classSymbol))
{
context.ReportDiagnostic(Diagnostic.Create(
new DiagnosticDescriptor("GEN003", "无效符号",
$"类名称为{classDecl.Identifier.Text} 符号为空", "Error", DiagnosticSeverity.Error, true),
Location.None));
continue;
}
var code3 = BuildAccessorsForSourceEntity(classSymbol, compilation, processedTypes);
context.AddSource($"{classSymbol.Name}Accessor.g.cs", code3);
}
// 生成工厂注册代码
context.AddSource("SourceEntityAccessorFactory.g.cs", BuildFactoryCode());
}
/// <summary>
/// 获取泛型参数
/// </summary>
/// <param name="symbol"></param>
/// <returns></returns>
public static string GetGenericParams(INamedTypeSymbol symbol)
{
if (!symbol.IsGenericType) return "";
var parameters = symbol.TypeParameters.Select(t => t.Name);
return $"<{string.Join(", ", parameters)}>";
}
/// <summary>
/// 生成标准属性的访问器
/// </summary>
/// <param name="prop"></param>
/// <param name="propType"></param>
/// <param name="code"></param>
private static void GenerateStandardAccessors(IPropertySymbol prop, INamedTypeSymbol propType, StringBuilder code)
{
var parentType = prop.ContainingType.ToDisplayString();
code.AppendLine($" public static {propType.ToDisplayString()} Get{prop.Name}({parentType} obj) => obj.{prop.Name};");
if (prop.SetMethod != null)
{
code.AppendLine($" public static void Set{prop.Name}({parentType} obj, {propType.ToDisplayString()} value) => obj.{prop.Name} = value;");
}
}
/// <summary>
/// 构建实体访问器代码(支持泛型)
/// </summary>
private static string BuildAccessorsForSourceEntity(
INamedTypeSymbol classSymbol,
Compilation compilation,
HashSet<ITypeSymbol> processedTypes)
{
// 获取 SourceAnalyzers 特性的 EntityType 参数
var sourceAnalyzerAttr = classSymbol.GetAttributes()
.FirstOrDefault(attr =>
attr.AttributeClass?.ToDisplayString() == AttributeFullName);
// 解析 EntityType 枚举值
string entityTypeValue = "EntityTypeEnum.Other"; // 默认值
if (sourceAnalyzerAttr != null &&
sourceAnalyzerAttr.ConstructorArguments.Length > 0)
{
var arg = sourceAnalyzerAttr.ConstructorArguments[0];
if (arg.Kind == TypedConstantKind.Enum &&
arg.Type is INamedTypeSymbol enumType)
{
int enumValue = (int)arg.Value!;
entityTypeValue = GetEnumMemberName(enumType, enumValue);
}
}
var code = new StringBuilder();
code.AppendLine("// <auto-generated/>");
code.AppendLine("#nullable enable");
code.AppendLine("using System;");
code.AppendLine("using System.Reflection;");
code.AppendLine("using System.Collections.Generic;");
code.AppendLine("using JiShe.CollectBus.Analyzers.Shared;");
code.AppendLine($"namespace {classSymbol.ContainingNamespace.ToDisplayString()};");
code.AppendLine();
// 处理泛型类型名称
var accessibility = classSymbol.DeclaredAccessibility switch
{
Accessibility.Public => "public",
_ => "internal"
};
var genericParams = classSymbol.IsGenericType
? $"<{string.Join(", ", classSymbol.TypeParameters.Select(t => t.Name))}>"
: "";
code.AppendLine(
$"{accessibility} sealed class {classSymbol.Name}Accessor{genericParams} " + // 保留泛型参数
$": ISourceEntityAccessor<{classSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)}>");
code.AppendLine("{");
var propList = GetAllPropertiesInHierarchy(classSymbol);
//类名称
code.AppendLine($" public string EntityName {{get;}} = \"{classSymbol.Name}\";");
// 添加 EntityType 属性
code.AppendLine($" public EntityTypeEnum? EntityType {{ get; }} = {entityTypeValue};");
foreach (var prop in propList)
{
// 安全类型转换
if (prop.Type is not ITypeSymbol propType) continue;
if (propType is INamedTypeSymbol namedType)
{
GenerateStandardAccessors(prop, namedType, code);
}
if (propType is INamedTypeSymbol { IsTupleType: true } tupleType)
{
GenerateTupleAccessors(prop, tupleType, code);
}
}
//生成当前类属性名称集合
GeneratePropertyListForSourceEntity(propList, code, compilation, classSymbol);
//生成当前类属性信息集合
GenerateEntityMemberInfoList(propList, code, compilation, classSymbol);
//生成当前类属性访问
GetGeneratePropertyValueForSourceEntity(propList, code, compilation, classSymbol);
//生成当前类属性设置
SetGeneratePropertyValueForSourceEntity(propList, code, compilation, classSymbol);
code.AppendLine("}");
return code.ToString();
}
/// <summary>
/// 生成ValueTuple元组属性访问器
/// </summary>
/// <param name="prop"></param>
/// <param name="tupleType"></param>
/// <param name="code"></param>
private static void GenerateTupleAccessors(
IPropertySymbol prop,
INamedTypeSymbol tupleType,
StringBuilder code)
{
var parentType = prop.ContainingType.ToDisplayString();
var tupleElements = tupleType.TupleElements;
for (int i = 0; i < tupleElements.Length; i++)
{
var element = tupleElements[i];
var elementType = element.Type.ToDisplayString();
var elementName = element.Name;
// Getter
code.AppendLine($"public static {elementType} Get{prop.Name}_{elementName}({parentType} obj) => obj.{prop.Name}.{elementName};");
// Setter
if (prop.SetMethod != null)
{
code.AppendLine($"public static void Set{prop.Name}_{elementName}({parentType} obj, {elementType} value) => obj.{prop.Name} = ({string.Join(", ", GetTupleElements(prop.Name, tupleElements, i))});");
}
}
}
private static IEnumerable<string> GetTupleElements(
string propName,
ImmutableArray<IFieldSymbol> elements,
int targetIndex)
{
for (int i = 0; i < elements.Length; i++)
{
yield return i == targetIndex
? "value"
: $"obj.{propName}.{elements[i].Name}";
}
}
/// <summary>
/// 处理System.Tuple类型的访问器生成
/// </summary>
private static void GenerateSystemTupleAccessors(
IPropertySymbol prop,
INamedTypeSymbol tupleType,
StringBuilder code,
INamedTypeSymbol classSymbol)
{
var parentType = classSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
var elementTypes = tupleType.TypeArguments;
var tupleTypeName = tupleType.ToDisplayString();
for (int i = 0; i < elementTypes.Length; i++)
{
var elementType = elementTypes[i].ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
var elementName = $"Item{i + 1}";
// Getter
code.AppendLine(
$" public static {elementType} Get{prop.Name}_{elementName}" +
$"({parentType} obj) => obj.{prop.Name}.{elementName};");
// Setter
if (prop.SetMethod != null)
{
var assignments = elementTypes.Select((_, idx) =>
idx == i ? "value" : $"obj.{prop.Name}.Item{idx + 1}"
).ToList();
code.AppendLine(
$" public static void Set{prop.Name}_{elementName}" +
$"({parentType} obj, {elementType} value) => " +
$"obj.{prop.Name} = new {tupleTypeName}({string.Join(", ", assignments)});");
}
}
}
/// <summary>
/// 增强的工厂类实现
/// </summary>
private static string BuildFactoryCode()
{
return """
using System;
using System.Collections.Concurrent;
using System.Reflection;
namespace JiShe.CollectBus.Analyzers.Shared;
public static class SourceEntityAccessorFactory
{
private static readonly ConcurrentDictionary<Type, object> _accessors = new();
public static ISourceEntityAccessor<T> GetAccessor<T>()
{
return (ISourceEntityAccessor<T>)_accessors.GetOrAdd(typeof(T), t =>
{
// 获取泛型类型定义信息(如果是泛型类型)
var isGeneric = t.IsGenericType;
var genericTypeDef = isGeneric ? t.GetGenericTypeDefinition() : null;
var arity = isGeneric ? genericTypeDef!.GetGenericArguments().Length : 0;
// 构建访问器类名
var typeName = isGeneric
? $"{t.Namespace}.{genericTypeDef!.Name.Split('`')[0]}Accessor`{arity}"
: $"{t.Namespace}.{t.Name}Accessor";
// 尝试从当前程序集加载
var accessorType = Assembly.GetAssembly(t)!.GetType(typeName)
?? throw new InvalidOperationException($"Accessor type {typeName} not found");
// 处理泛型参数
if (isGeneric && accessorType.IsGenericTypeDefinition)
{
accessorType = accessorType.MakeGenericType(t.GetGenericArguments());
}
return Activator.CreateInstance(accessorType)!;
});
}
}
""";
}
/// <summary>
/// 属性访问生成逻辑
/// </summary>
/// <param name="propList">属性集合</param>
/// <param name="code"></param>
/// <param name="compilation"></param>
/// <param name="classSymbol"></param>
private static void GetGeneratePropertyValueForSourceEntity(
IEnumerable<IPropertySymbol> propList,
StringBuilder code,
Compilation compilation,
INamedTypeSymbol classSymbol)
{
code.AppendLine($" public object GetPropertyValue({classSymbol} targetEntity, string propertyName)");
code.AppendLine(" {");
code.AppendLine(" return propertyName switch");
code.AppendLine(" {");
foreach (var prop in propList)
{
code.AppendLine(
$" \"{prop.Name}\" => " +
$"Get{prop.Name}(targetEntity),");
if (prop.Type is INamedTypeSymbol { IsTupleType: true } tupleType)
{
foreach (var element in tupleType.TupleElements)
{
code.AppendLine(
$" \"{prop.Name}.{element.Name}\" => " +
$"Get{prop.Name}_{element.Name}(targetEntity),");
}
}
}
code.AppendLine(" _ => throw new ArgumentException($\"Unknown property: {propertyName}\")");
code.AppendLine(" };");
code.AppendLine(" }");
}
/// <summary>
/// 属性设置生成逻辑
/// </summary>
/// <param name="propList">属性集合</param>
/// <param name="code"></param>
/// <param name="compilation"></param>
/// <param name="classSymbol"></param>
private static void SetGeneratePropertyValueForSourceEntity(
IEnumerable<IPropertySymbol> propList,
StringBuilder code,
Compilation compilation,
INamedTypeSymbol classSymbol)
{
code.AppendLine($" public void SetPropertyValue({classSymbol} targetEntity, string propertyName, object value)");
code.AppendLine(" {");
code.AppendLine(" switch (propertyName)");
code.AppendLine(" {");
foreach (var prop in propList)
{
var propType = prop.Type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
code.AppendLine($" case \"{prop.Name}\":");
code.AppendLine($" Set{prop.Name}(");
code.AppendLine($" targetEntity, ({propType})value);");
code.AppendLine(" break;");
if (prop.Type is INamedTypeSymbol { IsTupleType: true } tupleType)
{
foreach (var element in tupleType.TupleElements)
{
var elementType = element.Type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
code.AppendLine($" case \"{prop.Name}.{element.Name}\":");
code.AppendLine($" Set{prop.Name}_{element.Name}(");
code.AppendLine($" targetEntity, ({elementType})value);");
code.AppendLine(" break;");
}
}
}
code.AppendLine(" default:");
code.AppendLine(" throw new ArgumentException($\"Unknown property: {propertyName}\");");
code.AppendLine(" }");
code.AppendLine(" }");
}
/// <summary>
/// 属性名称集合
/// </summary>
/// <param name="propList">属性集合</param>
/// <param name="code"></param>
/// <param name="compilation"></param>
/// <param name="classSymbol"></param>
private static void GeneratePropertyListForSourceEntity(
IEnumerable<IPropertySymbol> propList,
StringBuilder code,
Compilation compilation,
INamedTypeSymbol classSymbol)
{
code.AppendLine(" public List<string> PropertyNameList {get;} = new List<string>()");
code.AppendLine(" {");
List<string> tempPropList = new List<string>();
foreach (var prop in propList)
{
if (prop.Type is INamedTypeSymbol { IsTupleType: true } tupleType)
{
foreach (var element in tupleType.TupleElements)
{
tempPropList.Add($"\"{prop.Name}.{element.Name}\"");
}
}
else
{
tempPropList.Add($"\"{prop.Name}\"");
}
}
code.Append(string.Join(",", tempPropList));
code.AppendLine(" };");
}
/// <summary>
/// 生成当前类属性信息集合
/// </summary>
private static void GenerateEntityMemberInfoList(
IEnumerable<IPropertySymbol> propList,
StringBuilder code,
Compilation compilation,
INamedTypeSymbol classSymbol)
{
code.AppendLine(" public List<EntityMemberInfo> MemberList { get; } = new()");
code.AppendLine(" {");
var initializerLines = new List<string>();
foreach (var prop in propList)
{
var entityType = prop.ContainingType.ToDisplayString();//entity 实体类型名称
var propType = prop.Type;//实体属性的类型
var propTypeName = propType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
// var declaredTypeName = propType.Name; // 直接获取类型名称(如 "Int32"
// 处理可空类型,获取底层具体类型名称
var declaredTypeName = propType switch
{
INamedTypeSymbol { OriginalDefinition.SpecialType: SpecialType.System_Nullable_T } nullableType =>
nullableType.TypeArguments[0].Name, // 提取如 "Int32"
_ => propType.Name
};
// 处理主属性
var propAttributes = prop.GetAttributes()
.Where(a => !IsCompilerGeneratedAttribute(a))
.ToList();
var attributeInitializers = propAttributes
.Select(GenerateAttributeInitializer)
.Where(s => !string.IsNullOrEmpty(s));
var mainMember = new StringBuilder();
mainMember.Append(
$"new EntityMemberInfo(" +
$"\"{prop.Name}\", " +
$"typeof({propTypeName}), " +
$"\"{declaredTypeName}\", " +
$"(e) => Get{prop.Name}(({entityType})e), " +
$"(e, v) => Set{prop.Name}(({entityType})e, ({propTypeName})v))");
if (attributeInitializers.Any())
{
mainMember.AppendLine();
mainMember.Append(" { CustomAttributes = new List<Attribute>");
mainMember.Append($" {{ {string.Join(", ", attributeInitializers)} }} }}");
}
initializerLines.Add(mainMember.ToString());
// 处理元组元素,(暂不需要处理元组元素的特性)
if (prop.Type is INamedTypeSymbol { IsTupleType: true } tupleType)
{
foreach (var element in tupleType.TupleElements)
{
var elementType = element.Type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);//元组元素的类型
var elementName = element.Name;//元组元素名称
var elementDeclaredName = element.Type.Name;//元组元素类型名称
initializerLines.Add(
$"new EntityMemberInfo(" +
$"\"{prop.Name}.{elementName}\", " +
$"typeof({elementType}), " +
$"GetValueTupleElementDeclaredTypeName(typeof({elementType})), " +//$"\"{elementDeclaredName}\", " +
$"(e) => Get{prop.Name}_{elementName}(({entityType})e), " +
$"(e, v) => Set{prop.Name}_{elementName}(({entityType})e, ({elementType})v))");
}
}
}
code.AppendLine(string.Join(",\n", initializerLines));
code.AppendLine(" };");
code.AppendLine(GetValueTupleElementName());
}
private static string GetValueTupleElementName()
{
return """
public static string GetValueTupleElementDeclaredTypeName(Type declaredType)
{
string typeName;
// 处理可空类型
if (declaredType.IsGenericType && declaredType.GetGenericTypeDefinition() == typeof(Nullable<>))
{
Type underlyingType = Nullable.GetUnderlyingType(declaredType);
typeName = underlyingType.Name;
}
else
{
typeName = declaredType.Name;
}
return typeName;
}
""";
}
private static string GenerateAttributeInitializer(AttributeData attribute)
{
if (attribute.AttributeClass == null)
return string.Empty;
var attributeClass = attribute.AttributeClass.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
var args = attribute.ConstructorArguments;
var namedArgs = attribute.NamedArguments;
var parameters = new List<string>();
foreach (var arg in args)
{
parameters.Add(ConvertTypedConstantToCode(arg));
}
var constructorArgs = string.Join(", ", parameters);
var initializer = new StringBuilder();
initializer.Append($"new {attributeClass}({constructorArgs})");
if (namedArgs.Any())
{
initializer.Append(" { ");
var namedArgsList = namedArgs.Select(n => $"{n.Key} = {ConvertTypedConstantToCode(n.Value)}");
initializer.Append(string.Join(", ", namedArgsList));
initializer.Append(" }");
}
return initializer.ToString();
}
private static string ConvertTypedConstantToCode(TypedConstant constant)
{
if (constant.IsNull)
return "null";
switch (constant.Kind)
{
case TypedConstantKind.Array:
var elements = constant.Values.Select(ConvertTypedConstantToCode);
return $"new[] {{ {string.Join(", ", elements)} }}";
case TypedConstantKind.Type:
var typeSymbol = (ITypeSymbol)constant.Value!;
return $"typeof({typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)})";
case TypedConstantKind.Enum:
return ConvertEnumTypedConstant(constant);
default:
return ConvertPrimitiveConstant(constant);
}
}
private static string ConvertEnumTypedConstant(TypedConstant constant)
{
var enumType = constant.Type!;
var enumValue = constant.Value!;
var enumTypeName = enumType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
foreach (var member in enumType.GetMembers().OfType<IFieldSymbol>())
{
if (member.ConstantValue != null && member.ConstantValue.Equals(enumValue))
return $"{enumTypeName}.{member.Name}";
}
return $"({enumTypeName})({enumValue})";
}
private static string ConvertPrimitiveConstant(TypedConstant constant)
{
var value = constant.Value!;
return value switch
{
string s => $"\"{s}\"",
char c => $"'{c}'",
bool b => b ? "true" : "false",
_ => value.ToString()
};
}
private static bool IsCompilerGeneratedAttribute(AttributeData attribute)
{
return attribute.AttributeClass?.ToDisplayString() == "System.Runtime.CompilerServices.CompilerGeneratedAttribute";
}
/// <summary>
/// 获取枚举的参数
/// </summary>
/// <param name="enumType"></param>
/// <param name="value"></param>
/// <returns></returns>
private static string GetEnumMemberName(INamedTypeSymbol enumType, int value)
{
foreach (var member in enumType.GetMembers().OfType<IFieldSymbol>())
{
if (member.ConstantValue is int intValue && intValue == value)
{
return $"{enumType.ToDisplayString()}.{member.Name}";
}
}
return $"{enumType.ToDisplayString()}.Other";
}
}
}

View File

@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<IsRoslynComponent>true</IsRoslynComponent>
<NoPackageAnalysis>true</NoPackageAnalysis>
<ImportDirectoryBuildProps>false</ImportDirectoryBuildProps>
<BaseOutputPath>bin</BaseOutputPath>
<IncludeBuildOutput>false</IncludeBuildOutput>
<LangVersion>latest</LangVersion>
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
<EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0" PrivateAssets="all" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.11.0" PrivateAssets="all" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Analyzers.Shared\JiShe.CollectBus.Analyzers.Shared.csproj" />
</ItemGroup>
</Project>

View File

@ -1,156 +0,0 @@
using System.Collections.Concurrent;
using Cassandra;
using Cassandra.Mapping;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
namespace JiShe.CollectBus.Cassandra
{
public class CassandraQueryOptimizer
{
private readonly ISession _session;
private readonly ILogger<CassandraQueryOptimizer> _logger;
private readonly IMemoryCache _cache;
private readonly ConcurrentDictionary<string, PreparedStatement> _preparedStatements;
private readonly int _batchSize;
private readonly TimeSpan _cacheExpiration;
public CassandraQueryOptimizer(
ISession session,
ILogger<CassandraQueryOptimizer> logger,
IMemoryCache cache,
int batchSize = 100,
TimeSpan? cacheExpiration = null)
{
_session = session;
_logger = logger;
_cache = cache;
_preparedStatements = new ConcurrentDictionary<string, PreparedStatement>();
_batchSize = batchSize;
_cacheExpiration = cacheExpiration ?? TimeSpan.FromMinutes(5);
}
public async Task<PreparedStatement> GetOrPrepareStatementAsync(string cql)
{
return _preparedStatements.GetOrAdd(cql, key =>
{
try
{
var statement = _session.Prepare(key);
_logger.LogDebug($"Prepared statement for CQL: {key}");
return statement;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed to prepare statement for CQL: {key}");
throw;
}
});
}
public async Task ExecuteBatchAsync(IEnumerable<BoundStatement> statements)
{
var batch = new BatchStatement();
var count = 0;
foreach (var statement in statements)
{
batch.Add(statement);
count++;
if (count >= _batchSize)
{
await ExecuteBatchAsync(batch);
batch = new BatchStatement();
count = 0;
}
}
if (count > 0)
{
await ExecuteBatchAsync(batch);
}
}
private async Task ExecuteBatchAsync(BatchStatement batch)
{
try
{
await _session.ExecuteAsync(batch);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to execute batch statement");
throw;
}
}
public async Task<T> GetOrSetFromCacheAsync<T>(string cacheKey, Func<Task<T>> getData)
{
if (_cache.TryGetValue(cacheKey, out T cachedValue))
{
_logger.LogDebug($"Cache hit for key: {cacheKey}");
return cachedValue;
}
var data = await getData();
_cache.Set(cacheKey, data, _cacheExpiration);
_logger.LogDebug($"Cache miss for key: {cacheKey}, data cached");
return data;
}
public async Task<IEnumerable<T>> ExecutePagedQueryAsync<T>(
string cql,
object[] parameters,
int pageSize = 100,
string pagingState = null) where T : class
{
var statement = await GetOrPrepareStatementAsync(cql);
var boundStatement = statement.Bind(parameters);
if (!string.IsNullOrEmpty(pagingState))
{
boundStatement.SetPagingState(Convert.FromBase64String(pagingState));
}
boundStatement.SetPageSize(pageSize);
try
{
var result = await _session.ExecuteAsync(boundStatement);
//TODO: RETURN OBJECT
throw new NotImplementedException();
//result.GetRows()
//return result.Select(row => row);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed to execute paged query: {cql}");
throw;
}
}
public async Task BulkInsertAsync<T>(IEnumerable<T> items, string tableName)
{
var mapper = new Mapper(_session);
var batch = new List<BoundStatement>();
var cql = $"INSERT INTO {tableName} ({{0}}) VALUES ({{1}})";
foreach (var chunk in items.Chunk(_batchSize))
{
var statements = chunk.Select(item =>
{
var props = typeof(T).GetProperties();
var columns = string.Join(", ", props.Select(p => p.Name));
var values = string.Join(", ", props.Select(p => "?"));
var statement = _session.Prepare(string.Format(cql, columns, values));
return statement.Bind(props.Select(p => p.GetValue(item)).ToArray());
});
batch.AddRange(statements);
}
await ExecuteBatchAsync(batch);
}
}
}

View File

@ -1,19 +1,13 @@
using Cassandra;
using Cassandra.Data.Linq;
using System.Linq.Expressions;
using Cassandra.Mapping;
using JiShe.CollectBus.Cassandra.Extensions;
using JiShe.CollectBus.Common.Attributes;
using Microsoft.AspNetCore.Http;
using System.Reflection;
using Thrift.Protocol.Entities;
using Volo.Abp.Domain.Entities;
using Volo.Abp.Domain.Repositories;
namespace JiShe.CollectBus.Cassandra
{
public class CassandraRepository<TEntity, TKey>
: ICassandraRepository<TEntity, TKey>
where TEntity : class
where TEntity : class, ICassandraEntity<TKey>
{
private readonly ICassandraProvider _cassandraProvider;
public CassandraRepository(ICassandraProvider cassandraProvider, MappingConfiguration mappingConfig)
@ -27,12 +21,29 @@ namespace JiShe.CollectBus.Cassandra
public virtual async Task<TEntity> GetAsync(TKey id)
{
return await Mapper.SingleOrDefaultAsync<TEntity>("WHERE id = ?", id);
return await GetAsync("WHERE id = ?", id);
}
public virtual async Task<List<TEntity>> GetListAsync()
public virtual async Task<TEntity?> GetAsync(string cql, params object[] args)
{
return (await Mapper.FetchAsync<TEntity>()).ToList();
return await Mapper.SingleAsync<TEntity?>(cql, args);
}
public virtual async Task<TEntity> FirstOrDefaultAsync(TKey id)
{
return await FirstOrDefaultAsync("WHERE id = ?", id);
}
public virtual async Task<TEntity?> FirstOrDefaultAsync(string cql, params object[] args)
{
return await Mapper.FirstOrDefaultAsync<TEntity>(cql, args);
}
public virtual async Task<List<TEntity>?> GetListAsync(string? cql = null, params object[] args)
{
return cql.IsNullOrWhiteSpace() ? (await Mapper.FetchAsync<TEntity>()).ToList() : (await Mapper.FetchAsync<TEntity>(cql, args)).ToList();
}
public virtual async Task<TEntity> InsertAsync(TEntity entity)

View File

@ -1,7 +1,4 @@
using Cassandra;
using Cassandra.Mapping;
using JiShe.CollectBus.Cassandra.Mappers;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection;
using Volo.Abp;
using Volo.Abp.Autofac;
using Volo.Abp.Modularity;

View File

@ -1,10 +1,4 @@
using Autofac.Core;
using Cassandra;
using Cassandra.Mapping;
using JiShe.CollectBus.Cassandra;
using JiShe.CollectBus.Cassandra.Mappers;
using Microsoft.Extensions.Options;
using System.Reflection;
using JiShe.CollectBus.Cassandra;
using Volo.Abp;
using Volo.Abp.Modularity;
@ -26,8 +20,6 @@ namespace Microsoft.Extensions.DependencyInjection
public static void AddCassandra(this ServiceConfigurationContext context)
{
context.Services.AddTransient(typeof(ICassandraRepository<,>), typeof(CassandraRepository<,>));
context.Services.AddSingleton(new MappingConfiguration()
.Define(new CollectBusMapping()));
}
}
}

View File

@ -3,9 +3,7 @@ using System.Text;
using Cassandra;
using System.ComponentModel.DataAnnotations;
using JiShe.CollectBus.Common.Attributes;
using Cassandra.Mapping;
using Cassandra.Data.Linq;
using Thrift.Protocol.Entities;
using Volo.Abp.Data;
namespace JiShe.CollectBus.Cassandra.Extensions
{
@ -16,17 +14,26 @@ namespace JiShe.CollectBus.Cassandra.Extensions
var type = typeof(TEntity);
var tableAttribute = type.GetCustomAttribute<CassandraTableAttribute>();
var tableName = tableAttribute?.Name ?? type.Name.ToLower();
//var tableKeyspace = tableAttribute?.Keyspace ?? defaultKeyspace;
var tableKeyspace = session.Keyspace;
var properties = type.GetProperties();
var primaryKey = properties.FirstOrDefault(p => p.GetCustomAttribute<KeyAttribute>() != null);
// 分区键设置
var primaryKey = properties.FirstOrDefault(p => p.GetCustomAttribute<PartitionKeyAttribute>() != null);
if (primaryKey == null)
{
throw new InvalidOperationException($"No primary key defined for type {type.Name}");
}
// 集群键设置
var clusteringKeys = properties.Where(p => p.GetCustomAttribute<ClusteringKeyAttribute>() != null).Select(a=>a.Name).ToList();
var clusteringKeyCql = string.Empty;
if (clusteringKeys.Any())
{
clusteringKeyCql = $", {string.Join(", ", clusteringKeys)}";
}
var cql = new StringBuilder();
cql.Append($"CREATE TABLE IF NOT EXISTS {tableKeyspace}.{tableName} (");
@ -40,7 +47,7 @@ namespace JiShe.CollectBus.Cassandra.Extensions
cql.Append($"{columnName} {cqlType}, ");
}
cql.Length -= 2; // Remove last comma and space
cql.Append($", PRIMARY KEY ({primaryKey.Name.ToLower()}))");
cql.Append($", PRIMARY KEY (({primaryKey.Name.ToLower()}){clusteringKeyCql}))");
session.Execute(cql.ToString());
}
@ -61,6 +68,7 @@ namespace JiShe.CollectBus.Cassandra.Extensions
if (type == typeof(Guid)) return "uuid";
if (type == typeof(DateTimeOffset)) return "timestamp";
if (type == typeof(Byte[])) return "blob";
if (type == typeof(ExtraPropertyDictionary)) return "map<text,text>";
// 处理集合类型
if (type.IsGenericType)
@ -72,6 +80,8 @@ namespace JiShe.CollectBus.Cassandra.Extensions
return $"list<{GetCassandraType(elementType)}>";
if (genericType == typeof(HashSet<>))
return $"set<{GetCassandraType(elementType)}>";
if (genericType == typeof(Nullable<>))
return GetCassandraType(elementType);
if (genericType == typeof(Dictionary<,>))
{
var keyType = type.GetGenericArguments()[0];

View File

@ -10,7 +10,10 @@ namespace JiShe.CollectBus.Cassandra
public interface ICassandraRepository<TEntity, TKey> where TEntity : class
{
Task<TEntity> GetAsync(TKey id);
Task<List<TEntity>> GetListAsync();
Task<TEntity?> GetAsync(string cql, params object[] args);
Task<TEntity> FirstOrDefaultAsync(TKey id);
Task<TEntity?> FirstOrDefaultAsync(string cql, params object[] args);
Task<List<TEntity>?> GetListAsync(string? cql = null, params object[] args);
Task<TEntity> InsertAsync(TEntity entity);
Task<TEntity> UpdateAsync(TEntity entity);
Task DeleteAsync(TEntity entity);

View File

@ -15,8 +15,8 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\services\JiShe.CollectBus.Domain\JiShe.CollectBus.Domain.csproj" />
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj" />
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Domain.Shared\JiShe.CollectBus.Domain.Shared.csproj" />
</ItemGroup>
</Project>

View File

@ -37,466 +37,6 @@ namespace JiShe.CollectBus.FreeRedis
Instance.Deserialize = (json, type) => BusJsonSerializer.Deserialize(json, type);
Instance.Notice += (s, e) => Trace.WriteLine(e.Log);
return Instance;
}
///// <summary>
///// 单个添加数据
///// </summary>
///// <typeparam name="T"></typeparam>
///// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
///// <param name="redisCacheFocusIndexKey">集中器索引Set缓存Key</param>
///// <param name="redisCacheScoresIndexKey">集中器排序索引ZSET缓存Key</param>
///// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
///// <param name="data">表计信息</param>
///// <param name="timestamp">可选时间戳</param>
///// <returns></returns>
//public async Task AddMeterCacheData<T>(
//string redisCacheKey,
//string redisCacheFocusIndexKey,
//string redisCacheScoresIndexKey,
//string redisCacheGlobalIndexKey,
//T data,
//DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel
//{
// // 参数校验增强
// if (data == null || string.IsNullOrWhiteSpace(redisCacheKey)
// || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey)
// || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey)
// || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey))
// {
// throw new ArgumentException($"{nameof(AddMeterCacheData)} 参数异常,-101");
// }
// // 计算组合score分类ID + 时间戳)
// var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow;
// long scoreValue = ((long)data.FocusId << 32) | (uint)actualTimestamp.Ticks;
// //全局索引写入
// long globalScore = actualTimestamp.ToUnixTimeMilliseconds();
// // 使用事务保证原子性
// using (var trans = Instance.Multi())
// {
// // 主数据存储Hash
// trans.HSet(redisCacheKey, data.MemberID, data.Serialize());
// // 分类索引
// trans.SAdd(redisCacheFocusIndexKey, data.MemberID);
// // 排序索引使用ZSET
// trans.ZAdd(redisCacheScoresIndexKey, scoreValue, data.MemberID);
// //全局索引
// trans.ZAdd(redisCacheGlobalIndexKey, globalScore, data.MemberID);
// var results = trans.Exec();
// if (results == null || results.Length <= 0)
// throw new Exception($"{nameof(AddMeterCacheData)} 事务提交失败,-102");
// }
// await Task.CompletedTask;
//}
///// <summary>
///// 批量添加数据
///// </summary>
///// <typeparam name="T"></typeparam>
///// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
///// <param name="redisCacheFocusIndexKey">集中器索引Set缓存Key</param>
///// <param name="redisCacheScoresIndexKey">集中器排序索引ZSET缓存Key</param>
///// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
///// <param name="items">数据集合</param>
///// <param name="timestamp">可选时间戳</param>
///// <returns></returns>
//public async Task BatchAddMeterData<T>(
//string redisCacheKey,
//string redisCacheFocusIndexKey,
//string redisCacheScoresIndexKey,
//string redisCacheGlobalIndexKey,
//IEnumerable<T> items,
//DateTimeOffset? timestamp = null) where T : DeviceCacheBasicModel
//{
// if (items == null
// || items.Count() <=0
// || string.IsNullOrWhiteSpace(redisCacheKey)
// || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey)
// || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey)
// || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey))
// {
// throw new ArgumentException($"{nameof(BatchAddMeterData)} 参数异常,-101");
// }
// const int BATCH_SIZE = 1000; // 每批1000条
// var semaphore = new SemaphoreSlim(Environment.ProcessorCount * 2);
// foreach (var batch in items.Batch(BATCH_SIZE))
// {
// await semaphore.WaitAsync();
// _ = Task.Run(() =>
// {
// using (var pipe = Instance.StartPipe())
// {
// foreach (var item in batch)
// {
// // 计算组合score分类ID + 时间戳)
// var actualTimestamp = timestamp ?? DateTimeOffset.UtcNow;
// long scoreValue = ((long)item.FocusId << 32) | (uint)actualTimestamp.Ticks;
// //全局索引写入
// long globalScore = actualTimestamp.ToUnixTimeMilliseconds();
// // 主数据存储Hash
// pipe.HSet(redisCacheKey, item.MemberID, item.Serialize());
// // 分类索引Set
// pipe.SAdd(redisCacheFocusIndexKey, item.MemberID);
// // 排序索引使用ZSET
// pipe.ZAdd(redisCacheScoresIndexKey, scoreValue, item.MemberID);
// //全局索引
// pipe.ZAdd(redisCacheGlobalIndexKey, globalScore, item.MemberID);
// }
// pipe.EndPipe();
// }
// semaphore.Release();
// });
// }
// await Task.CompletedTask;
//}
///// <summary>
///// 删除指定redis缓存key的缓存数据
///// </summary>
///// <typeparam name="T"></typeparam>
///// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
///// <param name="redisCacheFocusIndexKey">集中器索引Set缓存Key</param>
///// <param name="redisCacheScoresIndexKey">集中器排序索引ZSET缓存Key</param>
///// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
///// <param name="data">表计信息</param>
///// <returns></returns>
//public async Task RemoveMeterData<T>(
//string redisCacheKey,
//string redisCacheFocusIndexKey,
//string redisCacheScoresIndexKey,
//string redisCacheGlobalIndexKey,
//T data) where T : DeviceCacheBasicModel
//{
// if (data == null
// || string.IsNullOrWhiteSpace(redisCacheKey)
// || string.IsNullOrWhiteSpace(redisCacheFocusIndexKey)
// || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey)
// || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey))
// {
// throw new ArgumentException($"{nameof(RemoveMeterData)} 参数异常,-101");
// }
// const string luaScript = @"
// local mainKey = KEYS[1]
// local focusIndexKey = KEYS[2]
// local scoresIndexKey = KEYS[3]
// local globalIndexKey = KEYS[4]
// local member = ARGV[1]
// local deleted = 0
// if redis.call('HDEL', mainKey, member) > 0 then
// deleted = 1
// end
// redis.call('SREM', focusIndexKey, member)
// redis.call('ZREM', scoresIndexKey, member)
// redis.call('ZREM', globalIndexKey, member)
// return deleted
// ";
// var keys = new[]
// {
// redisCacheKey,
// redisCacheFocusIndexKey,
// redisCacheScoresIndexKey,
// redisCacheGlobalIndexKey
// };
// var result = await Instance.EvalAsync(luaScript, keys, new[] { data.MemberID });
// if ((int)result == 0)
// throw new KeyNotFoundException("指定数据不存在");
//}
///// <summary>
///// 修改表计缓存信息
///// </summary>
///// <typeparam name="T"></typeparam>
///// <param name="redisCacheKey">主数据存储Hash缓存Key</param>
///// <param name="oldRedisCacheFocusIndexKey">旧集中器索引Set缓存Key</param>
///// <param name="newRedisCacheFocusIndexKey">新集中器索引Set缓存Key</param>
///// <param name="redisCacheScoresIndexKey">集中器排序索引ZSET缓存Key</param>
///// <param name="redisCacheGlobalIndexKey">集中器采集频率分组全局索引ZSet缓存Key</param>
///// <param name="newData">表计信息</param>
///// <param name="newTimestamp">可选时间戳</param>
///// <returns></returns>
//public async Task UpdateMeterData<T>(
//string redisCacheKey,
//string oldRedisCacheFocusIndexKey,
//string newRedisCacheFocusIndexKey,
//string redisCacheScoresIndexKey,
//string redisCacheGlobalIndexKey,
//T newData,
//DateTimeOffset? newTimestamp = null) where T : DeviceCacheBasicModel
//{
// if (newData == null
// || string.IsNullOrWhiteSpace(redisCacheKey)
// || string.IsNullOrWhiteSpace(oldRedisCacheFocusIndexKey)
// || string.IsNullOrWhiteSpace(newRedisCacheFocusIndexKey)
// || string.IsNullOrWhiteSpace(redisCacheScoresIndexKey)
// || string.IsNullOrWhiteSpace(redisCacheGlobalIndexKey))
// {
// throw new ArgumentException($"{nameof(UpdateMeterData)} 参数异常,-101");
// }
// var luaScript = @"
// local mainKey = KEYS[1]
// local oldFocusIndexKey = KEYS[2]
// local newFocusIndexKey = KEYS[3]
// local scoresIndexKey = KEYS[4]
// local globalIndexKey = KEYS[5]
// local member = ARGV[1]
// local newData = ARGV[2]
// local newScore = ARGV[3]
// local newGlobalScore = ARGV[4]
// -- 校验存在性
// if redis.call('HEXISTS', mainKey, member) == 0 then
// return 0
// end
// -- 更新主数据
// redis.call('HSET', mainKey, member, newData)
// -- 处理变更
// if newScore ~= '' then
// -- 删除旧索引
// redis.call('SREM', oldFocusIndexKey, member)
// redis.call('ZREM', scoresIndexKey, member)
// -- 添加新索引
// redis.call('SADD', newFocusIndexKey, member)
// redis.call('ZADD', scoresIndexKey, newScore, member)
// end
// -- 更新全局索引
// if newGlobalScore ~= '' then
// -- 删除旧索引
// redis.call('ZREM', globalIndexKey, member)
// -- 添加新索引
// redis.call('ZADD', globalIndexKey, newGlobalScore, member)
// end
// return 1
// ";
// var actualTimestamp = newTimestamp ?? DateTimeOffset.UtcNow;
// var newGlobalScore = actualTimestamp.ToUnixTimeMilliseconds();
// var newScoreValue = ((long)newData.FocusId << 32) | (uint)actualTimestamp.Ticks;
// var result = await Instance.EvalAsync(luaScript,
// new[]
// {
// redisCacheKey,
// oldRedisCacheFocusIndexKey,
// newRedisCacheFocusIndexKey,
// redisCacheScoresIndexKey,
// redisCacheGlobalIndexKey
// },
// new object[]
// {
// newData.MemberID,
// newData.Serialize(),
// newScoreValue.ToString() ?? "",
// newGlobalScore.ToString() ?? ""
// });
// if ((int)result == 0)
// {
// throw new KeyNotFoundException($"{nameof(UpdateMeterData)}指定Key{redisCacheKey}的数据不存在");
// }
//}
//public async Task<BusPagedResult<T>> SingleGetMeterPagedData<T>(
//string redisCacheKey,
//string redisCacheScoresIndexKey,
//int focusId,
//int pageSize = 10,
//int pageIndex = 1,
//bool descending = true)
//{
// // 计算score范围
// long minScore = (long)focusId << 32;
// long maxScore = ((long)focusId + 1) << 32;
// // 分页参数计算
// int start = (pageIndex - 1) * pageSize;
// // 获取排序后的member列表
// var members = descending
// ? await Instance.ZRevRangeByScoreAsync(
// redisCacheScoresIndexKey,
// maxScore,
// minScore,
// start,
// pageSize)
// : await Instance.ZRangeByScoreAsync(
// redisCacheScoresIndexKey,
// minScore,
// maxScore,
// start,
// pageSize);
// // 批量获取实际数据
// var dataTasks = members.Select(m =>
// Instance.HGetAsync<T>(redisCacheKey, m)).ToArray();
// await Task.WhenAll(dataTasks);
// // 总数统计优化
// var total = await Instance.ZCountAsync(
// redisCacheScoresIndexKey,
// minScore,
// maxScore);
// return new BusPagedResult<T>
// {
// Items = dataTasks.Select(t => t.Result).ToList(),
// TotalCount = total,
// PageIndex = pageIndex,
// PageSize = pageSize
// };
//}
//public async Task<BusPagedResult<T>> GetFocusPagedData<T>(
//string redisCacheKey,
//string redisCacheScoresIndexKey,
//int focusId,
//int pageSize = 10,
//long? lastScore = null,
//string lastMember = null,
//bool descending = true) where T : DeviceCacheBasicModel
//{
// // 计算分数范围
// long minScore = (long)focusId << 32;
// long maxScore = ((long)focusId + 1) << 32;
// // 获取成员列表
// var members = await GetSortedMembers(
// redisCacheScoresIndexKey,
// minScore,
// maxScore,
// pageSize,
// lastScore,
// lastMember,
// descending);
// // 批量获取数据
// var dataDict = await Instance.HMGetAsync<T>(redisCacheKey, members.CurrentItems);
// return new BusPagedResult<T>
// {
// Items = dataDict,
// TotalCount = await GetTotalCount(redisCacheScoresIndexKey, minScore, maxScore),
// HasNext = members.HasNext,
// NextScore = members.NextScore,
// NextMember = members.NextMember
// };
//}
//private async Task<(string[] CurrentItems, bool HasNext, decimal? NextScore, string NextMember)>
// GetSortedMembers(
// string zsetKey,
// long minScore,
// long maxScore,
// int pageSize,
// long? lastScore,
// string lastMember,
// bool descending)
//{
// var querySize = pageSize + 1;
// var (startScore, exclude) = descending
// ? (lastScore ?? maxScore, lastMember)
// : (lastScore ?? minScore, lastMember);
// var members = descending
// ? await Instance.ZRevRangeByScoreAsync(
// zsetKey,
// max: startScore,
// min: minScore,
// offset: 0,
// count: querySize)
// : await Instance.ZRangeByScoreAsync(
// zsetKey,
// min: startScore,
// max: maxScore,
// offset: 0,
// count: querySize);
// var hasNext = members.Length > pageSize;
// var currentItems = members.Take(pageSize).ToArray();
// var nextCursor = currentItems.Any()
// ? await GetNextCursor(zsetKey, currentItems.Last(), descending)
// : (null, null);
// return (currentItems, hasNext, nextCursor.score, nextCursor.member);
//}
//private async Task<long> GetTotalCount(string zsetKey, long min, long max)
//{
// // 缓存计数优化
// var cacheKey = $"{zsetKey}_count_{min}_{max}";
// var cached = await Instance.GetAsync<long?>(cacheKey);
// if (cached.HasValue)
// return cached.Value;
// var count = await Instance.ZCountAsync(zsetKey, min, max);
// await Instance.SetExAsync(cacheKey, 60, count); // 缓存60秒
// return count;
//}
//public async Task<Dictionary<int, BusPagedResult<T>>> BatchGetMeterPagedData<T>(
//string redisCacheKey,
//string redisCacheScoresIndexKey,
//IEnumerable<int> focusIds,
//int pageSizePerFocus = 10) where T : DeviceCacheBasicModel
//{
// var results = new ConcurrentDictionary<int, BusPagedResult<T>>();
// var parallelOptions = new ParallelOptions
// {
// MaxDegreeOfParallelism = Environment.ProcessorCount * 2
// };
// await Parallel.ForEachAsync(focusIds, parallelOptions, async (focusId, _) =>
// {
// var data = await SingleGetMeterPagedData<T>(
// redisCacheKey,
// redisCacheScoresIndexKey,
// focusId,
// pageSizePerFocus);
// results.TryAdd(focusId, data);
// });
// return new Dictionary<int, BusPagedResult<T>>(results);
//}
}
}
}

View File

@ -1,4 +1,4 @@
namespace JiShe.CollectBus.IoTDB.Attribute
namespace JiShe.CollectBus.IoTDB.Attributes
{
/// <summary>
/// Column分类标记特性ATTRIBUTE字段,也就是属性字段

View File

@ -1,4 +1,4 @@
namespace JiShe.CollectBus.IoTDB.Attribute
namespace JiShe.CollectBus.IoTDB.Attributes
{
/// <summary>
/// Column分类标记特性FIELD字段数据列字段

View File

@ -1,7 +1,7 @@
namespace JiShe.CollectBus.IoTDB.Attribute
namespace JiShe.CollectBus.IoTDB.Attributes
{
/// <summary>
/// 用于标识当前实体为单侧点模式单侧点模式只有一个Filed标识字段,类型是Tuple<string,object>,Item1=>测点名称Item2=>测点值,泛型
/// 用于标识当前实体为单侧点模式单侧点模式只有一个Filed标识字段,类型是Tuple<string,T>,Item1=>测点名称Item2=>测点值,泛型
/// </summary>
[AttributeUsage(AttributeTargets.Property)]
public class SingleMeasuringAttribute : System.Attribute

View File

@ -1,4 +1,4 @@
namespace JiShe.CollectBus.IoTDB.Attribute
namespace JiShe.CollectBus.IoTDB.Attributes
{
/// <summary>
/// Column分类标记特性TAG字段标签字段

View File

@ -0,0 +1,17 @@

namespace JiShe.CollectBus.IoTDB.Attributes
{
/// <summary>
/// IoTDB实体存储路径或表名称一般用于已经明确的存储路径或表名称例如日志存储
/// </summary>
[AttributeUsage(AttributeTargets.Class)]
public class TableNameOrTreePathAttribute : System.Attribute
{
public string TableNameOrTreePath { get; }
public TableNameOrTreePathAttribute(string tableNameOrTreePath)
{
TableNameOrTreePath = tableNameOrTreePath;
}
}
}

View File

@ -1,33 +1,22 @@
using JiShe.CollectBus.IoTDB.Context;
using JiShe.CollectBus.IoTDB.Interface;
using JiShe.CollectBus.IoTDB.Options;
using JiShe.CollectBus.IoTDB.Provider;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Volo.Abp.Modularity;
namespace JiShe.CollectBus.IoTDB
namespace JiShe.CollectBus.IoTDB;
/// <summary>
/// CollectBusIoTDBModule
/// </summary>
public class CollectBusIoTDbModule : AbpModule
{
public class CollectBusIoTDBModule : AbpModule
public override void ConfigureServices(ServiceConfigurationContext context)
{
public override void ConfigureServices(ServiceConfigurationContext context)
{
var configuration = context.Services.GetConfiguration();
Configure<IoTDBOptions>(options =>
{
configuration.GetSection(nameof(IoTDBOptions)).Bind(options);
});
var configuration = context.Services.GetConfiguration();
Configure<IoTDbOptions>(options => { configuration.GetSection(nameof(IoTDbOptions)).Bind(options); });
// 注册上下文为Scoped
context.Services.AddScoped<IoTDBRuntimeContext>();
// 注册Session工厂
context.Services.AddSingleton<IIoTDBSessionFactory, IoTDBSessionFactory>();
// 注册Provider
context.Services.AddScoped<IIoTDBProvider, IoTDBProvider>();
}
//// 注册上下文为Scoped
//context.Services.AddScoped<IoTDBRuntimeContext>();
}
}
}

View File

@ -1,23 +1,24 @@
using JiShe.CollectBus.IoTDB.Options;
using Microsoft.Extensions.Options;
using Volo.Abp.DependencyInjection;
namespace JiShe.CollectBus.IoTDB.Context
{
/// <summary>
/// IoTDB SessionPool 运行时上下文
/// </summary>
public class IoTDBRuntimeContext
public class IoTDBRuntimeContext: IScopedDependency//ITransientDependency
{
private readonly bool _defaultValue;
public IoTDBRuntimeContext(IOptions<IoTDBOptions> options)
public IoTDBRuntimeContext(IOptions<IoTDbOptions> options)
{
_defaultValue = options.Value.UseTableSessionPoolByDefault;
UseTableSessionPool = _defaultValue;
}
/// <summary>
/// 是否使用表模型存储, 默认false使用tree模型存储
/// 存储模型切换标识是否使用table模型存储, 默认为false标识tree模型存储
/// </summary>
public bool UseTableSessionPool { get; set; }

View File

@ -0,0 +1,22 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.IoTDB.Exceptions
{
/// <summary>
/// IoTDB异常
/// </summary>
public class IoTException : Exception
{
public int ErrorCode { get; }
public IoTException(string message, int errorCode)
: base($"{message} (Code: {errorCode})")
{
ErrorCode = errorCode;
}
}
}

View File

@ -1,4 +1,5 @@
using JiShe.CollectBus.Common.Models;
using JiShe.CollectBus.IoTDB.Model;
using JiShe.CollectBus.IoTDB.Options;
using JiShe.CollectBus.IoTDB.Provider;
@ -7,7 +8,7 @@ namespace JiShe.CollectBus.IoTDB.Interface
/// <summary>
/// IoTDB数据源,数据库能同时存多个时序模型,但数据是完全隔离的,不能跨时序模型查询,通过连接字符串配置
/// </summary>
public interface IIoTDBProvider
public interface IIoTDbProvider
{
///// <summary>
///// 切换 SessionPool
@ -15,6 +16,8 @@ namespace JiShe.CollectBus.IoTDB.Interface
///// <param name="useTableSession">是否使用表模型</param>
//void SwitchSessionPool(bool useTableSession);
IIoTDbProvider GetSessionPool(bool sessionpolType);
/// <summary>
/// 插入数据
/// </summary>
@ -31,6 +34,15 @@ namespace JiShe.CollectBus.IoTDB.Interface
/// <returns></returns>
Task BatchInsertAsync<T>(IEnumerable<T> entities) where T : IoTEntity;
/// <summary>
/// 批量插入数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="deviceMetadata">设备元数据</param>
/// <param name="entities"></param>
/// <returns></returns>
Task BatchInsertAsync<T>(DeviceMetadata deviceMetadata,IEnumerable<T> entities) where T : IoTEntity;
/// <summary>
/// 删除数据
@ -38,7 +50,14 @@ namespace JiShe.CollectBus.IoTDB.Interface
/// <typeparam name="T"></typeparam>
/// <param name="options"></param>
/// <returns></returns>
Task<object> DeleteAsync<T>(QueryOptions options) where T : IoTEntity;
Task<object> DeleteAsync<T>(IoTDBQueryOptions options) where T : IoTEntity;
/// <summary>
/// 获取设备元数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
Task<DeviceMetadata> GetMetadata<T>() where T : IoTEntity;
/// <summary>
/// 查询数据
@ -46,6 +65,6 @@ namespace JiShe.CollectBus.IoTDB.Interface
/// <typeparam name="T"></typeparam>
/// <param name="options"></param>
/// <returns></returns>
Task<BusPagedResult<T>> QueryAsync<T>(QueryOptions options) where T : IoTEntity, new();
Task<BusPagedResult<T>> QueryAsync<T>(IoTDBQueryOptions options) where T : IoTEntity, new();
}
}

View File

@ -3,8 +3,8 @@
/// <summary>
/// Session 工厂接口
/// </summary>
public interface IIoTDBSessionFactory:IDisposable
public interface IIoTDbSessionFactory:IDisposable
{
IIoTDBSessionPool GetSessionPool(bool useTableSession);
IIoTDbSessionPool GetSessionPool(bool useTableSession);
}
}

View File

@ -5,7 +5,7 @@ namespace JiShe.CollectBus.IoTDB.Interface
/// <summary>
/// Session 连接池
/// </summary>
public interface IIoTDBSessionPool : IDisposable
public interface IIoTDbSessionPool : IDisposable
{
/// <summary>
/// 打开连接池
@ -13,6 +13,12 @@ namespace JiShe.CollectBus.IoTDB.Interface
/// <returns></returns>
Task OpenAsync();
/// <summary>
/// 关闭连接池
/// </summary>
/// <returns></returns>
Task CloseAsync();
/// <summary>
/// 插入数据
/// </summary>

View File

@ -1,16 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<!--<PackageReference Include="Apache.IoTDB" Version="1.3.3.1" />-->
<PackageReference Include="Apache.IoTDB" Version="2.0.2" />
<PackageReference Include="Volo.Abp" Version="8.3.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj" />
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Analyzers.Shared\JiShe.CollectBus.Analyzers.Shared.csproj" />
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj" />
<ProjectReference Include="..\..\modules\JiShe.CollectBus.Analyzers\JiShe.CollectBus.Analyzers.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,72 @@
using JiShe.CollectBus.Common.Attributes;
using JiShe.CollectBus.Common.Consts;
using JiShe.CollectBus.IoTDB.Attributes;
using Volo.Abp.Domain.Entities;
namespace JiShe.CollectBus.IoTDB.Model
{
/// <summary>
/// IoT实体基类此类适用于多个数据测点记录场景单个测点请使用子类 SingleMeasuring新增字段只能现有字段末尾添加否则会导致数据写入失败。
/// </summary>
public abstract class IoTEntity
{
/// <summary>
/// 系统名称
/// </summary>
[TAGColumn]
public string SystemName { get; set; }
/// <summary>
/// 项目编码
/// </summary>
[TAGColumn]
public string ProjectId { get; set; }
/// <summary>
/// 数据类型
/// </summary>
[TAGColumn]
public string DataType { get; set; } = IOTDBDataTypeConst.Data;
/// <summary>
/// 设备类型集中器、电表、水表、流量计、传感器等
/// </summary>
[TAGColumn]
public string DeviceType { get; set; }
/// <summary>
/// 设备ID,数据生成者例如集中器ID,电表ID、水表ID、流量计ID、传感器ID等
/// </summary>
[TAGColumn]
public string DeviceId { get; set; }
/// <summary>
/// 时标,也就是业务时间戳,单位毫秒,必须通过DateTimeOffset获取
/// </summary>
public long Timestamps { get; set; } = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
/// <summary>
/// 设备路径
/// </summary>
private string _devicePath;
/// <summary>
/// 设备路径
/// </summary>
public virtual string DevicePath
{
get
{
// 如果未手动设置路径,则自动生成
if (string.IsNullOrWhiteSpace(_devicePath))
{
return $"root.{SystemName.ToLower()}.`{ProjectId}`.`{DeviceType}`.{DataType}.`{DeviceId}`";
}
return _devicePath;
}
set
{
_devicePath = value; // 直接赋值给支持字段,避免递归
}
}
}
}

View File

@ -0,0 +1,18 @@
using JiShe.CollectBus.Analyzers.Shared;
using JiShe.CollectBus.IoTDB.Attributes;
namespace JiShe.CollectBus.IoTDB.Model
{
/// <summary>
/// Table模型单项数据实体
/// </summary>
[SourceAnalyzers(EntityTypeEnum.TableModel)]
public class TableModelSingleMeasuringEntity<T> : IoTEntity
{
/// <summary>
/// 单项数据键值对
/// </summary>
[SingleMeasuring(nameof(SingleColumn))]
public required ValueTuple<string, T> SingleColumn { get; set; }
}
}

View File

@ -0,0 +1,18 @@
using JiShe.CollectBus.Analyzers.Shared;
using JiShe.CollectBus.IoTDB.Attributes;
namespace JiShe.CollectBus.IoTDB.Model
{
/// <summary>
/// Tree模型单项数据实体
/// </summary>
[SourceAnalyzers(EntityTypeEnum.TreeModel)]
public class TreeModelSingleMeasuringEntity<T> : IoTEntity
{
/// <summary>
/// 单项数据键值对
/// </summary>
[SingleMeasuring(nameof(SingleMeasuring))]
public required ValueTuple<string, T> SingleMeasuring { get; set; }
}
}

View File

@ -3,7 +3,7 @@
/// <summary>
/// IOTDB配置
/// </summary>
public class IoTDBOptions
public class IoTDbOptions
{
/// <summary>
/// 数据库名称,表模型才有,树模型为空
@ -26,7 +26,7 @@
/// <summary>
/// 连接池大小
/// </summary>
public int PoolSize { get; set; } = 2;
public int PoolSize { get; set; } = 8;
/// <summary>
/// 查询时每次查询的数据量默认1024
@ -42,5 +42,15 @@
/// 是否使用表模型存储, 默认false使用tree模型存储
/// </summary>
public bool UseTableSessionPoolByDefault { get; set; } = false;
/// <summary>
/// 时区,默认为:"UTC+08:00"
/// </summary>
public string ZoneId { get; set; } = "UTC+08:00";
/// <summary>
/// 请求超时时间单位毫秒默认为50000
/// </summary>
public long Timeout { get; set; } = 50000;
}
}

View File

@ -3,7 +3,7 @@
/// <summary>
/// 查询条件
/// </summary>
public class QueryOptions
public class IoTDBQueryOptions
{
/// <summary>
/// 表模型的表名称或者树模型的设备路径
@ -13,7 +13,7 @@
/// <summary>
/// 分页
/// </summary>
public int Page { get; set; }
public int PageIndex { get; set; }
/// <summary>
/// 分页大小
@ -23,6 +23,6 @@
/// <summary>
/// 查询条件
/// </summary>
public List<QueryCondition> Conditions { get; } = new();
public List<QueryCondition> Conditions { get; set; } = new();
}
}

View File

@ -1,4 +1,7 @@
namespace JiShe.CollectBus.IoTDB.Options
using JiShe.CollectBus.Common.Extensions;
using JiShe.CollectBus.Common.Helpers;
namespace JiShe.CollectBus.IoTDB.Options
{
/// <summary>
/// 查询条件
@ -9,13 +12,53 @@
/// 字段
/// </summary>
public string Field { get; set; }
/// <summary>
/// 操作符
/// 操作符,>=<
/// </summary>
public string Operator { get; set; }
/// <summary>
/// 是否数值,如果是数值,则进行数值比较,否则进行字符串比较
/// </summary>
public bool IsNumber { get; set; } = false;
private object _rawValue;
/// <summary>
/// 值
/// </summary>
public object Value { get; set; }
public object Value
{
get => ApplyValueConversion(_rawValue);
set => _rawValue = value;
}
/// <summary>
/// 值转换
/// </summary>
/// <param name="rawValue"></param>
/// <returns></returns>
private object ApplyValueConversion(object rawValue)
{
string declaredTypeName = rawValue.GetType().Name;
Func<object, object> converter = GetQueryConditionValue(declaredTypeName);
return converter(rawValue);
}
/// <summary>
/// 查询条件值转换委托
/// </summary>
/// <param name="declaredTypeName"></param>
/// <returns></returns>
private Func<object, object> GetQueryConditionValue(string declaredTypeName)
{
return declaredTypeName?.ToUpper() switch
{
"DATETIME" => v => v != null ? ((DateTime)v).ToUniversalTime().Ticks : null,
"STRING" => v => v != null ? $"'{v}'" : "''",
_ => v => v
};
}
}
}

View File

@ -1,12 +1,28 @@
using Apache.IoTDB;
using JiShe.CollectBus.Analyzers.Shared;
namespace JiShe.CollectBus.IoTDB.Provider
{
/// <summary>
/// 设备元数据
/// </summary>
public class DeviceMetadata
public sealed class DeviceMetadata
{
/// <summary>
/// 实体类名称
/// </summary>
public string EntityName { get; set; }
/// <summary>
/// 设备表名或树路径如果实体没有添加TableNameOrTreePath,此处为空
/// </summary>
public string TableNameOrTreePath { get; set; }
/// <summary>
/// 实体类型枚举
/// </summary>
public EntityTypeEnum? EntityType { get; set; }
/// <summary>
/// 是否有单测量值
/// </summary>
@ -25,6 +41,57 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary>
/// 值类型集合用于构建Table的值类型也就是dataTypes参数
/// </summary>
public List<TSDataType> DataTypes { get; } = new();
public List<TSDataType> DataTypes { get; set; } = new();
/// <summary>
/// 列处理信息集合
/// </summary>
public List<ColumnProcessor> Processors { get; } = new List<ColumnProcessor>();
}
/// <summary>
/// 列处理信息结构
/// </summary>
public struct ColumnProcessor
{
/// <summary>
/// 列名
/// </summary>
public string ColumnName;
/// <summary>
/// 数据类型
/// </summary>
public TSDataType TSDataType { get; set;}
/// <summary>
/// 值获取委托(参数:实体对象)
/// </summary>
public Func<object, object> ValueGetter;
/// <summary>
/// 值设置委托(参数:实体对象,新值)
/// </summary>
public Action<object, object> ValueSetter;
/// <summary>
/// 类型转换委托
/// </summary>
public Func<object, object> GetConverter;
/// <summary>
/// 类型转换委托
/// </summary>
public Func<object, object> SetConverter;
/// <summary>
/// 是否单测点
/// </summary>
public bool IsSingleMeasuring;
/// <summary>
/// 单测点名称委托
/// </summary>
public Func<object, object> SingleMeasuringNameGetter;
}
}

View File

@ -1,4 +1,6 @@
namespace JiShe.CollectBus.IoTDB.Provider
using JiShe.CollectBus.IoTDB.Model;
namespace JiShe.CollectBus.IoTDB.Provider
{
/// <summary>
/// 设备路径构建器
@ -13,7 +15,7 @@
/// <returns></returns>
public static string GetDevicePath<T>(T entity) where T : IoTEntity
{
return $"root.{entity.SystemName.ToLower()}.`{entity.ProjectCode}`.`{entity.DeviceId}`";
return $"root.{entity.SystemName.ToLower()}.`{entity.ProjectId}`.`{entity.DeviceType}`.{entity.DataType}.`{entity.DeviceId}`";
}
@ -28,6 +30,17 @@
var type = typeof(T);
return $"{type.Name.ToLower()}";
}
/// <summary>
/// 获取表名称,用作单侧点表模型特殊处理。
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entity"></param>
/// <returns></returns>
public static string GetDeviceTableName<T>(T entity) where T : IoTEntity
{
return $"{entity.SystemName.ToLower()}.`{entity.ProjectId}`.`{entity.DeviceType}`.`{entity.DeviceId}`";
}
}
}

File diff suppressed because it is too large Load Diff

View File

@ -2,6 +2,7 @@
using JiShe.CollectBus.IoTDB.Interface;
using JiShe.CollectBus.IoTDB.Options;
using Microsoft.Extensions.Options;
using Volo.Abp.DependencyInjection;
namespace JiShe.CollectBus.IoTDB.Provider
{
@ -9,25 +10,29 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary>
/// 实现带缓存的Session工厂
/// </summary>
public class IoTDBSessionFactory : IIoTDBSessionFactory
public class IoTDbSessionFactory : IIoTDbSessionFactory, ISingletonDependency
{
private readonly IoTDBOptions _options;
private readonly ConcurrentDictionary<bool, IIoTDBSessionPool> _pools = new();
private readonly IoTDbOptions _options;
private readonly ConcurrentDictionary<bool, IIoTDbSessionPool> _pools = new();
private bool _disposed;
public IoTDBSessionFactory(IOptions<IoTDBOptions> options)
/// <summary>
/// IoTDbSessionFactory
/// </summary>
/// <param name="options"></param>
public IoTDbSessionFactory(IOptions<IoTDbOptions> options)
{
_options = options.Value;
}
public IIoTDBSessionPool GetSessionPool(bool useTableSession)
public IIoTDbSessionPool GetSessionPool(bool useTableSession)
{
if (_disposed) throw new ObjectDisposedException(nameof(IoTDBSessionFactory));
if (_disposed) throw new ObjectDisposedException(nameof(IoTDbSessionFactory));
return _pools.GetOrAdd(useTableSession, key =>
{
var pool = key
? (IIoTDBSessionPool)new TableSessionPoolAdapter(_options)
? (IIoTDbSessionPool)new TableSessionPoolAdapter(_options)
: new SessionPoolAdapter(_options);
pool.OpenAsync().ConfigureAwait(false).GetAwaiter().GetResult(); ;

View File

@ -1,39 +0,0 @@
using JiShe.CollectBus.IoTDB.Attribute;
namespace JiShe.CollectBus.IoTDB.Provider
{
/// <summary>
/// IoT实体基类
/// </summary>
public abstract class IoTEntity
{
/// <summary>
/// 系统名称
/// </summary>
[TAGColumn]
public string SystemName { get; set; }
/// <summary>
/// 项目编码
/// </summary>
[TAGColumn]
public string ProjectCode { get; set; }
/// <summary>
/// 设备类型集中器、电表、水表、流量计、传感器等
/// </summary>
[TAGColumn]
public string DeviceType { get; set; }
/// <summary>
/// 设备ID
/// </summary>
[TAGColumn]
public string DeviceId { get; set; }
/// <summary>
/// 当前时间戳,单位毫秒
/// </summary>
public long Timestamps { get; set; } = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
}

View File

@ -9,18 +9,23 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary>
/// 树模型连接池
/// </summary>
public class SessionPoolAdapter : IIoTDBSessionPool
public class SessionPoolAdapter : IIoTDbSessionPool
{
private readonly SessionPool _sessionPool;
private readonly IoTDBOptions _options;
private readonly IoTDbOptions _options;
public SessionPoolAdapter(IoTDBOptions options)
/// <summary>
/// SessionPoolAdapter
/// </summary>
/// <param name="options"></param>
public SessionPoolAdapter(IoTDbOptions options)
{
_options = options;
_sessionPool = new SessionPool.Builder()
.SetNodeUrl(options.ClusterList)
.SetUsername(options.UserName)
.SetPassword(options.Password)
.SetZoneId(options.ZoneId)
.SetFetchSize(options.FetchSize)
.SetPoolSize(options.PoolSize)
.Build();
@ -42,6 +47,19 @@ namespace JiShe.CollectBus.IoTDB.Provider
}
}
/// <summary>
/// 关闭连接池
/// </summary>
/// <returns></returns>
public async Task CloseAsync()
{
if (_sessionPool == null)
{
return;
}
await _sessionPool.Close();
}
/// <summary>
/// 批量插入对齐时间序列数据
/// </summary>
@ -52,9 +70,9 @@ namespace JiShe.CollectBus.IoTDB.Provider
var result = await _sessionPool.InsertAlignedTabletAsync(tablet);
if (result != 0)
{
throw new Exception($"{nameof(TableSessionPoolAdapter)} ");
throw new Exception($"{nameof(SessionPoolAdapter)} Tree模型数据入库没有成功返回结果为{result}请检查IoTEntity继承子类属性索引是否有变动。");
}
//await CloseAsync();
return result;
}
@ -65,7 +83,10 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <returns></returns>
public async Task<SessionDataSet> ExecuteQueryStatementAsync(string sql)
{
return await _sessionPool.ExecuteQueryStatementAsync(sql);
var result = await _sessionPool.ExecuteQueryStatementAsync(sql, _options.Timeout);
//await result.Close();
//await CloseAsync();
return result;
}
public void Dispose()

View File

@ -9,18 +9,23 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <summary>
/// 表模型Session连接池
/// </summary>
public class TableSessionPoolAdapter : IIoTDBSessionPool
public class TableSessionPoolAdapter : IIoTDbSessionPool
{
private readonly TableSessionPool _sessionPool;
private readonly IoTDBOptions _options;
private readonly IoTDbOptions _options;
public TableSessionPoolAdapter(IoTDBOptions options)
/// <summary>
/// TableSessionPoolAdapter
/// </summary>
/// <param name="options"></param>
public TableSessionPoolAdapter(IoTDbOptions options)
{
_options = options;
_sessionPool = new TableSessionPool.Builder()
.SetNodeUrls(options.ClusterList)
.SetUsername(options.UserName)
.SetPassword(options.Password)
.SetZoneId(options.ZoneId)
.SetFetchSize(options.FetchSize)
.SetPoolSize(options.PoolSize)
.SetDatabase(options.DataBaseName)
@ -40,6 +45,19 @@ namespace JiShe.CollectBus.IoTDB.Provider
}
}
/// <summary>
/// 关闭连接池
/// </summary>
/// <returns></returns>
public async Task CloseAsync()
{
if (_sessionPool == null)
{
return;
}
await _sessionPool.Close();
}
/// <summary>
/// 批量插入
/// </summary>
@ -50,9 +68,10 @@ namespace JiShe.CollectBus.IoTDB.Provider
var result = await _sessionPool.InsertAsync(tablet);
if (result != 0)
{
throw new Exception($"{nameof(TableSessionPoolAdapter)} ");
throw new Exception($"{nameof(TableSessionPoolAdapter)} table模型数据入库没有成功返回结果为{result}请检查IoTEntity继承子类属性索引是否有变动。");
}
//await CloseAsync();
return result;
}
@ -63,7 +82,10 @@ namespace JiShe.CollectBus.IoTDB.Provider
/// <returns></returns>
public async Task<SessionDataSet> ExecuteQueryStatementAsync(string sql)
{
return await _sessionPool.ExecuteQueryStatementAsync(sql);
var result = await _sessionPool.ExecuteQueryStatementAsync(sql,_options.Timeout);
//await result.Close();
//await CloseAsync();
return result;
}
public void Dispose()

View File

@ -31,10 +31,14 @@
<ProjectReference Include="..\JiShe.CollectBus.Kafka\JiShe.CollectBus.Kafka.csproj" />
</ItemGroup>
<!--<ItemGroup>
<Reference Include="JiShe.CollectBus.Kafka">
<HintPath>Lib\JiShe.CollectBus.Kafka.dll</HintPath>
</Reference>
</ItemGroup>-->
<!--注意基准测试需要引用dll测试-->
<!--<ItemGroup>
<Reference Include="JiShe.CollectBus.Common">
<HintPath>Lib\JiShe.CollectBus.Common.dll</HintPath>
</Reference>
<Reference Include="JiShe.CollectBus.Kafka">
<HintPath>Lib\JiShe.CollectBus.Kafka.dll</HintPath>
</Reference>
</ItemGroup>-->
</Project>

View File

@ -1,8 +1,10 @@
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Jobs;
using Confluent.Kafka;
using JiShe.CollectBus.Common;
using JiShe.CollectBus.Kafka.AdminClient;
using JiShe.CollectBus.Kafka.Consumer;
using JiShe.CollectBus.Kafka.Internal;
using JiShe.CollectBus.Kafka.Producer;
using Microsoft.Extensions.Configuration;
@ -24,11 +26,11 @@ namespace JiShe.CollectBus.Kafka.Test
{
// 每批消息数量
[Params(1000, 10000, 100000)]
[Params(1000, 10000, 100000, 1000000)]
public int N;
public ServiceProvider _serviceProvider;
public IConsumerService _consumerService;
public IProducerService _producerService;
public IConsumerService _consumerService;
public IProducerService _producerService;
public string topic = "test-topic1";
[GlobalSetup]
@ -40,13 +42,22 @@ namespace JiShe.CollectBus.Kafka.Test
.AddJsonFile("appsettings.json")
.Build();
// 直接读取配置项
var greeting = config["ServerTagName"];
var greeting = config["Kafka:ServerTagName"];
Console.WriteLine(greeting); // 输出: Hello, World!
// 创建服务容器
var services = new ServiceCollection();
// 注册 IConfiguration 实例
services.AddSingleton<IConfiguration>(config);
services.Configure<KafkaOptionConfig>(options =>
{
config.GetSection("Kafka").Bind(options);
});
services.Configure<ServerApplicationOptions>(options =>
{
config.GetSection(nameof(ServerApplicationOptions)).Bind(options);
});
// 初始化日志
Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(config) // 从 appsettings.json 读取配置
@ -61,6 +72,8 @@ namespace JiShe.CollectBus.Kafka.Test
services.AddSingleton<IAdminClientService, AdminClientService>();
services.AddSingleton<IProducerService, ProducerService>();
services.AddSingleton<IConsumerService, ConsumerService>();
services.AddSingleton<KafkaPollyPipeline>();
services.AddTransient<KafkaSubscribeTest>();
// 构建ServiceProvider
_serviceProvider = services.BuildServiceProvider();
@ -72,10 +85,10 @@ namespace JiShe.CollectBus.Kafka.Test
var adminClientService = _serviceProvider.GetRequiredService<IAdminClientService>();
//await adminClientService.DeleteTopicAsync(topic);
// 创建 topic
adminClientService.CreateTopicAsync(topic, 3, 3).ConfigureAwait(false).GetAwaiter();
//adminClientService.CreateTopicAsync(topic, 3, 3).ConfigureAwait(false).GetAwaiter();
_consumerService = _serviceProvider.GetRequiredService<IConsumerService>();
@ -100,9 +113,9 @@ namespace JiShe.CollectBus.Kafka.Test
List<Task> tasks = new();
for (int i = 0; i < N; ++i)
{
var task = _producerService.ProduceAsync<string>(topic, i.ToString(),null);
var task = _producerService.ProduceAsync<string>(topic, i.ToString(), null);
}
await Task.WhenAll(tasks);
}
}
}
}

View File

@ -1,8 +1,5 @@
// See https://aka.ms/new-console-template for more information
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Running;
using Confluent.Kafka;
using DeviceDetectorNET.Parser.Device;
using JiShe.CollectBus.Common;
using JiShe.CollectBus.Common.Consts;
using JiShe.CollectBus.Kafka;
using JiShe.CollectBus.Kafka.AdminClient;
@ -17,10 +14,6 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Serilog;
using System.Diagnostics;
using System.Reflection;
using System.Reflection.PortableExecutable;
using System.Text.Json;
#region
//var summary = BenchmarkRunner.Run<KafkaProduceBenchmark>();
@ -38,7 +31,7 @@ var host = Host.CreateDefaultBuilder(args)
.AddJsonFile("appsettings.json")
.Build();
// 直接读取配置项
var greeting = config["Kafka:ServerTagName"];
var greeting = config["ServerApplicationOptions:ServerTagName"];
Console.WriteLine(greeting); // 输出: Hello, World!
@ -58,11 +51,23 @@ var host = Host.CreateDefaultBuilder(args)
logging.ClearProviders();
logging.AddSerilog();
});
services.Configure<KafkaOptionConfig>(config.GetSection("Kafka"));
//services.Configure<KafkaOptionConfig>(config.GetSection("Kafka"));
//services.Configure<ServerApplicationOptions>(config.GetSection("ServerApplicationOptions"));
var dss = config.GetSection("Kafka");
services.Configure<KafkaOptionConfig>(options =>
{
config.GetSection("Kafka").Bind(options);
});
services.Configure<ServerApplicationOptions>(options =>
{
config.GetSection(nameof(ServerApplicationOptions)).Bind(options);
});
services.AddSingleton<IAdminClientService, AdminClientService>();
services.AddSingleton<IProducerService, ProducerService>();
services.AddSingleton<IConsumerService, ConsumerService>();
services.AddSingleton<KafkaPollyPipeline>();
services.AddTransient<KafkaSubscribeTest>();
})
@ -85,8 +90,20 @@ var host = Host.CreateDefaultBuilder(args)
var loggerFactory = host.Services.GetRequiredService<ILoggerFactory>();
var logger = loggerFactory.CreateLogger<Program>();
logger.LogInformation("程序启动");
var _kafkaPollyPipeline = host.Services.GetRequiredService<KafkaPollyPipeline>();
if (_kafkaPollyPipeline == null)
{
logger.LogInformation("KafkaPollyPipeline未注册");
}
var adminClientService = host.Services.GetRequiredService<IAdminClientService>();
var configuration = host.Services.GetRequiredService<IConfiguration>();
var kafkaOptionConfig=host.Services.GetRequiredService<IOptions<ServerApplicationOptions>>();
string topic = ProtocolConst.TESTTOPIC;
//await adminClientService.DeleteTopicAsync(topic);
// 创建 topic

View File

@ -0,0 +1,8 @@
{
"profiles": {
"WSL": {
"commandName": "WSL2",
"distributionName": ""
}
}
}

View File

@ -13,7 +13,8 @@
"DotNetCore.CAP": "Warning",
"Serilog.AspNetCore": "Information",
"Microsoft.EntityFrameworkCore": "Warning",
"Microsoft.AspNetCore": "Warning"
"Microsoft.AspNetCore": "Warning",
"Microsoft.AspNetCore.Diagnostics.HealthChecks": "Warning"
}
},
"WriteTo": [
@ -34,7 +35,7 @@
"CorsOrigins": "http://localhost:4200,http://localhost:3100"
},
"ConnectionStrings": {
"Default": "mongodb://admin:admin02023@118.190.144.92:37117,118.190.144.92:37119,118.190.144.92:37120/JiSheCollectBus?authSource=admin&maxPoolSize=400&minPoolSize=10&waitQueueTimeoutMS=5000",
"Default": "mongodb://mongo_PmEeF3:lixiao1980@192.168.1.9:27017/JiSheCollectBus?authSource=admin&maxPoolSize=400&minPoolSize=10&waitQueueTimeoutMS=5000",
"Kafka": "192.168.1.9:29092,192.168.1.9:39092,192.168.1.9:49092",
"PrepayDB": "server=118.190.144.92;database=jishe.sysdb;uid=sa;pwd=admin@2023;Encrypt=False;Trust Server Certificate=False",
"EnergyDB": "server=118.190.144.92;database=db_energy;uid=sa;pwd=admin@2023;Encrypt=False;Trust Server Certificate=False"
@ -43,7 +44,7 @@
"Configuration": "192.168.1.9:6380,password=1q2w3e!@#,syncTimeout=30000,abortConnect=false,connectTimeout=30000,allowAdmin=true",
"MaxPoolSize": "50",
"DefaultDB": "14",
"HangfireDB": "15"
"HangfireDB": "13"
},
"Jwt": {
"Audience": "JiShe.CollectBus",
@ -51,16 +52,11 @@
"Issuer": "JiShe.CollectBus",
"ExpirationTime": 2
},
"HealthCheck": {
"HealthChecks": {
"IsEnable": true,
"MySql": {
"IsEnable": true
},
"Pings": {
"IsEnable": true,
"Host": "https://www.baidu.com/",
"TimeOut": 5000
}
"HealthCheckDatabaseName": "HealthChecks",
"EvaluationTimeInSeconds": 10,
"MinimumSecondsBetweenFailureNotifications": 60
},
"SwaggerConfig": [
{
@ -74,14 +70,6 @@
"Version": "V1"
}
],
"Cap": {
"RabbitMq": {
"HostName": "118.190.144.92",
"UserName": "collectbus",
"Password": "123456",
"Port": 5672
}
},
"Kafka": {
"BootstrapServers": "192.168.1.9:29092,192.168.1.9:39092,192.168.1.9:49092",
"EnableFilter": true,
@ -92,48 +80,17 @@
"SaslPassword": "lixiao1980",
"KafkaReplicationFactor": 3,
"NumPartitions": 30,
"ServerTagName": "JiSheCollectBus99"
//"Topic": {
// "ReplicationFactor": 3,
// "NumPartitions": 1000
//}
"FirstCollectionTime": "2025-04-22 16:07:00"
},
//"Kafka": {
// "Connections": {
// "Default": {
// "BootstrapServers": "192.168.1.9:29092,192.168.1.9:39092,192.168.1.9:49092"
// // "SecurityProtocol": "SASL_PLAINTEXT",
// // "SaslMechanism": "PLAIN",
// // "SaslUserName": "lixiao",
// // "SaslPassword": "lixiao1980",
// }
// },
// "Consumer": {
// "GroupId": "JiShe.CollectBus"
// },
// "Producer": {
// "MessageTimeoutMs": 6000,
// "Acks": -1
// },
// "Topic": {
// "ReplicationFactor": 3,
// "NumPartitions": 1000
// },
// "EventBus": {
// "GroupId": "JiShe.CollectBus",
// "TopicName": "DefaultTopicName"
// }
//},
"IoTDBOptions": {
"UserName": "root",
"Password": "root",
"ClusterList": [ "192.168.1.9:6667" ],
"PoolSize": 2,
"PoolSize": 32,
"DataBaseName": "energy",
"OpenDebugMode": true,
"UseTableSessionPoolByDefault": false
},
"ServerTagName": "JiSheCollectBus3",
"Cassandra": {
"ReplicationStrategy": {
"Class": "NetworkTopologyStrategy", //NetworkTopologyStrategySimpleStrategy
@ -156,6 +113,12 @@
"Port": 9043,
"DataCenter": "dc1",
"Rack": "RAC2"
},
{
"Host": "192.168.1.9",
"Port": 9044,
"DataCenter": "dc1",
"Rack": "RAC2"
}
],
"Username": "admin",
@ -176,5 +139,17 @@
"SerialConsistencyLevel": "Serial",
"DefaultIdempotence": true
}
}
},
"ServerApplicationOptions": {
"ServerTagName": "JiSheCollectBus4",
"SystemType": "Energy",
"FirstCollectionTime": "2025-04-28 15:07:00",
"AutomaticVerificationTime": "16:07:00",
"AutomaticTerminalVersionTime": "17:07:00",
"AutomaticTelematicsModuleTime": "17:30:00",
"AutomaticDayFreezeTime": "02:30:00",
"AutomaticMonthFreezeTime": "03:30:00",
"DefaultProtocolPlugin": "T37612012ProtocolPlugin"
},
"PlugInFolder": ""
}

View File

@ -1,204 +1,190 @@
using Confluent.Kafka;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Confluent.Kafka.Admin;
using JiShe.CollectBus.Kafka.Internal;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Volo.Abp.DependencyInjection;
namespace JiShe.CollectBus.Kafka.AdminClient
namespace JiShe.CollectBus.Kafka.AdminClient;
public class AdminClientService : IAdminClientService, IDisposable, ISingletonDependency
{
public class AdminClientService : IAdminClientService, IDisposable,ISingletonDependency
private readonly ILogger<AdminClientService> _logger;
private readonly KafkaOptionConfig _kafkaOptionConfig;
/// <summary>
/// Initializes a new instance of the <see cref="AdminClientService" /> class.
/// </summary>
/// <param name="configuration"></param>
/// <param name="logger"></param>
public AdminClientService(IConfiguration configuration, ILogger<AdminClientService> logger, IOptions<KafkaOptionConfig> kafkaOptionConfig)
{
_logger = logger;
_kafkaOptionConfig = kafkaOptionConfig.Value;
Instance = GetInstance();
}
private readonly ILogger<AdminClientService> _logger;
/// <summary>
/// Gets or sets the instance.
/// </summary>
/// <value>
/// The instance.
/// </value>
public IAdminClient Instance { get; set; }
/// <summary>
/// Initializes a new instance of the <see cref="AdminClientService"/> class.
/// </summary>
/// <param name="configuration">The configuration.</param>
/// <param name="logger">The logger.</param>
public AdminClientService(IConfiguration configuration, ILogger<AdminClientService> logger)
/// <summary>
/// 创建Kafka主题
/// </summary>
/// <param name="topic"></param>
/// <param name="numPartitions"></param>
/// <param name="replicationFactor"></param>
/// <returns></returns>
public async Task CreateTopicAsync(string topic, int numPartitions, short replicationFactor)
{
try
{
_logger = logger;
GetInstance(configuration);
}
if (await CheckTopicAsync(topic)) return;
/// <summary>
/// Gets or sets the instance.
/// </summary>
/// <value>
/// The instance.
/// </value>
public IAdminClient Instance { get; set; } = default;
/// <summary>
/// Gets the instance.
/// </summary>
/// <param name="configuration">The configuration.</param>
/// <returns></returns>
public IAdminClient GetInstance(IConfiguration configuration)
{
ArgumentNullException.ThrowIfNullOrWhiteSpace(configuration["Kafka:EnableAuthorization"]);
var enableAuthorization = bool.Parse(configuration["Kafka:EnableAuthorization"]!);
var adminClientConfig = new AdminClientConfig()
await Instance.CreateTopicsAsync(new[]
{
BootstrapServers = configuration["Kafka:BootstrapServers"],
};
if (enableAuthorization)
{
adminClientConfig.SecurityProtocol = SecurityProtocol.SaslPlaintext;
adminClientConfig.SaslMechanism = SaslMechanism.Plain;
adminClientConfig.SaslUsername = configuration["Kafka:SaslUserName"];
adminClientConfig.SaslPassword = configuration["Kafka:SaslPassword"];
}
Instance = new AdminClientBuilder(adminClientConfig).Build();
return Instance;
}
/// <summary>
/// Checks the topic asynchronous.
/// </summary>
/// <param name="topic">The topic.</param>
/// <returns></returns>
public async Task<bool> CheckTopicAsync(string topic)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(5));
return await Task.FromResult(metadata.Topics.Exists(a => a.Topic == topic));
}
/// <summary>
/// 判断Kafka主题是否存在
/// </summary>
/// <param name="topic">主题名称</param>
/// <param name="numPartitions">副本数量不能高于Brokers数量</param>
/// <returns></returns>
public async Task<bool> CheckTopicAsync(string topic,int numPartitions)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(5));
if(numPartitions > metadata.Brokers.Count)
{
throw new Exception($"{nameof(CheckTopicAsync)} 主题检查时,副本数量大于了节点数量。") ;
}
return await Task.FromResult(metadata.Topics.Exists(a => a.Topic == topic));
}
//// <summary>
/// 创建Kafka主题
/// </summary>
/// <param name="topic">主题名称</param>
/// <param name="numPartitions">主题分区数量</param>
/// <param name="replicationFactor">副本数量不能高于Brokers数量</param>
/// <returns></returns>
public async Task CreateTopicAsync(string topic, int numPartitions, short replicationFactor)
{
try
{
if (await CheckTopicAsync(topic)) return;
await Instance.CreateTopicsAsync(new[]
new TopicSpecification
{
new TopicSpecification
{
Name = topic,
NumPartitions = numPartitions,
ReplicationFactor = replicationFactor
}
});
}
catch (CreateTopicsException e)
{
if (e.Results[0].Error.Code != ErrorCode.TopicAlreadyExists)
{
throw;
Name = topic,
NumPartitions = numPartitions,
ReplicationFactor = replicationFactor
}
}
});
}
/// <summary>
/// 删除Kafka主题
/// </summary>
/// <param name="topic"></param>
/// <returns></returns>
public async Task DeleteTopicAsync(string topic)
catch (CreateTopicsException e)
{
await Instance.DeleteTopicsAsync(new[] { topic });
}
/// <summary>
/// 获取Kafka主题列表
/// </summary>
/// <returns></returns>
public async Task<List<string>> ListTopicsAsync()
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(10));
return new List<string>(metadata.Topics.Select(t => t.Topic));
}
/// <summary>
/// 判断Kafka主题是否存在
/// </summary>
/// <param name="topic"></param>
/// <returns></returns>
public async Task<bool> TopicExistsAsync(string topic)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(10));
return metadata.Topics.Any(t => t.Topic == topic);
}
/// <summary>
/// 检测分区是否存在
/// </summary>
/// <param name="topic"></param>
/// <param name="partitions"></param>
/// <returns></returns>
public Dictionary<int, bool> CheckPartitionsExists(string topic, int[] partitions)
{
var result = new Dictionary<int, bool>();
var metadata = Instance.GetMetadata(topic, TimeSpan.FromSeconds(10));
if (metadata.Topics.Count == 0)
return partitions.ToDictionary(p => p, p => false);
var existingPartitions = metadata.Topics[0].Partitions.Select(p => p.PartitionId).ToHashSet();
return partitions.ToDictionary(p => p, p => existingPartitions.Contains(p));
}
/// <summary>
/// 检测分区是否存在
/// </summary>
/// <param name="topic"></param>
/// <param name="targetPartition"></param>
/// <returns></returns>
public bool CheckPartitionsExist(string topic, int targetPartition)
{
var metadata = Instance.GetMetadata(topic, TimeSpan.FromSeconds(10));
if (metadata.Topics.Count == 0)
return false;
var partitions = metadata.Topics[0].Partitions;
return partitions.Any(p => p.PartitionId == targetPartition);
}
/// <summary>
/// 获取主题的分区数量
/// </summary>
/// <param name="topic"></param>
/// <returns></returns>
public int GetTopicPartitionsNum(string topic)
{
var metadata = Instance.GetMetadata(topic, TimeSpan.FromSeconds(10));
if (metadata.Topics.Count == 0)
return 0;
return metadata.Topics[0].Partitions.Count;
}
public void Dispose()
{
Instance?.Dispose();
if (e.Results[0].Error.Code != ErrorCode.TopicAlreadyExists) throw;
}
}
}
/// <summary>
/// 删除Kafka主题
/// </summary>
/// <param name="topic"></param>
/// <returns></returns>
public async Task DeleteTopicAsync(string topic)
{
await Instance.DeleteTopicsAsync(new[] { topic });
}
/// <summary>
/// 获取Kafka主题列表
/// </summary>
/// <returns></returns>
public async Task<List<string>> ListTopicsAsync()
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(10));
return new List<string>(metadata.Topics.Select(t => t.Topic));
}
/// <summary>
/// 判断Kafka主题是否存在
/// </summary>
/// <param name="topic"></param>
/// <returns></returns>
public async Task<bool> TopicExistsAsync(string topic)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(10));
return metadata.Topics.Any(t => t.Topic == topic);
}
/// <summary>
/// 检测分区是否存在
/// </summary>
/// <param name="topic"></param>
/// <param name="partitions"></param>
/// <returns></returns>
public Dictionary<int, bool> CheckPartitionsExists(string topic, int[] partitions)
{
var result = new Dictionary<int, bool>();
var metadata = Instance.GetMetadata(topic, TimeSpan.FromSeconds(10));
if (metadata.Topics.Count == 0)
return partitions.ToDictionary(p => p, p => false);
var existingPartitions = metadata.Topics[0].Partitions.Select(p => p.PartitionId).ToHashSet();
return partitions.ToDictionary(p => p, p => existingPartitions.Contains(p));
}
/// <summary>
/// 检测分区是否存在
/// </summary>
/// <param name="topic"></param>
/// <param name="targetPartition"></param>
/// <returns></returns>
public bool CheckPartitionsExist(string topic, int targetPartition)
{
var metadata = Instance.GetMetadata(topic, TimeSpan.FromSeconds(10));
if (metadata.Topics.Count == 0)
return false;
var partitions = metadata.Topics[0].Partitions;
return partitions.Any(p => p.PartitionId == targetPartition);
}
/// <summary>
/// 获取主题的分区数量
/// </summary>
/// <param name="topic"></param>
/// <returns></returns>
public int GetTopicPartitionsNum(string topic)
{
var metadata = Instance.GetMetadata(topic, TimeSpan.FromSeconds(10));
if (metadata.Topics.Count == 0)
return 0;
return metadata.Topics[0].Partitions.Count;
}
public void Dispose()
{
Instance?.Dispose();
}
/// <summary>
/// Gets the instance.
/// </summary>
/// <returns></returns>
public IAdminClient GetInstance()
{
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = _kafkaOptionConfig.BootstrapServers
};
if (_kafkaOptionConfig.EnableAuthorization)
{
adminClientConfig.SecurityProtocol = _kafkaOptionConfig.SecurityProtocol;
adminClientConfig.SaslMechanism = _kafkaOptionConfig.SaslMechanism;
adminClientConfig.SaslUsername = _kafkaOptionConfig.SaslUserName;
adminClientConfig.SaslPassword = _kafkaOptionConfig.SaslPassword;
}
return new AdminClientBuilder(adminClientConfig).Build();
}
/// <summary>
/// Checks the topic asynchronous.
/// </summary>
/// <param name="topic">The topic.</param>
/// <returns></returns>
public async Task<bool> CheckTopicAsync(string topic)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(5));
return await Task.FromResult(metadata.Topics.Exists(a => a.Topic == topic));
}
/// <summary>
/// 判断Kafka主题是否存在
/// </summary>
/// <param name="topic">主题名称</param>
/// <param name="numPartitions">副本数量不能高于Brokers数量</param>
/// <returns></returns>
public async Task<bool> CheckTopicAsync(string topic, int numPartitions)
{
var metadata = Instance.GetMetadata(TimeSpan.FromSeconds(5));
if (numPartitions > metadata.Brokers.Count)
throw new Exception($"{nameof(CheckTopicAsync)} 主题检查时,副本数量大于了节点数量。");
return await Task.FromResult(metadata.Topics.Exists(a => a.Topic == topic));
}
}

View File

@ -1,68 +1,60 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Attributes;
namespace JiShe.CollectBus.Kafka.Attributes
[AttributeUsage(AttributeTargets.Method)]
public class KafkaSubscribeAttribute : Attribute
{
[AttributeUsage(AttributeTargets.Method)]
public class KafkaSubscribeAttribute : Attribute
/// <summary>
/// 订阅主题
/// </summary>
/// <param name="batchTimeout"></param>
public KafkaSubscribeAttribute(string topic)
{
/// <summary>
/// 订阅的主题
/// </summary>
public string Topic { get; set; } = null!;
/// <summary>
/// 分区
/// </summary>
public int Partition { get; set; } = -1;
/// <summary>
/// 消费者组
/// </summary>
public string? GroupId { get; set; } = null;//"default"
/// <summary>
/// 任务数(默认是多少个分区多少个任务)
/// 如设置订阅指定Partition则任务数始终为1
/// </summary>
public int TaskCount { get; set; } = -1;
/// <summary>
/// 批量处理数量
/// </summary>
public int BatchSize { get; set; } = 100;
/// <summary>
/// 是否启用批量处理
/// </summary>
public bool EnableBatch { get; set; } = false;
/// <summary>
/// 批次超时时间
/// 格式:("00:05:00")
/// </summary>
public TimeSpan? BatchTimeout { get; set; }=null;
/// <summary>
/// 订阅主题
/// </summary>
/// <param name="batchTimeout"></param>
public KafkaSubscribeAttribute(string topic)
{
this.Topic = topic;
}
/// <summary>
/// 订阅主题
/// </summary>
public KafkaSubscribeAttribute(string topic, int partition)
{
this.Topic = topic;
this.Partition = partition;
}
Topic = topic;
}
}
/// <summary>
/// 订阅主题
/// </summary>
public KafkaSubscribeAttribute(string topic, int partition)
{
Topic = topic;
Partition = partition;
}
/// <summary>
/// 订阅的主题
/// </summary>
public string Topic { get; set; } = null!;
/// <summary>
/// 分区
/// </summary>
public int Partition { get; set; } = -1;
/// <summary>
/// 消费者组
/// </summary>
public string? GroupId { get; set; } = null; //"default"
/// <summary>
/// 任务数(默认是多少个分区多少个任务)
/// 如设置订阅指定Partition则任务数始终为1
/// </summary>
public int TaskCount { get; set; } = -1;
/// <summary>
/// 批量处理数量
/// </summary>
public int BatchSize { get; set; } = 100;
/// <summary>
/// 是否启用批量处理
/// </summary>
public bool EnableBatch { get; set; } = false;
/// <summary>
/// 批次超时时间
/// 格式:("00:05:00")
/// </summary>
public TimeSpan? BatchTimeout { get; set; } = null;
}

View File

@ -1,29 +1,22 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Attributes;
namespace JiShe.CollectBus.Kafka.Attributes
[AttributeUsage(AttributeTargets.Class, Inherited = false)]
public class TopicAttribute : Attribute
{
[AttributeUsage(AttributeTargets.Class, Inherited = false)]
public class TopicAttribute: Attribute
/// <summary>
/// Initializes a new instance of the <see cref="TopicAttribute" /> class.
/// </summary>
/// <param name="name">The name.</param>
public TopicAttribute(string name = "Default")
{
/// <summary>
/// Initializes a new instance of the <see cref="TopicAttribute"/> class.
/// </summary>
/// <param name="name">The name.</param>
public TopicAttribute(string name = "Default")
{
Name = name;
}
/// <summary>
/// Gets or sets the name.
/// </summary>
/// <value>
/// The name.
/// </value>
public string Name { get; set; }
Name = name;
}
}
/// <summary>
/// Gets or sets the name.
/// </summary>
/// <value>
/// The name.
/// </value>
public string Name { get; set; }
}

View File

@ -41,6 +41,9 @@ namespace JiShe.CollectBus.Kafka
// 注册Consumer
context.Services.AddSingleton<IConsumerService, ConsumerService>();
// 注册Polly
context.Services.AddSingleton<KafkaPollyPipeline>();
//context.Services.AddHostedService<HostedService>();
}

View File

@ -1,29 +1,52 @@
using Confluent.Kafka;
using JiShe.CollectBus.Common;
using JiShe.CollectBus.Common.Consts;
using JiShe.CollectBus.Common.Helpers;
using JiShe.CollectBus.Kafka.Internal;
using JiShe.CollectBus.Kafka.Serialization;
using Microsoft.AspNetCore.DataProtection.KeyManagement;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Collections.Concurrent;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
namespace JiShe.CollectBus.Kafka.Consumer
{
public class ConsumerService : IConsumerService, IDisposable
{
private readonly ILogger<ConsumerService> _logger;
private readonly IConfiguration _configuration;
private readonly ConcurrentDictionary<Type, (object Consumer, CancellationTokenSource CTS)>
/// <summary>
/// 消费者存储
/// Key 格式:{groupId}_{topic}_{TKey}_{TValue}
/// </summary>
private readonly ConcurrentDictionary<string, (object Consumer, CancellationTokenSource CTS)>
_consumerStore = new();
private readonly KafkaOptionConfig _kafkaOptionConfig;
private class KafkaConsumer<TKey, TValue> where TKey : notnull where TValue : class { }
public ConsumerService(IConfiguration configuration, ILogger<ConsumerService> logger, IOptions<KafkaOptionConfig> kafkaOptionConfig)
/// <summary>
/// 消费完或者无数据时的延迟时间
/// </summary>
private static TimeSpan DelayTime => TimeSpan.FromMilliseconds(100);
private readonly KafkaOptionConfig _kafkaOptionConfig;
private readonly ServerApplicationOptions _applicationOptions;
private readonly KafkaPollyPipeline _kafkaPollyPipeline;
/// <summary>
/// ConsumerService
/// </summary>
/// <param name="logger"></param>
/// <param name="kafkaOptionConfig"></param>
public ConsumerService(ILogger<ConsumerService> logger, IOptions<KafkaOptionConfig> kafkaOptionConfig, KafkaPollyPipeline kafkaPollyPipeline, IOptions<ServerApplicationOptions> applicationOptions)
{
_configuration = configuration;
_logger = logger;
_kafkaOptionConfig = kafkaOptionConfig.Value;
_applicationOptions = applicationOptions.Value;
_kafkaPollyPipeline = kafkaPollyPipeline;
}
#region private
@ -49,7 +72,7 @@ namespace JiShe.CollectBus.Kafka.Consumer
var config = new ConsumerConfig
{
BootstrapServers = _kafkaOptionConfig.BootstrapServers,
GroupId = groupId ?? _kafkaOptionConfig.ServerTagName,
GroupId = groupId ?? _applicationOptions.ServerTagName,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false, // 禁止AutoCommit
EnablePartitionEof = true, // 启用分区末尾标记
@ -92,7 +115,7 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <returns></returns>
public async Task SubscribeAsync<TValue>(string topic, Func<TValue, Task<bool>> messageHandler, string? groupId = null) where TValue : class
{
await SubscribeAsync<TValue>(new[] { topic }, messageHandler,groupId);
await SubscribeAsync<TValue>(new[] { topic }, messageHandler, groupId);
}
/// <summary>
@ -105,59 +128,88 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <returns></returns>
public async Task SubscribeAsync<TKey, TValue>(string[] topics, Func<TKey, TValue, Task<bool>> messageHandler, string? groupId = null) where TKey : notnull where TValue : class
{
var consumerKey = typeof(KafkaConsumer<TKey, TValue>);
var cts = new CancellationTokenSource();
//var consumer = _consumerStore.GetOrAdd(consumerKey, _ =>
//(
// CreateConsumer<TKey, TValue>(groupId),
// cts
//)).Consumer as IConsumer<TKey, TValue>;
var consumer = CreateConsumer<TKey, TValue>(groupId);
consumer!.Subscribe(topics);
await Task.Run(async () =>
try
{
while (!cts.IsCancellationRequested)
await _kafkaPollyPipeline.KafkaPipeline.ExecuteAsync(async token =>
{
try
{
//_logger.LogInformation($"Kafka消费: {string.Join("", topics)} 开始拉取消息....");
var result = consumer.Consume(cts.Token);
if (result == null || result.Message==null || result.Message.Value == null)
continue;
if (result.IsPartitionEOF)
var consumerKey = $"{groupId}_{string.Join("_", topics)}_{typeof(TKey).Name}_{typeof(TValue).Name}";
var cts = new CancellationTokenSource();
var consumer = _consumerStore.GetOrAdd(consumerKey, _ =>
(
CreateConsumer<TKey, TValue>(groupId),
cts
)).Consumer as IConsumer<TKey, TValue>;
consumer!.Subscribe(topics);
_ = Task.Run(async () =>
{
while (!cts.IsCancellationRequested)
{
_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);
await Task.Delay(TimeSpan.FromSeconds(1),cts.Token);
continue;
}
if (_kafkaOptionConfig.EnableFilter)
{
var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } };
// 检查 Header 是否符合条件
if (!headersFilter.Match(result.Message.Headers))
try
{
//consumer.Commit(result); // 提交偏移量
// 跳过消息
continue;
//_logger.LogInformation($"Kafka消费: {string.Join("", topics)} 开始拉取消息....");
var result = consumer.Consume(cts.Token);
if (result == null || result.Message == null || result.Message.Value == null)
{
await Task.Delay(DelayTime, cts.Token);
continue;
}
if (result.IsPartitionEOF)
{
#if DEBUG
_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);
#endif
await Task.Delay(DelayTime, cts.Token);
continue;
}
if (_kafkaOptionConfig.EnableFilter)
{
var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_applicationOptions.ServerTagName) } };
// 检查 Header 是否符合条件
if (!headersFilter.Match(result.Message.Headers))
{
consumer.Commit(result); // 提交偏移量
// 跳过消息
continue;
}
}
bool sucess = await messageHandler(result.Message.Key, result.Message.Value);
if (sucess)
consumer.Commit(result); // 手动提交
}
catch (ConsumeException ex) when (KafkaPollyPipeline.IsRecoverableError(ex))
{
_logger.LogError(ex, $"{string.Join("", topics)}消息消费失败: {ex.Error.Reason}");
throw; // 抛出异常,以便重试
}
catch (KafkaException ex) when (KafkaPollyPipeline.IsRecoverableError(ex))
{
_logger.LogError(ex, $"{string.Join("", topics)} 消息消费失败: {ex.Error.Reason}");
throw; // 抛出异常,以便重试
}
catch (OperationCanceledException)
{
//ignore
}
catch (Exception ex)
{
_logger.LogError(ex, "处理消息时发生未知错误");
}
}
bool sucess= await messageHandler(result.Message.Key, result.Message.Value);
if (sucess)
{
consumer.Commit(result); // 手动提交
}
}
catch (ConsumeException ex)
{
_logger.LogError(ex, $"{string.Join("", topics)}消息消费失败: {ex.Error.Reason}");
}
}
});
await Task.CompletedTask;
}, cts.Token);
await Task.CompletedTask;
});
}
catch (Exception ex)
{
throw;
}
}
@ -165,82 +217,91 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <summary>
/// 订阅消息
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TValue"></typeparam>
/// <param name="topics"></param>
/// <param name="messageHandler"></param>
/// <param name="groupId"></param>
/// <returns></returns>
public async Task SubscribeAsync<TValue>(string[] topics, Func<TValue, Task<bool>> messageHandler, string? groupId) where TValue : class
{
try {
var consumerKey = typeof(KafkaConsumer<Ignore, TValue>);
var cts = new CancellationTokenSource();
//if (topics.Contains(ProtocolConst.SubscriberLoginReceivedEventName))
//{
// string ssss = "";
//}
//var consumer = _consumerStore.GetOrAdd(consumerKey, _ =>
//(
// CreateConsumer<string, TValue>(groupId),
// cts
//)).Consumer as IConsumer<string, TValue>;
var consumer = CreateConsumer<Ignore, TValue>(groupId);
consumer!.Subscribe(topics);
_ = Task.Run(async () =>
try
{
await _kafkaPollyPipeline.KafkaPipeline.ExecuteAsync(async token =>
{
int count = 0;
while (!cts.IsCancellationRequested)
{
try
{
//_logger.LogInformation($"Kafka消费: {string.Join("", topics)}_{count} 开始拉取消息....");
count++;
var result = consumer.Consume(cts.Token);
if (result == null || result.Message == null || result.Message.Value == null)
{
await Task.Delay(500, cts.Token);
continue;
}
var consumerKey = $"{groupId}_{string.Join("_", topics)}_{typeof(Ignore).Name}_{typeof(TValue).Name}";
var cts = new CancellationTokenSource();
var consumer = _consumerStore.GetOrAdd(consumerKey, _ =>
(
CreateConsumer<Ignore, TValue>(groupId),
cts
)).Consumer as IConsumer<Ignore, TValue>;
if (result.IsPartitionEOF)
consumer!.Subscribe(topics);
_ = Task.Run(async () =>
{
int count = 0;
while (!cts.IsCancellationRequested)
{
try
{
_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);
await Task.Delay(100, cts.Token);
continue;
}
if (_kafkaOptionConfig.EnableFilter)
{
var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } };
// 检查 Header 是否符合条件
if (!headersFilter.Match(result.Message.Headers))
//_logger.LogInformation($"Kafka消费: {string.Join("", topics)}_{count} 开始拉取消息....");
count++;
var result = consumer.Consume(cts.Token);
if (result == null || result.Message == null || result.Message.Value == null)
{
await Task.Delay(500, cts.Token);
//consumer.Commit(result); // 提交偏移量
// 跳过消息
await Task.Delay(DelayTime, cts.Token);
continue;
}
if (result.IsPartitionEOF)
{
#if DEBUG
_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);
#endif
await Task.Delay(DelayTime, cts.Token);
continue;
}
if (_kafkaOptionConfig.EnableFilter)
{
var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_applicationOptions.ServerTagName) } };
// 检查 Header 是否符合条件
if (!headersFilter.Match(result.Message.Headers))
{
consumer.Commit(result); // 提交偏移量
// 跳过消息
continue;
}
}
bool sucess = await messageHandler(result.Message.Value);
if (sucess)
consumer.Commit(result); // 手动提交
//else
// consumer.StoreOffset(result);
}
catch (ConsumeException ex) when (KafkaPollyPipeline.IsRecoverableError(ex))
{
_logger.LogError(ex, $"{string.Join("", topics)}消息消费失败: {ex.Error.Reason}");
throw; // 抛出异常,以便重试
}
catch (OperationCanceledException)
{
//ignore
}
catch (Exception ex)
{
_logger.LogError(ex, "处理消息时发生未知错误");
}
bool sucess = await messageHandler(result.Message.Value);
if (sucess)
consumer.Commit(result); // 手动提交
else
consumer.StoreOffset(result);
}
catch (ConsumeException ex)
{
_logger.LogError(ex, $"{string.Join("", topics)}消息消费失败: {ex.Error.Reason}");
}
}
}, cts.Token);
await Task.CompletedTask;
});
} catch (Exception ex)
{
_logger.LogWarning($"Kafka消费异常: {ex.Message}");
}
catch (Exception ex)
{
await Task.CompletedTask;
throw;
}
}
@ -256,7 +317,15 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <param name="batchTimeout">批次超时时间</param>
public async Task SubscribeBatchAsync<TKey, TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class
{
await SubscribeBatchAsync<TKey, TValue>(new[] { topic }, messageBatchHandler, groupId, batchSize, batchTimeout);
try
{
await SubscribeBatchAsync<TKey, TValue>(new[] { topic }, messageBatchHandler, groupId, batchSize, batchTimeout);
}
catch (Exception ex)
{
throw;
}
}
/// <summary>
@ -269,109 +338,127 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <param name="groupId">消费组ID</param>
/// <param name="batchSize">批次大小</param>
/// <param name="batchTimeout">批次超时时间</param>
public async Task SubscribeBatchAsync<TKey, TValue>(string[] topics,Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null,int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class
public async Task SubscribeBatchAsync<TKey, TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class
{
var consumerKey = typeof(KafkaConsumer<TKey, TValue>);
var cts = new CancellationTokenSource();
//var consumer = _consumerStore.GetOrAdd(consumerKey, _ =>
// (
// CreateConsumer<TKey, TValue>(groupId),
// cts
// )).Consumer as IConsumer<TKey, TValue>;
var consumer = CreateConsumer<string, TValue>(groupId);
consumer!.Subscribe(topics);
var timeout = batchTimeout ?? TimeSpan.FromSeconds(5); // 默认超时时间调整为5秒
_ = Task.Run(async () =>
try
{
var messages = new List<(TValue Value, TopicPartitionOffset Offset)>();
var startTime = DateTime.UtcNow;
while (!cts.IsCancellationRequested)
await _kafkaPollyPipeline.KafkaPipeline.ExecuteAsync(async token =>
{
try
{
// 非阻塞快速累积消息
while (messages.Count < batchSize && (DateTime.UtcNow - startTime) < timeout)
{
var result = consumer.Consume(TimeSpan.Zero); // 非阻塞调用
if (result != null)
var consumerKey = $"{groupId}_{string.Join("_", topics)}_{typeof(TKey).Name}_{typeof(TValue).Name}";
var cts = new CancellationTokenSource();
var consumer = _consumerStore.GetOrAdd(consumerKey, _ =>
(
CreateConsumer<TKey, TValue>(groupId),
cts
)).Consumer as IConsumer<TKey, TValue>;
consumer!.Subscribe(topics);
var timeout = batchTimeout ?? TimeSpan.FromSeconds(5); // 默认超时时间调整为5秒
_ = Task.Run(async () =>
{
var messages = new List<(TValue Value, TopicPartitionOffset Offset)>();
var startTime = DateTime.UtcNow;
while (!cts.IsCancellationRequested)
{
try
{
if (result.IsPartitionEOF)
// 非阻塞快速累积消息
while (messages.Count < batchSize && (DateTime.UtcNow - startTime) < timeout)
{
//_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);
await Task.Delay(10, cts.Token);
}
else if (result.Message.Value != null)
{
if (_kafkaOptionConfig.EnableFilter)
var result = consumer.Consume(TimeSpan.Zero); // 非阻塞调用
if (result != null)
{
var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } };
// 检查 Header 是否符合条件
if (!headersFilter.Match(result.Message.Headers))
if (result.IsPartitionEOF)
{
//consumer.Commit(result); // 提交偏移量
// 跳过消息
continue;
#if DEBUG
_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);
#endif
await Task.Delay(DelayTime, cts.Token);
}
else if (result.Message.Value != null)
{
if (_kafkaOptionConfig.EnableFilter)
{
var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_applicationOptions.ServerTagName) } };
// 检查 Header 是否符合条件
if (!headersFilter.Match(result.Message.Headers))
{
consumer.Commit(result); // 提交偏移量
// 跳过消息
continue;
}
}
messages.Add((result.Message.Value, result.TopicPartitionOffset));
}
}
messages.Add((result.Message.Value, result.TopicPartitionOffset));
//messages.Add(result.Message.Value);
}
}
else
{
// 无消息时短暂等待
await Task.Delay(10, cts.Token);
}
}
// 处理批次
if (messages.Count > 0)
{
bool success = await messageBatchHandler(messages.Select(m => m.Value).ToList());
if (success)
{
var offsetsByPartition = new Dictionary<TopicPartition, long>();
foreach (var msg in messages)
{
var tp = msg.Offset.TopicPartition;
var offset = msg.Offset.Offset;
if (!offsetsByPartition.TryGetValue(tp, out var currentMax) || offset > currentMax)
else
{
offsetsByPartition[tp] = offset;
// 无消息时短暂等待
await Task.Delay(DelayTime, cts.Token);
}
}
var offsetsToCommit = offsetsByPartition
.Select(kv => new TopicPartitionOffset(kv.Key, new Offset(kv.Value + 1)))
.ToList();
consumer.Commit(offsetsToCommit);
// 处理批次
if (messages.Count > 0)
{
bool success = await messageBatchHandler(messages.Select(m => m.Value).ToList());
if (success)
{
var offsetsByPartition = new Dictionary<TopicPartition, long>();
foreach (var msg in messages)
{
var tp = msg.Offset.TopicPartition;
var offset = msg.Offset.Offset;
if (!offsetsByPartition.TryGetValue(tp, out var currentMax) || offset > currentMax)
{
offsetsByPartition[tp] = offset;
}
}
var offsetsToCommit = offsetsByPartition
.Select(kv => new TopicPartitionOffset(kv.Key, new Offset(kv.Value + 1)))
.ToList();
consumer.Commit(offsetsToCommit);
}
messages.Clear();
}
startTime = DateTime.UtcNow;
}
catch (ConsumeException ex) when (KafkaPollyPipeline.IsRecoverableError(ex))
{
_logger.LogError(ex, $"{string.Join("", topics)} 消息消费失败: {ex.Error.Reason}");
throw; // 抛出异常,以便重试
}
catch (KafkaException ex) when (KafkaPollyPipeline.IsRecoverableError(ex))
{
_logger.LogError(ex, $"{string.Join("", topics)} 消息消费失败: {ex.Error.Reason}");
throw; // 抛出异常,以便重试
}
catch (OperationCanceledException)
{
//ignore
}
catch (Exception ex)
{
_logger.LogError(ex, "处理批量消息时发生未知错误");
}
messages.Clear();
}
}, cts.Token);
startTime = DateTime.UtcNow;
}
catch (ConsumeException ex)
{
_logger.LogError(ex, $"{string.Join("", topics)} 消息消费失败: {ex.Error.Reason}");
}
catch (OperationCanceledException)
{
// 任务取消,正常退出
}
catch (Exception ex)
{
_logger.LogError(ex, "处理批量消息时发生未知错误");
}
}
}, cts.Token);
await Task.CompletedTask;
});
}
catch (Exception ex)
{
await Task.CompletedTask;
throw;
}
}
@ -387,7 +474,15 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <param name="consumeTimeout">消费等待时间</param>
public async Task SubscribeBatchAsync<TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class
{
await SubscribeBatchAsync<TValue>(new[] { topic }, messageBatchHandler, groupId, batchSize, batchTimeout, consumeTimeout);
try
{
await SubscribeBatchAsync(new[] { topic }, messageBatchHandler, groupId, batchSize, batchTimeout, consumeTimeout);
}
catch (Exception ex)
{
throw;
}
}
@ -402,111 +497,126 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// <param name="batchSize">批次大小</param>
/// <param name="batchTimeout">批次超时时间</param>
/// <param name="consumeTimeout">消费等待时间</param>
public async Task SubscribeBatchAsync<TValue>(string[] topics,Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100,TimeSpan? batchTimeout = null,TimeSpan? consumeTimeout = null)where TValue : class
public async Task SubscribeBatchAsync<TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class
{
var consumerKey = typeof(KafkaConsumer<string, TValue>);
var cts = new CancellationTokenSource();
//var consumer = _consumerStore.GetOrAdd(consumerKey, _ =>
// (
// CreateConsumer<string, TValue>(groupId),
// cts
// )).Consumer as IConsumer<string, TValue>;
var consumer= CreateConsumer<string, TValue> (groupId);
consumer!.Subscribe(topics);
var timeout = batchTimeout ?? TimeSpan.FromSeconds(5); // 默认超时时间调整为5秒
_ = Task.Run(async () =>
try
{
var messages = new List<(TValue Value, TopicPartitionOffset Offset)>();
//var messages = new List<ConsumeResult<TKey, TValue>>();
var startTime = DateTime.UtcNow;
while (!cts.IsCancellationRequested)
await _kafkaPollyPipeline.KafkaPipeline.ExecuteAsync(async token =>
{
try
{
// 非阻塞快速累积消息
while (messages.Count < batchSize && (DateTime.UtcNow - startTime) < timeout)
{
var result = consumer.Consume(TimeSpan.Zero); // 非阻塞调用
if (result != null)
var consumerKey = $"{groupId}_{string.Join("_", topics)}_{typeof(Ignore).Name}_{typeof(TValue).Name}";
var cts = new CancellationTokenSource();
var consumer = _consumerStore.GetOrAdd(consumerKey, _ =>
(
CreateConsumer<Ignore, TValue>(groupId),
cts
)).Consumer as IConsumer<Ignore, TValue>;
consumer!.Subscribe(topics);
var timeout = batchTimeout ?? TimeSpan.FromSeconds(5); // 默认超时时间调整为5秒
_ = Task.Run(async () =>
{
var messages = new List<(TValue Value, TopicPartitionOffset Offset)>();
var startTime = DateTime.UtcNow;
while (!cts.IsCancellationRequested)
{
try
{
if (result.IsPartitionEOF)
// 非阻塞快速累积消息
while (messages.Count < batchSize && (DateTime.UtcNow - startTime) < timeout)
{
//_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);
await Task.Delay(10, cts.Token);
}
else if (result.Message.Value != null)
{
if (_kafkaOptionConfig.EnableFilter)
var result = consumer.Consume(TimeSpan.Zero); // 非阻塞调用
if (result != null)
{
var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) } };
// 检查 Header 是否符合条件
if (!headersFilter.Match(result.Message.Headers))
if (result.IsPartitionEOF)
{
//consumer.Commit(result); // 提交偏移量
// 跳过消息
continue;
//_logger.LogInformation("Kafka消费: {Topic} 分区 {Partition} 已消费完", result.Topic, result.Partition);
await Task.Delay(DelayTime, cts.Token);
}
else if (result.Message.Value != null)
{
if (_kafkaOptionConfig.EnableFilter)
{
var headersFilter = new HeadersFilter { { "route-key", Encoding.UTF8.GetBytes(_applicationOptions.ServerTagName) } };
// 检查 Header 是否符合条件
if (!headersFilter.Match(result.Message.Headers))
{
consumer.Commit(result); // 提交偏移量
// 跳过消息
continue;
}
}
messages.Add((result.Message.Value, result.TopicPartitionOffset));
}
}
messages.Add((result.Message.Value, result.TopicPartitionOffset));
//messages.Add(result.Message.Value);
}
}
else
{
// 无消息时短暂等待
await Task.Delay(10, cts.Token);
}
}
// 处理批次
if (messages.Count > 0)
{
bool success = await messageBatchHandler(messages.Select(m => m.Value).ToList());
if (success)
{
var offsetsByPartition = new Dictionary<TopicPartition, long>();
foreach (var msg in messages)
{
var tp = msg.Offset.TopicPartition;
var offset = msg.Offset.Offset;
if (!offsetsByPartition.TryGetValue(tp, out var currentMax) || offset > currentMax)
else
{
offsetsByPartition[tp] = offset;
// 无消息时短暂等待
await Task.Delay(DelayTime, cts.Token);
}
}
var offsetsToCommit = offsetsByPartition
.Select(kv => new TopicPartitionOffset(kv.Key, new Offset(kv.Value + 1)))
.ToList();
consumer.Commit(offsetsToCommit);
// 处理批次
if (messages.Count > 0)
{
bool success = await messageBatchHandler(messages.Select(m => m.Value).ToList());
if (success)
{
var offsetsByPartition = new Dictionary<TopicPartition, long>();
foreach (var msg in messages)
{
var tp = msg.Offset.TopicPartition;
var offset = msg.Offset.Offset;
if (!offsetsByPartition.TryGetValue(tp, out var currentMax) || offset > currentMax)
{
offsetsByPartition[tp] = offset;
}
}
var offsetsToCommit = offsetsByPartition
.Select(kv => new TopicPartitionOffset(kv.Key, new Offset(kv.Value + 1)))
.ToList();
consumer.Commit(offsetsToCommit);
}
messages.Clear();
}
startTime = DateTime.UtcNow;
}
catch (ConsumeException ex) when (KafkaPollyPipeline.IsRecoverableError(ex))
{
_logger.LogError(ex, $"消息消费失败: {ex.Error.Reason}");
throw; // 抛出异常,以便重试
}
catch (KafkaException ex) when (KafkaPollyPipeline.IsRecoverableError(ex))
{
_logger.LogError(ex, $"{string.Join("", topics)} 消息消费失败: {ex.Error.Reason}");
throw; // 抛出异常,以便重试
}
catch (OperationCanceledException)
{
//ignore
}
catch (Exception ex)
{
_logger.LogError(ex, "处理批量消息时发生未知错误");
}
messages.Clear();
}
}, cts.Token);
startTime = DateTime.UtcNow;
}
catch (ConsumeException ex)
{
_logger.LogError(ex, $"消息消费失败: {ex.Error.Reason}");
}
catch (OperationCanceledException)
{
// 任务取消,正常退出
}
catch (Exception ex)
{
_logger.LogError(ex, "处理批量消息时发生未知错误");
}
}
}, cts.Token);
await Task.CompletedTask;
});
}
catch (Exception ex)
{
await Task.CompletedTask;
throw;
}
}
@ -515,14 +625,22 @@ namespace JiShe.CollectBus.Kafka.Consumer
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TValue"></typeparam>
public void Unsubscribe<TKey, TValue>() where TKey : notnull where TValue : class
public void Unsubscribe<TKey, TValue>(string[] topics, string? groupId) where TKey : notnull where TValue : class
{
var consumerKey = typeof((TKey, TValue));
if (_consumerStore.TryRemove(consumerKey, out var entry))
try
{
entry.CTS.Cancel();
(entry.Consumer as IDisposable)?.Dispose();
entry.CTS.Dispose();
var consumerKey = $"{groupId}_{string.Join("_", topics)}_{typeof(TKey).Name}_{typeof(TValue).Name}";
if (_consumerStore.TryRemove(consumerKey, out var entry))
{
entry.CTS.Cancel();
(entry.Consumer as IDisposable)?.Dispose();
entry.CTS.Dispose();
}
}
catch (Exception ex)
{
throw;
}
}

View File

@ -1,46 +1,50 @@
using Confluent.Kafka;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Consumer;
namespace JiShe.CollectBus.Kafka.Consumer
public interface IConsumerService
{
public interface IConsumerService
{
Task SubscribeAsync<TKey, TValue>(string topic, Func<TKey, TValue, Task<bool>> messageHandler, string? groupId=null) where TKey : notnull where TValue : class;
Task SubscribeAsync<TKey, TValue>(string topic, Func<TKey, TValue, Task<bool>> messageHandler,
string? groupId = null) where TKey : notnull where TValue : class;
/// <summary>
/// 订阅消息
/// </summary>
/// <typeparam name="TValue"></typeparam>
/// <param name="topic"></param>
/// <param name="messageHandler"></param>
/// <returns></returns>
Task SubscribeAsync<TValue>(string topic, Func<TValue, Task<bool>> messageHandler, string? groupId = null) where TValue : class;
/// <summary>
/// 订阅消息
/// </summary>
/// <typeparam name="TValue"></typeparam>
/// <param name="topic"></param>
/// <param name="messageHandler"></param>
/// <returns></returns>
Task SubscribeAsync<TValue>(string topic, Func<TValue, Task<bool>> messageHandler, string? groupId = null)
where TValue : class;
Task SubscribeAsync<TKey, TValue>(string[] topics, Func<TKey, TValue, Task<bool>> messageHandler, string? groupId) where TKey : notnull where TValue : class;
Task SubscribeAsync<TKey, TValue>(string[] topics, Func<TKey, TValue, Task<bool>> messageHandler, string? groupId)
where TKey : notnull where TValue : class;
/// <summary>
/// 订阅消息
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TValue"></typeparam>
/// <param name="topics"></param>
/// <param name="messageHandler"></param>
/// <returns></returns>
Task SubscribeAsync<TValue>(string[] topics, Func<TValue, Task<bool>> messageHandler, string? groupId = null) where TValue : class;
/// <summary>
/// 订阅消息
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TValue"></typeparam>
/// <param name="topics"></param>
/// <param name="messageHandler"></param>
/// <returns></returns>
Task SubscribeAsync<TValue>(string[] topics, Func<TValue, Task<bool>> messageHandler, string? groupId = null)
where TValue : class;
Task SubscribeBatchAsync<TKey, TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class;
Task SubscribeBatchAsync<TKey, TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler,
string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null)
where TKey : notnull where TValue : class;
Task SubscribeBatchAsync<TKey, TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null) where TKey : notnull where TValue : class;
Task SubscribeBatchAsync<TKey, TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler,
string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null)
where TKey : notnull where TValue : class;
Task SubscribeBatchAsync<TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class;
Task SubscribeBatchAsync<TValue>(string topic, Func<List<TValue>, Task<bool>> messageBatchHandler,
string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null)
where TValue : class;
Task SubscribeBatchAsync<TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler, string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null) where TValue : class;
Task SubscribeBatchAsync<TValue>(string[] topics, Func<List<TValue>, Task<bool>> messageBatchHandler,
string? groupId = null, int batchSize = 100, TimeSpan? batchTimeout = null, TimeSpan? consumeTimeout = null)
where TValue : class;
void Unsubscribe<TKey, TValue>() where TKey : notnull where TValue : class;
}
}
void Unsubscribe<TKey, TValue>(string[] topics, string groupId) where TKey : notnull where TValue : class;
}

View File

@ -1,30 +1,22 @@
using Confluent.Kafka;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal
namespace JiShe.CollectBus.Kafka.Internal;
/// <summary>
/// 消息头过滤器
/// </summary>
public class HeadersFilter : Dictionary<string, byte[]>
{
/// <summary>
/// 消息头过滤器
/// 判断Headers是否匹配
/// </summary>
public class HeadersFilter : Dictionary<string, byte[]>
/// <param name="headers"></param>
/// <returns></returns>
public bool Match(Headers headers)
{
/// <summary>
/// 判断Headers是否匹配
/// </summary>
/// <param name="headers"></param>
/// <returns></returns>
public bool Match(Headers headers)
{
foreach (var kvp in this)
{
if (!headers.TryGetLastBytes(kvp.Key, out var value) || !value.SequenceEqual(kvp.Value))
return false;
}
return true;
}
foreach (var kvp in this)
if (!headers.TryGetLastBytes(kvp.Key, out var value) || !value.SequenceEqual(kvp.Value))
return false;
return true;
}
}
}

View File

@ -1,18 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal;
namespace JiShe.CollectBus.Kafka.Internal
/// <summary>
/// Kafka订阅者
/// <para>
/// 订阅者需要继承此接口并需要依赖注入,并使用<see cref="KafkaSubscribeAttribute" />标记
/// </para>
/// </summary>
public interface IKafkaSubscribe
{
/// <summary>
/// Kafka订阅者
/// <para>
/// 订阅者需要继承此接口并需要依赖注入,并使用<see cref="KafkaSubscribeAttribute"/>标记
/// </para>
/// </summary>
public interface IKafkaSubscribe
{
}
}
}

View File

@ -1,21 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal;
namespace JiShe.CollectBus.Kafka.Internal
public interface ISubscribeAck
{
public interface ISubscribeAck
{
/// <summary>
/// 是否成功标记
/// </summary>
bool Ack { get; set; }
/// <summary>
/// 是否成功标记
/// </summary>
bool Ack { get; set; }
/// <summary>
/// 消息
/// </summary>
string? Msg { get; set; }
}
}
/// <summary>
/// 消息
/// </summary>
string? Msg { get; set; }
}

View File

@ -1,68 +1,52 @@
using Confluent.Kafka;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal
namespace JiShe.CollectBus.Kafka.Internal;
public class KafkaOptionConfig
{
public class KafkaOptionConfig
{
/// <summary>
/// kafka地址
/// </summary>
public string BootstrapServers { get; set; } = null!;
/// <summary>
/// kafka地址
/// </summary>
public string BootstrapServers { get; set; } = null!;
/// <summary>
/// kafka主题副本数量
/// </summary>
public short KafkaReplicationFactor { get; set; }
/// <summary>
/// 服务器标识
/// </summary>
public string ServerTagName { get; set; }= "KafkaFilterKey";
/// <summary>
/// kafka主题分区数量
/// </summary>
public int NumPartitions { get; set; }
/// <summary>
/// kafka主题副本数量
/// </summary>
public short KafkaReplicationFactor { get; set; }
/// <summary>
/// 是否开启过滤器
/// </summary>
public bool EnableFilter { get; set; } = true;
/// <summary>
/// kafka主题分区数量
/// </summary>
public int NumPartitions { get; set; }
/// <summary>
/// 是否开启认证
/// </summary>
public bool EnableAuthorization { get; set; } = false;
/// <summary>
/// 是否开启过滤器
/// </summary>
public bool EnableFilter { get; set; }= true;
/// <summary>
/// 安全协议
/// </summary>
public SecurityProtocol SecurityProtocol { get; set; } = SecurityProtocol.SaslPlaintext;
/// <summary>
/// 是否开启认证
/// </summary>
public bool EnableAuthorization { get; set; } = false;
/// <summary>
/// 认证方式
/// </summary>
public SaslMechanism SaslMechanism { get; set; } = SaslMechanism.Plain;
/// <summary>
/// 安全协议
/// </summary>
public SecurityProtocol SecurityProtocol { get; set; } = SecurityProtocol.SaslPlaintext;
/// <summary>
/// 用户名
/// </summary>
public string? SaslUserName { get; set; }
/// <summary>
/// 认证方式
/// </summary>
public SaslMechanism SaslMechanism { get; set; }= SaslMechanism.Plain;
/// <summary>
/// 密码
/// </summary>
public string? SaslPassword { get; set; }
/// <summary>
/// 用户名
/// </summary>
public string? SaslUserName { get; set; }
/// <summary>
/// 密码
/// </summary>
public string? SaslPassword { get; set; }
/// <summary>
/// 首次采集时间
/// </summary>
public DateTime FirstCollectionTime { get; set; }
}
}
}

View File

@ -0,0 +1,105 @@
using Confluent.Kafka;
using Microsoft.Extensions.Logging;
using Polly;
using Polly.CircuitBreaker;
using Polly.Retry;
namespace JiShe.CollectBus.Kafka.Internal
{
public class KafkaPollyPipeline
{
private readonly ILogger<KafkaPollyPipeline> _logger;
public KafkaPollyPipeline(ILogger<KafkaPollyPipeline> logger)
{
_logger= logger;
}
/// <summary>
/// 判断是否可恢复的异常
/// </summary>
/// <param name="ex"></param>
/// <returns></returns>
public static bool IsRecoverableError(Exception ex)
{
var errorList= new List<ErrorCode>
{
ErrorCode.GroupLoadInProgress,
ErrorCode.Local_Retry,
ErrorCode.Local_MaxPollExceeded,
ErrorCode.RequestTimedOut,
ErrorCode.LeaderNotAvailable,
ErrorCode.NotLeaderForPartition,
ErrorCode.RebalanceInProgress,
ErrorCode.NotCoordinatorForGroup,
ErrorCode.NetworkException,
ErrorCode.GroupCoordinatorNotAvailable,
ErrorCode.InvalidGroupId,
ErrorCode.IllegalGeneration
};
return ex switch
{
ConsumeException kafkaEx => errorList.Contains(kafkaEx.Error.Code),
KafkaException kafkaEx =>kafkaEx.Error.IsFatal && errorList.Contains(kafkaEx.Error.Code),
_ => false
};
}
/// <summary>
/// 创建重试 + 断路器
/// </summary>
/// <returns></returns>
public ResiliencePipeline KafkaPipeline
{
get
{
// 组合重试 + 断路器
ResiliencePipeline pipeline = new ResiliencePipelineBuilder()
.AddRetry(new RetryStrategyOptions
{
ShouldHandle = args => args.Outcome.Exception switch
{
not null when IsRecoverableError(args.Outcome.Exception) =>
PredicateResult.True(),
_ => PredicateResult.False()
},
Delay = TimeSpan.FromSeconds(2),
OnRetry = args =>
{
_logger.LogWarning($"重试中... 第 {args.AttemptNumber} 次,原因: {args.Outcome.Exception?.Message}");
return default;
}
})
.AddCircuitBreaker(new CircuitBreakerStrategyOptions
{
ShouldHandle = args => args.Outcome.Exception switch
{
not null when IsRecoverableError(args.Outcome.Exception) =>
PredicateResult.True(),
_ => PredicateResult.False()
},
FailureRatio = 0.8, // 80% 失败触发熔断
SamplingDuration = TimeSpan.FromSeconds(10),
MinimumThroughput = 4, // 至少4次调用才计算失败率
BreakDuration = TimeSpan.FromSeconds(10),
OnOpened = args =>
{
_logger.LogWarning($"熔断器开启,等待 {args.BreakDuration} 后重试");
return default;
},
OnClosed = _ =>
{
_logger.LogWarning("熔断器关闭,再次开始重试");
return default;
}
})
.Build();
return pipeline;
}
}
}
}

View File

@ -1,113 +1,103 @@
using Newtonsoft.Json;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Collections;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Internal
namespace JiShe.CollectBus.Kafka.Internal;
/// <summary>
/// 反射辅助类
/// </summary>
public static class ReflectionHelper
{
/// <summary>
/// 反射辅助类
/// 集合类型
/// Item1参数类型
/// Item2集合元素类型
/// </summary>
public static class ReflectionHelper
public static Tuple<Type, Type?> GetParameterTypeInfo(this MethodInfo method, int parameterIndex = 0)
{
/// <summary>
///集合类型
///Item1参数类型
///Item2集合元素类型
/// </summary>
public static Tuple<Type,Type?> GetParameterTypeInfo(this MethodInfo method, int parameterIndex=0)
{
// 参数校验
if (method == null) throw new ArgumentNullException(nameof(method));
var parameters = method.GetParameters();
if (parameterIndex < 0 || parameterIndex >= parameters.Length)
throw new ArgumentOutOfRangeException(nameof(parameterIndex));
// 参数校验
if (method == null) throw new ArgumentNullException(nameof(method));
var parameters = method.GetParameters();
if (parameterIndex < 0 || parameterIndex >= parameters.Length)
throw new ArgumentOutOfRangeException(nameof(parameterIndex));
ParameterInfo param = parameters[parameterIndex];
Type paramType = param.ParameterType;
Type? elementType = null;
var param = parameters[parameterIndex];
var paramType = param.ParameterType;
Type? elementType = null;
// 判断是否是集合类型(排除字符串)
if (paramType != typeof(string) && IsEnumerableType(paramType))
{
elementType = GetEnumerableElementType(paramType);
}
// 判断是否是集合类型(排除字符串)
if (paramType != typeof(string) && IsEnumerableType(paramType))
elementType = GetEnumerableElementType(paramType);
return Tuple.Create(paramType, elementType);
}
return Tuple.Create(paramType, elementType);
}
/// <summary>
/// 判断是否是集合类型(排除字符串)
/// </summary>
public static bool IsEnumerableType(this Type type)
{
return type.IsArray
|| (type.IsGenericType && type.GetInterfaces()
.Any(t => t.IsGenericType
&& t.GetGenericTypeDefinition() == typeof(IEnumerable<>)))
|| type.GetInterfaces().Any(t => t == typeof(System.Collections.IEnumerable));
}
/// <summary>
/// 判断是否是集合类型(排除字符串)
/// </summary>
public static bool IsEnumerableType(this Type type)
{
return type.IsArray
|| (type.IsGenericType && type.GetInterfaces()
.Any(t => t.IsGenericType
&& t.GetGenericTypeDefinition() == typeof(IEnumerable<>)))
|| type.GetInterfaces().Any(t => t == typeof(IEnumerable));
}
/// <summary>
/// 获取集合元素的类型
/// </summary>
public static Type? GetEnumerableElementType(this Type type)
{
// 处理数组类型
if (type.IsArray)
return type.GetElementType();
/// <summary>
/// 获取集合元素的类型
/// </summary>
public static Type? GetEnumerableElementType(this Type type)
{
// 处理数组类型
if (type.IsArray)
return type.GetElementType();
// 处理直接实现IEnumerable<T>的类型如IEnumerable<int>本身)
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(IEnumerable<>))
return type.GetGenericArguments()[0];
// 处理直接实现IEnumerable<T>的类型如IEnumerable<int>本身)
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(IEnumerable<>))
return type.GetGenericArguments()[0];
// 处理通过接口实现IEnumerable<T>的泛型集合如List<T>
var genericEnumerable = type.GetInterfaces()
.FirstOrDefault(t => t.IsGenericType
&& t.GetGenericTypeDefinition() == typeof(IEnumerable<>));
if (genericEnumerable != null)
return genericEnumerable.GetGenericArguments()[0];
// 处理通过接口实现IEnumerable<T>的泛型集合如List<T>
var genericEnumerable = type.GetInterfaces()
.FirstOrDefault(t => t.IsGenericType
&& t.GetGenericTypeDefinition() == typeof(IEnumerable<>));
if (genericEnumerable != null)
return genericEnumerable.GetGenericArguments()[0];
// 处理非泛型集合类型(如 ArrayList
if (typeof(IEnumerable).IsAssignableFrom(type) && type == typeof(ArrayList))
return typeof(ArrayList);
// 返回null表示无法确定元素类型
return null;
}
// 处理非泛型集合类型(如 ArrayList
if (typeof(IEnumerable).IsAssignableFrom(type) && type == typeof(ArrayList))
return typeof(ArrayList);
// 返回null表示无法确定元素类型
return null;
}
// <summary>
/// 判断是否使用强转换
/// </summary>
/// <param name="targetType">目标类型</param>
/// <returns></returns>
public static bool IsConvertType(this Type targetType)
{
// 处理可空类型
Type underlyingType = Nullable.GetUnderlyingType(targetType) ?? targetType;
// 情况1值类型或基元类型如 int、DateTime
if (underlyingType.IsValueType || underlyingType.IsPrimitive)
return true;
// 情况2字符串类型直接赋值
else if (underlyingType == typeof(string))
return true;
// 情况3枚举类型处理
//else if (underlyingType.IsEnum)
//{
// if (Enum.IsDefined(underlyingType, msg))
// {
// convertedValue = Enum.Parse(underlyingType, msg.ToString());
// return true;
// }
// return false;
//}
return false;
}
/// <summary>
/// 判断是否使用强转换
/// </summary>
/// <param name="targetType"></param>
/// <returns></returns>
public static bool IsConvertType(this Type targetType)
{
// 处理可空类型
var underlyingType = Nullable.GetUnderlyingType(targetType) ?? targetType;
// 情况1值类型或基元类型如 int、DateTime
if (underlyingType.IsValueType || underlyingType.IsPrimitive)
return true;
// 情况2字符串类型直接赋值
if (underlyingType == typeof(string))
return true;
// 情况3枚举类型处理
//else if (underlyingType.IsEnum)
//{
// if (Enum.IsDefined(underlyingType, msg))
// {
// convertedValue = Enum.Parse(underlyingType, msg.ToString());
// return true;
// }
// return false;
//}
return false;
}
}

View File

@ -1,75 +1,62 @@
using Confluent.Kafka;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace JiShe.CollectBus.Kafka.Internal;
namespace JiShe.CollectBus.Kafka.Internal
public class SubscribeResult : ISubscribeAck
{
public class SubscribeResult: ISubscribeAck
/// <summary>
/// 是否成功
/// </summary>
public bool Ack { get; set; }
/// <summary>
/// 消息
/// </summary>
public string? Msg { get; set; }
/// <summary>
/// 成功
/// </summary>
/// <param name="msg">消息</param>
public SubscribeResult Success(string? msg = null)
{
/// <summary>
/// 是否成功
/// </summary>
public bool Ack { get; set; }
/// <summary>
/// 消息
/// </summary>
public string? Msg { get; set; }
/// <summary>
/// 成功
/// </summary>
/// <param name="msg">消息</param>
public SubscribeResult Success(string? msg = null)
{
Ack = true;
Msg = msg;
return this;
}
/// <summary>
/// 失败
/// </summary>
/// <param name="code"></param>
/// <param name="msg"></param>
/// <param name="data"></param>
/// <returns></returns>
public SubscribeResult Fail(string? msg = null)
{
Msg = msg;
Ack = false;
return this;
}
Ack = true;
Msg = msg;
return this;
}
public static partial class SubscribeAck
/// <summary>
/// 失败
/// </summary>
/// <param name="msg"></param>
/// <returns></returns>
public SubscribeResult Fail(string? msg = null)
{
/// <summary>
/// 成功
/// </summary>
/// <param name="msg">消息</param>
/// <returns></returns>
public static ISubscribeAck Success(string? msg = null)
{
return new SubscribeResult().Success(msg);
}
/// <summary>
/// 失败
/// </summary>
/// <param name="msg">消息</param>
/// <returns></returns>
public static ISubscribeAck Fail(string? msg = null)
{
return new SubscribeResult().Fail(msg);
}
Msg = msg;
Ack = false;
return this;
}
}
public static class SubscribeAck
{
/// <summary>
/// 成功
/// </summary>
/// <param name="msg">消息</param>
/// <returns></returns>
public static ISubscribeAck Success(string? msg = null)
{
return new SubscribeResult().Success(msg);
}
/// <summary>
/// 失败
/// </summary>
/// <param name="msg">消息</param>
/// <returns></returns>
public static ISubscribeAck Fail(string? msg = null)
{
return new SubscribeResult().Fail(msg);
}
}

View File

@ -8,6 +8,8 @@
<ItemGroup>
<PackageReference Include="Confluent.Kafka" Version="2.9.0" />
<PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
<PackageReference Include="Polly.Core" Version="8.5.2" />
<PackageReference Include="Volo.Abp.AspNetCore" Version="8.3.3" />
<PackageReference Include="Volo.Abp.Core" Version="8.3.3" />
</ItemGroup>

View File

@ -1,360 +0,0 @@
using Confluent.Kafka;
using JiShe.CollectBus.Common.Consts;
using JiShe.CollectBus.Common.Extensions;
using JiShe.CollectBus.Common.Helpers;
using JiShe.CollectBus.Kafka.AdminClient;
using JiShe.CollectBus.Kafka.Attributes;
using JiShe.CollectBus.Kafka.Consumer;
using JiShe.CollectBus.Kafka.Internal;
using JiShe.CollectBus.Kafka.Serialization;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Mvc.Abstractions;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq.Expressions;
using System.Reflection;
using System.Text.Json;
using System.Threading.Tasks;
using YamlDotNet.Core.Tokens;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace JiShe.CollectBus.Kafka
{
public static class KafkaSubcribesExtensions
{
public static void UseInitKafkaTopic(this IServiceProvider provider)
{
//初始化主题信息
var kafkaAdminClient = provider.GetRequiredService<IAdminClientService>();
var kafkaOptions = provider.GetRequiredService<IOptions<KafkaOptionConfig>>();
List<string> topics = ProtocolConstExtensions.GetAllTopicNamesByIssued();
topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived());
foreach (var item in topics)
{
kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor).ConfigureAwait(false).GetAwaiter().GetResult();
}
}
/// <summary>
/// 添加Kafka订阅
/// </summary>
/// <param name="app"></param>
/// <param name="assembly"></param>
public static void UseKafkaSubscribe(this IServiceProvider provider)
{
var lifetime = provider.GetRequiredService<IHostApplicationLifetime>();
var kafkaOptions = provider.GetRequiredService<IOptions<KafkaOptionConfig>>();
lifetime.ApplicationStarted.Register(() =>
{
var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>();
int threadCount = 0;
int topicCount = 0;
var assemblyPath = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location);
if (string.IsNullOrWhiteSpace(assemblyPath))
{
logger.LogInformation($"kafka订阅未能找到程序路径");
return;
}
var dllFiles = Directory.GetFiles(assemblyPath, "*.dll");
foreach (var file in dllFiles)
{
// 跳过已加载的程序集
var assemblyName = AssemblyName.GetAssemblyName(file);
var existingAssembly = AppDomain.CurrentDomain.GetAssemblies()
.FirstOrDefault(a => a.GetName().FullName == assemblyName.FullName);
var assembly = existingAssembly ?? Assembly.LoadFrom(file);
// 实现IKafkaSubscribe接口
var subscribeTypes = assembly.GetTypes().Where(type =>
typeof(IKafkaSubscribe).IsAssignableFrom(type) &&
!type.IsAbstract && !type.IsInterface).ToList(); ;
if (subscribeTypes.Count == 0)
continue;
foreach (var subscribeType in subscribeTypes)
{
var subscribes = provider.GetServices(subscribeType).ToList();
subscribes.ForEach(subscribe =>
{
if (subscribe != null)
{
Tuple<int, int> tuple = BuildKafkaSubscribe(subscribe, provider, logger, kafkaOptions.Value);
threadCount += tuple.Item1;
topicCount += tuple.Item2;
}
});
}
}
logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程");
});
}
public static void UseKafkaSubscribersAsync(this IApplicationBuilder app, Assembly assembly)
{
var provider = app.ApplicationServices;
var lifetime = provider.GetRequiredService<IHostApplicationLifetime>();
var kafkaOptions = provider.GetRequiredService<IOptions<KafkaOptionConfig>>();
lifetime.ApplicationStarted.Register(() =>
{
var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>();
int threadCount = 0;
int topicCount = 0;
var subscribeTypes = assembly.GetTypes()
.Where(t => typeof(IKafkaSubscribe).IsAssignableFrom(t))
.ToList();
if (subscribeTypes.Count == 0) return;
foreach (var subscribeType in subscribeTypes)
{
var subscribes = provider.GetServices(subscribeType).ToList();
subscribes.ForEach(subscribe =>
{
if (subscribe != null)
{
Tuple<int, int> tuple = BuildKafkaSubscribe(subscribe, provider, logger, kafkaOptions.Value);
threadCount += tuple.Item1;
topicCount += tuple.Item2;
}
});
}
logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程");
});
}
/// <summary>
/// 构建Kafka订阅
/// </summary>
/// <param name="subscribe"></param>
/// <param name="provider"></param>
private static Tuple<int, int> BuildKafkaSubscribe(object subscribe, IServiceProvider provider, ILogger<CollectBusKafkaModule> logger, KafkaOptionConfig kafkaOptionConfig)
{
var subscribedMethods = subscribe.GetType().GetMethods()
.Select(m => new { Method = m, Attribute = m.GetCustomAttribute<KafkaSubscribeAttribute>() })
.Where(x => x.Attribute != null)
.ToArray();
//var configuration = provider.GetRequiredService<IConfiguration>();
int threadCount = 0;
foreach (var sub in subscribedMethods)
{
int partitionCount = 3;// kafkaOptionConfig.NumPartitions;
#if DEBUG
var adminClientService = provider.GetRequiredService<IAdminClientService>();
int topicCount = adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic);
partitionCount = partitionCount > topicCount ? topicCount : partitionCount;
#endif
//int partitionCount = sub.Attribute!.TaskCount==-1?adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic) : sub.Attribute!.TaskCount;
if (partitionCount <= 0)
partitionCount = 1;
for (int i = 0; i < partitionCount; i++)
{
//if (sub.Attribute!.Topic == ProtocolConst.SubscriberLoginReceivedEventName)
Task.Run(() => StartConsumerAsync(provider, sub.Attribute!, sub.Method, subscribe, logger));
threadCount++;
}
}
return Tuple.Create(threadCount, subscribedMethods.Length);
}
/// <summary>
/// 启动后台消费线程
/// </summary>
/// <param name="config"></param>
/// <param name="attr"></param>
/// <param name="method"></param>
/// <param name="consumerInstance"></param>
/// <returns></returns>
private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr, MethodInfo method, object subscribe, ILogger<CollectBusKafkaModule> logger)
{
var consumerService = provider.GetRequiredService<IConsumerService>();
if (attr.EnableBatch)
{
await consumerService.SubscribeBatchAsync<dynamic>(attr.Topic, async (message) =>
{
try
{
#if DEBUG
logger.LogInformation($"kafka批量消费消息:{message.Serialize()}");
#endif
// 处理消息
return await ProcessMessageAsync(message.ToList(), method, subscribe);
}
catch (ConsumeException ex)
{
// 处理消费错误
logger.LogError($"kafka批量消费异常:{ex.Message}");
}
return await Task.FromResult(false);
}, attr.GroupId, attr.BatchSize, attr.BatchTimeout);
}
else
{
await consumerService.SubscribeAsync<dynamic>(attr.Topic, async (message) =>
{
try
{
#if DEBUG
logger.LogInformation($"kafka消费消息:{message}");
#endif
// 处理消息
return await ProcessMessageAsync(new List<object>() { message }, method, subscribe);
}
catch (ConsumeException ex)
{
// 处理消费错误
logger.LogError($"kafka消费异常:{ex.Message}");
}
return await Task.FromResult(false);
}, attr.GroupId);
}
}
/// <summary>
/// 处理消息
/// </summary>
/// <param name="message"></param>
/// <param name="method"></param>
/// <param name="subscribe"></param>
/// <returns></returns>
private static async Task<bool> ProcessMessageAsync(List<dynamic> messages, MethodInfo method, object subscribe)
{
var parameters = method.GetParameters();
bool isGenericTask = method.ReturnType.IsGenericType
&& method.ReturnType.GetGenericTypeDefinition() == typeof(Task<>);
bool existParameters = parameters.Length > 0;
object[]? executeParameters = null;
if (existParameters)
{
IList? list = null;
Tuple<Type, Type?> tuple = method.GetParameterTypeInfo();
bool isEnumerable = false;
if (tuple.Item2 != null)
{
Type listType = typeof(List<>).MakeGenericType(tuple.Item2);
list = (IList)Activator.CreateInstance(listType)!;
isEnumerable = tuple.Item2.IsConvertType();
}
else
{
isEnumerable = tuple.Item1.IsConvertType();
}
#region
//foreach (var msg in messages)
//{
// if (tuple.Item2 != null)
// {
// if (isEnumerable)
// {
// var parameterType = parameters[0].ParameterType;
// var data=messages?.Serialize().Deserialize(parameterType);
// messageObj = data!=null? new[] { data }:null;
// break;
// }
// else
// {
// // 集合类型
// var data = msg?.Serialize().Deserialize(tuple.Item2) /*isEnumerable ? Convert.ChangeType(msg, tuple.Item2) : msg?.Serialize().Deserialize(tuple.Item2)*/;
// if (data != null)
// list?.Add(data);
// }
// }
// else
// {
// // (dynamic)Convert.ChangeType(msg, tuple.Item1)
// using (var stream = new MemoryStream(msg))
// {
// var data1= System.Text.Json.JsonSerializer.Deserialize(stream, tuple.Item1);
// }
// var data = isEnumerable ? System.Text.Json.JsonSerializer.Deserialize(msg, tuple.Item1): msg?.ToString()?.Deserialize(tuple.Item1);
// if (data != null)
// messageObj = new[] { data };
// }
//}
//if (tuple.Item2 != null && list != null && list.Count > 0)
//{
// messageObj = new[] { list };
//}
#endregion
var parameterDescriptors = method.GetParameters();
executeParameters = new object?[parameterDescriptors.Length];
for (var i = 0; i < parameterDescriptors.Length; i++)
{
foreach (var item in messages)
{
object? tempParameter=null;
var parameterDescriptor = parameterDescriptors[i];
if (KafkaSerialization.IsJsonType(item))
{
tempParameter = KafkaSerialization.Deserialize(item, tuple.Item2 != null? tuple.Item2: parameterDescriptor.ParameterType);
}
else
{
var converter = TypeDescriptor.GetConverter(parameterDescriptor.ParameterType);
if (converter.CanConvertFrom(item.GetType()))
{
tempParameter = converter.ConvertFrom(item);
}
else
{
if (parameterDescriptor.ParameterType.IsInstanceOfType(item))
tempParameter = item;
else
tempParameter =Convert.ChangeType(item, parameterDescriptor.ParameterType);
}
}
if (tuple.Item2 == null)
{
executeParameters[i] = tempParameter;
}
else
{
list.Add(tempParameter);
}
}
if(list!=null && list.Count>0)
executeParameters[i] = list;
}
}
var result = method.Invoke(subscribe, executeParameters);
if (result is Task<ISubscribeAck> genericTask)
{
await genericTask.ConfigureAwait(false);
return genericTask.Result.Ack;
}
else if (result is Task nonGenericTask)
{
await nonGenericTask.ConfigureAwait(false);
return true;
}
else if (result is ISubscribeAck ackResult)
{
return ackResult.Ack;
}
return false;
}
}
}

View File

@ -0,0 +1,404 @@
using Confluent.Kafka;
using JiShe.CollectBus.Common.Consts;
using JiShe.CollectBus.Common.Extensions;
using JiShe.CollectBus.Common.Helpers;
using JiShe.CollectBus.Kafka.AdminClient;
using JiShe.CollectBus.Kafka.Attributes;
using JiShe.CollectBus.Kafka.Consumer;
using JiShe.CollectBus.Kafka.Internal;
using JiShe.CollectBus.Kafka.Serialization;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Collections;
using System.ComponentModel;
using System.Reflection;
namespace JiShe.CollectBus.Kafka
{
public static class KafkaSubscribeExtensions
{
private static long _threadCount = 0;
private static long _topicSubscribeCount = 0;
private static long _threadStartCount = 0;
public static void UseInitKafkaTopic(this IServiceProvider provider)
{
//初始化主题信息
var kafkaAdminClient = provider.GetRequiredService<IAdminClientService>();
var kafkaOptions = provider.GetRequiredService<IOptions<KafkaOptionConfig>>();
var topics = ProtocolConstExtensions.GetAllTopicNamesByIssued();
topics.AddRange(ProtocolConstExtensions.GetAllTopicNamesByReceived());
foreach (var item in topics)
{
kafkaAdminClient.CreateTopicAsync(item, kafkaOptions.Value.NumPartitions, kafkaOptions.Value.KafkaReplicationFactor).ConfigureAwait(false).GetAwaiter().GetResult();
}
}
/// <summary>
/// 添加Kafka订阅
/// </summary>
public static void UseKafkaSubscribe(this IServiceProvider provider)
{
var lifetime = provider.GetRequiredService<IHostApplicationLifetime>();
var kafkaOptions = provider.GetRequiredService<IOptions<KafkaOptionConfig>>();
lifetime.ApplicationStarted.Register(() =>
{
var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>();
//var threadCount = 0;
//var topicCount = 0;
var assemblyPath = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location);
if (string.IsNullOrWhiteSpace(assemblyPath))
{
logger.LogWarning($"kafka订阅未能找到程序路径");
return;
}
var dllFiles = Directory.GetFiles(assemblyPath, "*.dll");
foreach (var file in dllFiles)
{
// 跳过已加载的程序集
var assemblyName = AssemblyName.GetAssemblyName(file);
var existingAssembly = AppDomain.CurrentDomain.GetAssemblies()
.FirstOrDefault(a => a.GetName().FullName == assemblyName.FullName);
var assembly = existingAssembly ?? Assembly.LoadFrom(file);
// 实现IKafkaSubscribe接口
var subscribeTypes = assembly.GetTypes().Where(type =>
typeof(IKafkaSubscribe).IsAssignableFrom(type) &&
!type.IsAbstract && !type.IsInterface).ToList();
if (subscribeTypes.Count == 0)
continue;
// 并行处理
Parallel.ForEach(subscribeTypes, subscribeType =>
{
var subscribes = provider.GetServices(subscribeType).ToList();
Parallel.ForEach(subscribes,subscribe =>
{
if (subscribe != null)
{
Tuple<int, int> tuple = BuildKafkaSubscribe(subscribe, provider, logger, kafkaOptions.Value);
//threadCount += tuple.Item1;
//topicCount += tuple.Item2;
}
});
});
//foreach (var subscribeType in subscribeTypes)
//{
// var subscribes = provider.GetServices(subscribeType).ToList();
// subscribes.ForEach(subscribe =>
// {
// if (subscribe != null)
// {
// Tuple<int, int> tuple = BuildKafkaSubscribe(subscribe, provider, logger, kafkaOptions.Value);
// threadCount += tuple.Item1;
// topicCount += tuple.Item2;
// }
// });
//}
}
logger.LogWarning($"kafka订阅主题:{_topicSubscribeCount}数,共启动:{_threadCount}线程");
});
}
/// <summary>
/// 添加Kafka订阅
/// </summary>
public static void UseKafkaSubscribersAsync(this IApplicationBuilder app, Assembly assembly)
{
var provider = app.ApplicationServices;
var lifetime = provider.GetRequiredService<IHostApplicationLifetime>();
var kafkaOptions = provider.GetRequiredService<IOptions<KafkaOptionConfig>>();
lifetime.ApplicationStarted.Register(() =>
{
var logger = provider.GetRequiredService<ILogger<CollectBusKafkaModule>>();
int threadCount = 0;
int topicCount = 0;
var subscribeTypes = assembly.GetTypes()
.Where(t => typeof(IKafkaSubscribe).IsAssignableFrom(t))
.ToList();
if (subscribeTypes.Count == 0) return;
foreach (var subscribeType in subscribeTypes)
{
var subscribes = provider.GetServices(subscribeType).ToList();
subscribes.ForEach(subscribe =>
{
if (subscribe != null)
{
Tuple<int, int> tuple = BuildKafkaSubscribe(subscribe, provider, logger, kafkaOptions.Value);
threadCount += tuple.Item1;
topicCount += tuple.Item2;
}
});
}
logger.LogInformation($"kafka订阅主题:{topicCount}数,共启动:{threadCount}线程");
});
}
/// <summary>
/// 构建Kafka订阅
/// </summary>
private static Tuple<int, int> BuildKafkaSubscribe(object subscribe, IServiceProvider provider, ILogger<CollectBusKafkaModule> logger, KafkaOptionConfig kafkaOptionConfig)
{
var subscribedMethods = subscribe.GetType().GetMethods()
.Select(m => new { Method = m, Attribute = m.GetCustomAttribute<KafkaSubscribeAttribute>() })
.Where(x => x.Attribute != null)
.ToArray();
//var configuration = provider.GetRequiredService<IConfiguration>();
int threadCount = 0;
Parallel.ForEach(subscribedMethods, sub =>
{
Interlocked.Increment(ref _topicSubscribeCount);
int partitionCount = sub.Attribute!.TaskCount == -1 ? 3 : sub.Attribute!.TaskCount;// kafkaOptionConfig.NumPartitions;
var adminClientService = provider.GetRequiredService<IAdminClientService>();
int topicCount = adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic);
//int partitionCount = sub.Attribute!.TaskCount == -1 ? topicCount : sub.Attribute!.TaskCount;// kafkaOptionConfig.NumPartitions;
partitionCount = partitionCount > topicCount ? topicCount : partitionCount;
//partitionCount = sub.Attribute!.TaskCount == -1 ? adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic) : sub.Attribute!.TaskCount;
if (partitionCount <= 0)
partitionCount = 1;
Parallel.For(0,partitionCount, async (partition) =>
{
Interlocked.Increment(ref _threadCount);
//Task.Run(() => StartConsumerAsync(provider, sub.Attribute!, sub.Method, subscribe, logger));
//threadCount++;
await StartConsumerAsync(provider, sub.Attribute!, sub.Method, subscribe, logger);
});
});
//foreach (var sub in subscribedMethods)
//{
// //int partitionCount = sub.Attribute!.TaskCount==-1?3: sub.Attribute!.TaskCount;// kafkaOptionConfig.NumPartitions;
// var adminClientService = provider.GetRequiredService<IAdminClientService>();
// int topicCount = adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic);
// int partitionCount = sub.Attribute!.TaskCount == -1 ? topicCount : sub.Attribute!.TaskCount;// kafkaOptionConfig.NumPartitions;
// partitionCount = partitionCount > topicCount ? topicCount : partitionCount;
// //partitionCount = sub.Attribute!.TaskCount == -1 ? adminClientService.GetTopicPartitionsNum(sub.Attribute!.Topic) : sub.Attribute!.TaskCount;
// if (partitionCount <= 0)
// partitionCount = 1;
// for (int i = 0; i < partitionCount; i++)
// {
// //if (sub.Attribute!.Topic == ProtocolConst.SubscriberLoginReceivedEventName)
// Task.Run(() => StartConsumerAsync(provider, sub.Attribute!, sub.Method, subscribe, logger));
// threadCount++;
// }
//}
return Tuple.Create(threadCount, subscribedMethods.Length);
}
/// <summary>
/// 启动后台消费线程
/// </summary>
private static async Task StartConsumerAsync(IServiceProvider provider, KafkaSubscribeAttribute attr, MethodInfo method, object subscribe, ILogger<CollectBusKafkaModule> logger)
{
var consumerService = provider.GetRequiredService<IConsumerService>();
if (attr.EnableBatch)
{
Interlocked.Increment(ref _threadStartCount);
logger.LogInformation($"kafka开启线程消费:{_threadStartCount}");
await consumerService.SubscribeBatchAsync<dynamic>(attr.Topic, async (message) =>
{
try
{
#if DEBUG
logger.LogInformation($"kafka批量消费消息:{message.Serialize()}");
#endif
// 处理消息
return await ProcessMessageAsync(message.ToList(), method, subscribe);
}
catch (ConsumeException ex)
{
// 处理消费错误
logger.LogError($"kafka批量消费异常:{ex.Message}");
}
catch (Exception ex)
{
// 处理消费错误
logger.LogError($"kafka批量消费异常:{ex.Message}");
}
return await Task.FromResult(false);
}, attr.GroupId, attr.BatchSize, attr.BatchTimeout);
}
else
{
Interlocked.Increment(ref _threadStartCount);
logger.LogInformation($"kafka开启线程消费:{_threadStartCount}");
await consumerService.SubscribeAsync<dynamic>(attr.Topic, async (message) =>
{
try
{
#if DEBUG
logger.LogInformation($"kafka消费消息:{message}");
#endif
// 处理消息
return await ProcessMessageAsync(new List<object>() { message }, method, subscribe);
}
catch (ConsumeException ex)
{
// 处理消费错误
logger.LogError($"kafka消费异常:{ex.Message}");
}
catch (Exception ex)
{
// 处理消费错误
logger.LogError($"kafka批量消费异常:{ex.Message}");
}
return await Task.FromResult(false);
}, attr.GroupId);
}
}
/// <summary>
/// 处理消息
/// </summary>
private static async Task<bool> ProcessMessageAsync(List<dynamic> messages, MethodInfo method, object subscribe)
{
try
{
var parameters = method.GetParameters();
bool isGenericTask = method.ReturnType.IsGenericType
&& method.ReturnType.GetGenericTypeDefinition() == typeof(Task<>);
bool existParameters = parameters.Length > 0;
object[]? executeParameters = null;
if (existParameters)
{
IList? list = null;
Tuple<Type, Type?> tuple = method.GetParameterTypeInfo();
bool isEnumerable = false;
if (tuple.Item2 != null)
{
Type listType = typeof(List<>).MakeGenericType(tuple.Item2);
list = (IList)Activator.CreateInstance(listType)!;
isEnumerable = tuple.Item2.IsConvertType();
}
else
{
isEnumerable = tuple.Item1.IsConvertType();
}
#region
//foreach (var msg in messages)
//{
// if (tuple.Item2 != null)
// {
// if (isEnumerable)
// {
// var parameterType = parameters[0].ParameterType;
// var data=messages?.Serialize().Deserialize(parameterType);
// messageObj = data!=null? new[] { data }:null;
// break;
// }
// else
// {
// // 集合类型
// var data = msg?.Serialize().Deserialize(tuple.Item2) /*isEnumerable ? Convert.ChangeType(msg, tuple.Item2) : msg?.Serialize().Deserialize(tuple.Item2)*/;
// if (data != null)
// list?.Add(data);
// }
// }
// else
// {
// // (dynamic)Convert.ChangeType(msg, tuple.Item1)
// using (var stream = new MemoryStream(msg))
// {
// var data1= System.Text.Json.JsonSerializer.Deserialize(stream, tuple.Item1);
// }
// var data = isEnumerable ? System.Text.Json.JsonSerializer.Deserialize(msg, tuple.Item1): msg?.ToString()?.Deserialize(tuple.Item1);
// if (data != null)
// messageObj = new[] { data };
// }
//}
//if (tuple.Item2 != null && list != null && list.Count > 0)
//{
// messageObj = new[] { list };
//}
#endregion
var parameterDescriptors = method.GetParameters();
executeParameters = new object?[parameterDescriptors.Length];
for (var i = 0; i < parameterDescriptors.Length; i++)
{
foreach (var item in messages)
{
object? tempParameter = null;
var parameterDescriptor = parameterDescriptors[i];
if (KafkaSerialization.IsJsonType(item))
{
tempParameter = KafkaSerialization.Deserialize(item, tuple.Item2 != null ? tuple.Item2 : parameterDescriptor.ParameterType);
}
else
{
var converter = TypeDescriptor.GetConverter(parameterDescriptor.ParameterType);
if (converter.CanConvertFrom(item.GetType()))
{
tempParameter = converter.ConvertFrom(item);
}
else
{
if (parameterDescriptor.ParameterType.IsInstanceOfType(item))
tempParameter = item;
else
tempParameter = Convert.ChangeType(item, parameterDescriptor.ParameterType);
}
}
if (tuple.Item2 == null)
{
executeParameters[i] = tempParameter;
}
else
{
list.Add(tempParameter);
}
}
if (list != null && list.Count > 0)
executeParameters[i] = list;
}
}
var result = method.Invoke(subscribe, executeParameters);
if (result is Task<ISubscribeAck> genericTask)
{
await genericTask.ConfigureAwait(false);
return genericTask.Result.Ack;
}
else if (result is Task nonGenericTask)
{
await nonGenericTask.ConfigureAwait(false);
return true;
}
else if (result is ISubscribeAck ackResult)
{
return ackResult.Ack;
}
return false;
}
catch (Exception ex)
{
throw;
}
}
}
}

View File

@ -1,9 +1,4 @@
using Confluent.Kafka;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.Kafka.Producer
{

View File

@ -1,10 +1,13 @@
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Confluent.Kafka;
using JiShe.CollectBus.Common;
using JiShe.CollectBus.Common.Helpers;
using JiShe.CollectBus.Kafka.Consumer;
using JiShe.CollectBus.Kafka.Internal;
using JiShe.CollectBus.Kafka.Serialization;
@ -23,11 +26,19 @@ namespace JiShe.CollectBus.Kafka.Producer
private readonly ConcurrentDictionary<Type, object> _producerCache = new();
private class KafkaProducer<TKey, TValue> where TKey : notnull where TValue : class { }
private readonly KafkaOptionConfig _kafkaOptionConfig;
public ProducerService(IConfiguration configuration,ILogger<ProducerService> logger, IOptions<KafkaOptionConfig> kafkaOptionConfig)
private readonly ServerApplicationOptions _applicationOptions;
/// <summary>
/// ProducerService
/// </summary>
/// <param name="configuration"></param>
/// <param name="logger"></param>
/// <param name="kafkaOptionConfig"></param>
public ProducerService(IConfiguration configuration,ILogger<ProducerService> logger, IOptions<KafkaOptionConfig> kafkaOptionConfig, IOptions<ServerApplicationOptions> applicationOptions)
{
_configuration = configuration;
_logger = logger;
_kafkaOptionConfig = kafkaOptionConfig.Value;
_applicationOptions = applicationOptions.Value;
}
#region private
@ -59,15 +70,16 @@ namespace JiShe.CollectBus.Kafka.Producer
{
BootstrapServers = _kafkaOptionConfig.BootstrapServers,
//AllowAutoCreateTopics = true,
QueueBufferingMaxKbytes = 2_097_151, // 修改缓冲区最大为2GB默认为1GB
QueueBufferingMaxKbytes = 4_194_304, // 4_194_304 2_097_151 // 修改缓冲区最大为2GB默认为1GB
QueueBufferingMaxMessages = int.MaxValue, // 缓冲区消息条
CompressionType = CompressionType.Lz4, // 配置使用压缩算法LZ4其他gzip/snappy/zstd
BatchSize = 32_768, // 修改批次大小为32K
LingerMs = 20, // 修改等待时间为20ms
LingerMs = 20, // 修改等待时间为20ms默认为5ms
Acks = Acks.All, // 表明只有所有副本Broker都收到消息才算提交成功, 可以 Acks.Leader
MessageSendMaxRetries = 50, // 消息发送失败最大重试50次
MessageTimeoutMs = 120000, // 消息发送超时时间为2分钟,设置值MessageTimeoutMs > LingerMs
};
if (_kafkaOptionConfig.EnableAuthorization)
{
config.SecurityProtocol = _kafkaOptionConfig.SecurityProtocol;
@ -105,17 +117,25 @@ namespace JiShe.CollectBus.Kafka.Producer
/// <returns></returns>
public async Task ProduceAsync<TKey, TValue>(string topic, TKey key, TValue value)where TKey : notnull where TValue : class
{
var typeKey = typeof(KafkaProducer<TKey, TValue>);
var producer = GetProducer<TKey, TValue>(typeKey);
var message = new Message<TKey, TValue>
try
{
Key = key,
Value = value,
Headers = new Headers{
{ "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) }
var typeKey = typeof(KafkaProducer<TKey, TValue>);
var producer = GetProducer<TKey, TValue>(typeKey);
var message = new Message<TKey, TValue>
{
Key = key,
Value = value,
Headers = new Headers{
{ "route-key", Encoding.UTF8.GetBytes(_applicationOptions.ServerTagName) }
}
};
await producer.ProduceAsync(topic, message);
};
await producer.ProduceAsync(topic, message);
}
catch (Exception ex)
{
throw;
}
}
/// <summary>
@ -127,17 +147,24 @@ namespace JiShe.CollectBus.Kafka.Producer
/// <returns></returns>
public async Task ProduceAsync<TValue>(string topic, TValue value) where TValue : class
{
var typeKey = typeof(KafkaProducer<string, TValue>);
var producer = GetProducer<Null, TValue>(typeKey);
var message = new Message<Null, TValue>
try
{
//Key= _kafkaOptionConfig.ServerTagName,
Value = value,
Headers = new Headers{
{ "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) }
var typeKey = typeof(KafkaProducer<string, TValue>);
var producer = GetProducer<Null, TValue>(typeKey);
var message = new Message<Null, TValue>
{
Value = value,
Headers = new Headers{
{ "route-key", Encoding.UTF8.GetBytes(_applicationOptions.ServerTagName) }
}
};
await producer.ProduceAsync(topic, message);
};
await producer.ProduceAsync(topic, message);
}
catch (Exception ex)
{
throw;
}
}
/// <summary>
@ -153,26 +180,34 @@ namespace JiShe.CollectBus.Kafka.Producer
/// <returns></returns>
public async Task ProduceAsync<TKey, TValue>(string topic,TKey key,TValue value,int? partition=null, Action<DeliveryReport<TKey, TValue>>? deliveryHandler = null)where TKey : notnull where TValue : class
{
var message = new Message<TKey, TValue>
try
{
Key = key,
Value = value,
Headers = new Headers{
{ "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) }
var message = new Message<TKey, TValue>
{
Key = key,
Value = value,
Headers = new Headers{
{ "route-key", Encoding.UTF8.GetBytes(_applicationOptions.ServerTagName) }
}
};
var typeKey = typeof(KafkaProducer<TKey, TValue>);
var producer = GetProducer<TKey, TValue>(typeKey);
if (partition.HasValue)
{
var topicPartition = new TopicPartition(topic, partition.Value);
producer.Produce(topicPartition, message, deliveryHandler);
};
var typeKey = typeof(KafkaProducer<TKey, TValue>);
var producer = GetProducer<TKey, TValue>(typeKey);
if (partition.HasValue)
{
var topicPartition = new TopicPartition(topic, new Partition(partition.Value));
producer.Produce(topicPartition, message, deliveryHandler);
}
else
{
producer.Produce(topic, message, deliveryHandler);
}
await Task.CompletedTask;
}
else
catch (Exception ex)
{
producer.Produce(topic, message, deliveryHandler);
throw;
}
await Task.CompletedTask;
}
@ -188,26 +223,34 @@ namespace JiShe.CollectBus.Kafka.Producer
/// <returns></returns>
public async Task ProduceAsync<TValue>(string topic, TValue value, int? partition=null, Action<DeliveryReport<Null, TValue>>? deliveryHandler = null) where TValue : class
{
var message = new Message<Null, TValue>
try
{
//Key = _kafkaOptionConfig.ServerTagName,
Value = value,
Headers = new Headers{
{ "route-key", Encoding.UTF8.GetBytes(_kafkaOptionConfig.ServerTagName) }
var message = new Message<Null, TValue>
{
Value = value,
Headers = new Headers{
{ "route-key", Encoding.UTF8.GetBytes(_applicationOptions.ServerTagName) }
}
};
var typeKey = typeof(KafkaProducer<Null, TValue>);
var producer = GetProducer<Null, TValue>(typeKey);
if (partition.HasValue)
{
var topicPartition = new TopicPartition(topic, partition.Value);
producer.Produce(topicPartition, message, deliveryHandler);
};
var typeKey = typeof(KafkaProducer<Null, TValue>);
var producer = GetProducer<Null, TValue>(typeKey);
if (partition.HasValue)
{
var topicPartition = new TopicPartition(topic, new Partition(partition.Value));
//_logger.LogError($"push消息{topic}-{partition.Value}");
producer.Produce(topicPartition, message, deliveryHandler);
}
else
{
producer.Produce(topic, message, deliveryHandler);
}
await Task.CompletedTask;
}
else
catch (Exception ex)
{
producer.Produce(topic, message, deliveryHandler);
throw;
}
await Task.CompletedTask;
}
public void Dispose()

View File

@ -19,6 +19,7 @@ namespace JiShe.CollectBus.Kafka.Serialization
{
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
WriteIndented = false,// 设置格式化输出
IncludeFields = true,// 允许反序列化到非公共 setter 和字段
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,// 允许特殊字符
IgnoreReadOnlyFields = true,
IgnoreReadOnlyProperties = true,
@ -53,7 +54,7 @@ namespace JiShe.CollectBus.Kafka.Serialization
{
if (data.IsEmpty)
return default;
return JsonSerializer.Deserialize<T>(data, _options)!;
return JsonSerializer.Deserialize<T>(data, _options)!;
}
catch (Exception ex)
{
@ -102,24 +103,37 @@ namespace JiShe.CollectBus.Kafka.Serialization
}
public static object? Deserialize(object value, Type valueType)
{
var _jsonSerializerOptions = new JsonSerializerOptions
try
{
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
WriteIndented = false,// 设置格式化输出
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,// 允许特殊字符
IgnoreReadOnlyFields = true,
IgnoreReadOnlyProperties = true,
NumberHandling = JsonNumberHandling.AllowReadingFromString, // 允许数字字符串
AllowTrailingCommas = true, // 忽略尾随逗号
ReadCommentHandling = JsonCommentHandling.Skip, // 忽略注释
PropertyNameCaseInsensitive = true, // 属性名称大小写不敏感
PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // 属性名称使用驼峰命名规则
Converters = { new DateTimeJsonConverter() } // 注册你的自定义转换器,
};
var _jsonSerializerOptions = new JsonSerializerOptions
{
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
WriteIndented = false,// 设置格式化输出
IncludeFields = true,// 允许反序列化到非公共 setter 和字段
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,// 允许特殊字符
IgnoreReadOnlyFields = true,
IgnoreReadOnlyProperties = true,
NumberHandling = JsonNumberHandling.AllowReadingFromString, // 允许数字字符串
AllowTrailingCommas = true, // 忽略尾随逗号
ReadCommentHandling = JsonCommentHandling.Skip, // 忽略注释
PropertyNameCaseInsensitive = true, // 属性名称大小写不敏感
PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // 属性名称使用驼峰命名规则
Converters = { new DateTimeJsonConverter() } // 注册你的自定义转换器,
};
if (value is JsonElement jsonElement) return jsonElement.Deserialize(valueType, _jsonSerializerOptions);
if (value is JsonElement jsonElement)
{
//return jsonElement.Deserialize(valueType, _jsonSerializerOptions);
return JsonSerializer.Deserialize(jsonElement, valueType, _jsonSerializerOptions);
}
throw new NotSupportedException("Type is not of type JsonElement");
return null;
}
catch (Exception ex)
{
throw;
}
}
}
}

View File

@ -13,6 +13,7 @@ using JiShe.CollectBus.IotSystems.MessageIssueds;
using Volo.Abp.Data;
using Volo.Abp.MongoDB;
using Volo.Abp.MultiTenancy;
using JiShe.CollectBus.IotSystems.LogRecord;
namespace JiShe.CollectBus.MongoDB;
@ -32,7 +33,6 @@ public class CollectBusMongoDbContext : AbpMongoDbContext, ICollectBusMongoDbCon
public IMongoCollection<MessageIssued> MessageIssueds => Collection<MessageIssued>();
protected override void CreateModel(IMongoModelBuilder modelBuilder)
{

View File

@ -1,5 +1,7 @@
using JiShe.CollectBus.IotSystems.MeterReadingRecords;
using JiShe.CollectBus.IotSystems.LogRecord;
using JiShe.CollectBus.IotSystems.MeterReadingRecords;
using JiShe.CollectBus.Repository;
using JiShe.CollectBus.Repository.LogRecord;
using JiShe.CollectBus.Repository.MeterReadingRecord;
using JiShe.CollectBus.ShardingStrategy;
using Microsoft.Extensions.DependencyInjection;
@ -35,10 +37,14 @@ public class CollectBusMongoDbModule : AbpModule
typeof(IShardingStrategy<>),
typeof(DayShardingStrategy<>));
context.Services.AddTransient(typeof(HourShardingStrategy<>));
//// 分表策略仓储 替换默认仓储
//options.AddRepository<MeterReadingRecords, MeterReadingRecordRepository>();
});
options.AddRepository<LogRecords, LogRecordRepository>();
});
context.Services.AddAlwaysDisableUnitOfWorkTransaction();
Configure<AbpUnitOfWorkDefaultOptions>(options =>
{

View File

@ -0,0 +1,57 @@
using JiShe.CollectBus.IotSystems.LogRecord;
using JiShe.CollectBus.IotSystems.MeterReadingRecords;
using MongoDB.Driver;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Volo.Abp.Domain.Repositories;
namespace JiShe.CollectBus.Repository.LogRecord
{
public interface ILogRecordRepository : IRepository<LogRecords, Guid>
{
/// <summary>
/// 批量插入
/// </summary>
/// <param name="entities"></param>
/// <param name="dateTime"></param>
/// <returns></returns>
Task InsertManyAsync(List<LogRecords> entities,
DateTime? dateTime);
/// <summary>
/// 单个插入
/// </summary>
/// <param name="entity"></param>
/// <param name="dateTime"></param>
/// <returns></returns>
Task<LogRecords> InsertAsync(LogRecords entity, DateTime? dateTime);
/// <summary>
/// 单条更新
/// </summary>
/// <param name="filter">过滤条件示例Builders<LogRecords>.Filter.Eq(x => x.Id, filter.Id)</param>
/// <param name="update">包含待更新的内容示例Builders<LogRecords>.Update.Set(x => x.SendHexMessage, SendHexMessage).Set(x => x.MessageId, MessageId)</param>
/// <param name="entity">数据实体,用于获取对应的分片库</param>
/// <returns></returns>
Task<LogRecords> UpdateOneAsync(FilterDefinition<LogRecords> filter, UpdateDefinition<LogRecords> update, LogRecords entity);
/// <summary>
/// 单个获取
/// </summary>
/// <param name="entity"></param>
/// <param name="dateTime"></param>
/// <returns></returns>
Task<LogRecords> FirOrDefaultAsync(LogRecords entity, DateTime dateTime);
/// <summary>
/// 多集合数据查询
/// </summary>
/// <param name="startTime"></param>
/// <param name="endTime"></param>
/// <returns></returns>
Task<List<LogRecords>> ParallelQueryAsync(DateTime startTime, DateTime endTime);
}
}

View File

@ -0,0 +1,166 @@
using JiShe.CollectBus.IotSystems.LogRecord;
using JiShe.CollectBus.IotSystems.MeterReadingRecords;
using JiShe.CollectBus.MongoDB;
using JiShe.CollectBus.Repository.MeterReadingRecord;
using JiShe.CollectBus.ShardingStrategy;
using MongoDB.Driver;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Volo.Abp.DependencyInjection;
using Volo.Abp.Domain.Repositories.MongoDB;
using Volo.Abp.MongoDB;
namespace JiShe.CollectBus.Repository.LogRecord
{
public class LogRecordRepository : MongoDbRepository<CollectBusMongoDbContext, LogRecords, Guid>, ILogRecordRepository
{
private readonly HourShardingStrategy<LogRecords> _hourShardingStrategy;
private readonly IMongoDbContextProvider<CollectBusMongoDbContext> _dbContextProvider;
public LogRecordRepository(
IMongoDbContextProvider<CollectBusMongoDbContext> dbContextProvider,
HourShardingStrategy<LogRecords> hourShardingStrategy
)
: base(dbContextProvider)
{
_dbContextProvider = dbContextProvider;
_hourShardingStrategy = hourShardingStrategy;
}
/// <summary>
/// 批量插入
/// </summary>
/// <param name="entities"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public override async Task<IEnumerable<LogRecords>> InsertManyAsync(IEnumerable<LogRecords> entities, bool autoSave = false, CancellationToken cancellationToken = default(CancellationToken))
{
var collection = await GetShardedCollection(DateTime.Now);
await collection.InsertManyAsync(entities);
return entities;
}
/// <summary>
/// 批量插入
/// </summary>
/// <param name="entities"></param>
/// <param name="dateTime"></param>
/// <returns></returns>
public async Task InsertManyAsync(List<LogRecords> entities, DateTime? dateTime)
{
var collection = await GetShardedCollection(dateTime);
await collection.InsertManyAsync(entities);
}
/// <summary>
/// 单条插入
/// </summary>
/// <param name="entity"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public override async Task<LogRecords> InsertAsync(LogRecords entity, bool autoSave = false, CancellationToken cancellationToken = default(CancellationToken))
{
var collection = await GetShardedCollection(DateTime.Now);
await collection.InsertOneAsync(entity);
return entity;
}
/// <summary>
/// 单条插入
/// </summary>
/// <param name="entity"></param>
/// <param name="dateTime"></param>
/// <returns></returns>
public async Task<LogRecords> InsertAsync(LogRecords entity, DateTime? dateTime)
{
var collection = await GetShardedCollection(dateTime);
await collection.InsertOneAsync(entity);
return entity;
}
/// <summary>
/// 单条更新
/// </summary>
/// <param name="filter">过滤条件示例Builders<LogRecords>.Filter.Eq(x => x.Id, filter.Id)</param>
/// <param name="update">包含待更新的内容示例Builders<LogRecords>.Update.Set(x => x.SendHexMessage, SendHexMessage).Set(x => x.MessageId, MessageId)</param>
/// <param name="entity">数据实体,用于获取对应的分片库</param>
/// <returns></returns>
public async Task<LogRecords> UpdateOneAsync(FilterDefinition<LogRecords> filter, UpdateDefinition<LogRecords> update, LogRecords entity)
{
var collection = await GetShardedCollection(entity.CreationTime);
await collection.UpdateOneAsync(filter, update);
return entity;
}
/// <summary>
/// 单个获取
/// </summary>
/// <param name="entity"></param>
/// <param name="dateTime"></param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
public async Task<LogRecords> FirOrDefaultAsync(LogRecords entity, DateTime dateTime)
{
var collection = await GetShardedCollection(dateTime);
var query = await collection.FindAsync(d => d.CreationTime == dateTime && d.AFN == entity.AFN && d.Fn == entity.Fn && d.Code == entity.Code);
return await query.FirstOrDefaultAsync();
}
/// <summary>
/// 多集合数据查询
/// </summary>
/// <param name="startTime"></param>
/// <param name="endTime"></param>
/// <returns></returns>
public async Task<List<LogRecords>> ParallelQueryAsync(DateTime startTime, DateTime endTime)
{
var collectionNames = _hourShardingStrategy.GetQueryCollectionNames(startTime, endTime);
var dbContext = await DbContextProvider.GetDbContextAsync();
var tasks = collectionNames.Select(async name =>
{
var collection = dbContext.Database.GetCollection<LogRecords>(name);
var filter = Builders<LogRecords>.Filter.And(
Builders<LogRecords>.Filter.Gte(x => x.CreationTime, startTime),
Builders<LogRecords>.Filter.Lte(x => x.CreationTime, endTime)
);
return await collection.Find(filter).ToListAsync();
});
var results = await Task.WhenAll(tasks);
return results.SelectMany(r => r).ToList();
}
/// <summary>
/// 获得分片集合
/// </summary>
/// <returns></returns>
private async Task<IMongoCollection<LogRecords>> GetShardedCollection(DateTime? dateTime)
{
var dbContext = await DbContextProvider.GetDbContextAsync();
string collectionName = string.Empty;
if (dateTime != null)
{
collectionName = _hourShardingStrategy.GetCollectionName(dateTime.Value);
}
else
{
collectionName = _hourShardingStrategy.GetCurrentCollectionName();
}
return dbContext.Database.GetCollection<LogRecords>(collectionName);
}
}
}

View File

@ -1,4 +1,5 @@
using JiShe.CollectBus.Common.Extensions;
using JiShe.CollectBus.Common.Enums;
using JiShe.CollectBus.Common.Extensions;
using System;
using System.Collections.Generic;
using System.Linq;
@ -22,7 +23,7 @@ namespace JiShe.CollectBus.ShardingStrategy
public string GetCollectionName(DateTime dateTime)
{
var baseName = typeof(TEntity).Name;
return $"{baseName}_{dateTime.GetDataTableShardingStrategy()}";
return $"{baseName}_{dateTime.GetDataTableShardingStrategy(TableTimeStrategyEnum.DayShardingStrategy)}";
}
/// <summary>
@ -32,7 +33,7 @@ namespace JiShe.CollectBus.ShardingStrategy
public string GetCurrentCollectionName()
{
var baseName = typeof(TEntity).Name;
return $"{baseName}_{DateTime.Now.GetDataTableShardingStrategy()}";
return $"{baseName}_{DateTime.Now.GetDataTableShardingStrategy(TableTimeStrategyEnum.DayShardingStrategy)}";
}
/// <summary>
@ -50,7 +51,7 @@ namespace JiShe.CollectBus.ShardingStrategy
while (current <= end)
{
months.Add($"{baseName}_{current.GetDataTableShardingStrategy()}");
months.Add($"{baseName}_{current.GetDataTableShardingStrategy(TableTimeStrategyEnum.DayShardingStrategy)}");
current = current.AddMonths(1);
}

View File

@ -0,0 +1,58 @@
using JiShe.CollectBus.Common.Enums;
using JiShe.CollectBus.Common.Extensions;
using System;
using System.Collections.Generic;
using System.Linq;
namespace JiShe.CollectBus.ShardingStrategy
{
/// <summary>
/// 按小时分表
/// </summary>
/// <typeparam name="TEntity"></typeparam>
public class HourShardingStrategy<TEntity>
{
/// <summary>
/// 获取指定时间对应的集合名
/// </summary>
/// <param name="dateTime"></param>
/// <returns></returns>
public string GetCollectionName(DateTime dateTime)
{
var baseName = typeof(TEntity).Name;
return $"{baseName}_{dateTime.GetDataTableShardingStrategy(TableTimeStrategyEnum.HourShardingStrategy)}";
}
/// <summary>
/// 获取当前时间对应的集合名
/// </summary>
/// <returns></returns>
public string GetCurrentCollectionName()
{
var baseName = typeof(TEntity).Name;
return $"{baseName}_{DateTime.Now.GetDataTableShardingStrategy(TableTimeStrategyEnum.HourShardingStrategy)}";
}
/// <summary>
/// 用于查询时确定目标集合
/// </summary>
/// <param name="startTime"></param>
/// <param name="endTime"></param>
/// <returns></returns>
public IEnumerable<string> GetQueryCollectionNames(DateTime? startTime, DateTime? endTime)
{
var list = new List<string>();
var current = startTime ?? DateTime.MinValue;
var end = endTime ?? DateTime.MaxValue;
var baseName = typeof(TEntity).Name;
while (current <= end)
{
list.Add($"{baseName}_{current.GetDataTableShardingStrategy(TableTimeStrategyEnum.HourShardingStrategy)}");
current = current.AddHours(1);
}
return list.Distinct();
}
}
}

View File

@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiShe.CollectBus.ShardingStrategy
{
public interface IHourShardingStrategy<TEntity> : IShardingStrategy<TEntity>
{
}
}

View File

@ -1,368 +0,0 @@
using JiShe.CollectBus.Common.Extensions;
using JiShe.CollectBus.IotSystems.Protocols;
using JiShe.CollectBus.Protocol.Contracts.Interfaces;
using JiShe.CollectBus.Protocol.Contracts.Models;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using TouchSocket.Sockets;
using Volo.Abp.Domain.Repositories;
namespace JiShe.CollectBus.Protocol.Contracts.Abstracts
{
public abstract class ProtocolPlugin:IProtocolPlugin
{
//头部字节长度
public const int hearderLen = 6;
public const int tPLen = 6;
public const string errorData = "EE";
private readonly ILogger _logger;
private readonly IRepository<ProtocolInfo, Guid> _protocolInfoRepository;
public ProtocolPlugin(IServiceProvider serviceProvider, ILogger logger)
{
_logger = logger;
_protocolInfoRepository = serviceProvider.GetRequiredService<IRepository<ProtocolInfo, Guid>>();
}
public abstract ProtocolInfo Info { get; }
public virtual async Task<ProtocolInfo> GetAsync() => await Task.FromResult(Info);
public virtual async Task AddAsync()
{
if (Info == null)
{
throw new ArgumentNullException(nameof(Info));
}
await _protocolInfoRepository.DeleteDirectAsync(a => a.Name == Info.Name);
await _protocolInfoRepository.InsertAsync(Info);
//await _protocolInfoCache.Get()
}
public abstract Task<T> AnalyzeAsync<T>(ITcpSessionClient client, string messageReceived, Action<T>? receivedAction = null) where T :class;
/// <summary>
/// 解析376.1帧
/// </summary>
/// <param name="messageReceived"></param>
/// <returns></returns>
public virtual TB3761? Analysis3761(string messageReceived)
{
try
{
var hexStringList = messageReceived.StringToPairs();
// 初步校验
if (hexStringList.Count < 6 || hexStringList.FirstOrDefault() != "68" || hexStringList.Skip(5).Take(1).FirstOrDefault() != "68" || hexStringList.Count < 18 || hexStringList.LastOrDefault() != "16")
{
_logger.LogError($"解析Analysis3761校验不通过,报文:{messageReceived}");
}
else
{
TB3761 tB3761 = new TB3761
{
C = Analysis_C(hexStringList),
A = Analysis_A(hexStringList),
AFN_FC = Analysis_AFN_FC(hexStringList),
SEQ = Analysis_SEQ(hexStringList),
UnitData = Analysis_UnitData(hexStringList),
DA = Analysis_DA(hexStringList),
DT = Analysis_DT(hexStringList)
};
return tB3761;
}
}
catch (Exception ex)
{
_logger.LogError($"解析Analysis3761错误,报文:{messageReceived},异常:{ex.Message}");
}
return null;
}
/// <summary>
/// 控制域C解析
/// </summary>
/// <returns></returns>
public virtual C? Analysis_C(List<string> hexStringList)
{
try
{
if (hexStringList.Count > 6)
{
BaseHexMessage baseHexMessage = new BaseHexMessage
{
HexMessageList = hexStringList.GetRange(6, 1) // 控制域 1字节
};
baseHexMessage.HexMessageString = string.Join("", baseHexMessage.HexMessageList);
if (baseHexMessage.HexMessageList.Count == 0)
return null;
string binStr = baseHexMessage.HexMessageString.HexTo4BinZero();
C c = new C
{
BaseHexMessage = baseHexMessage,
FC = binStr.Substring(binStr.Length - 4, 4).BinToDec(),
FCV = binStr.Substring(3, 1).BinToDec(),
FCB = binStr.Substring(2, 1).BinToDec(),
PRM = binStr.Substring(1, 1).BinToDec(),
DIR = binStr.Substring(0, 1).BinToDec()
};
return c;
}
}
catch (Exception ex)
{
_logger.LogError($"解析Analysis_C错误,报文:{string.Join("", hexStringList)},异常:{ex.Message}");
}
return null;
}
/// <summary>
/// 地址域A解析
/// </summary>
/// <param name="hexStringList"></param>
/// <returns></returns>
public virtual A? Analysis_A(List<string> hexStringList)
{
try
{
if (hexStringList.Count > 7)
{
BaseHexMessage baseHexMessage = new BaseHexMessage
{
HexMessageList = hexStringList.GetRange(7, 5) // 地址域 5个字节
};
baseHexMessage.HexMessageString = string.Join("", baseHexMessage.HexMessageList);
if (baseHexMessage.HexMessageList.Count == 0)
return null;
A a = new A
{
BaseHexMessage = baseHexMessage,
A1 = baseHexMessage.HexMessageList.ListReverseToStr(0, 2),//.DataConvert(10);//行政区划码A1
A2 = baseHexMessage.HexMessageList.ListReverseToStr(2, 2).PadLeft(5, '0').HexToDec(),//终端地址A2
A3 = Analysis_A3(baseHexMessage.HexMessageList) //主站地址和组地址标志A3
};
a.Code = $"{a.A1.PadLeft(4, '0')}{a.A2.ToString().PadLeft(5, '0')}";
return a;
}
}
catch (Exception ex)
{
_logger.LogError($"解析Analysis_A错误,报文:{string.Join("", hexStringList)},异常:{ex.Message}");
}
return null;
}
/// <summary>
/// 站地址和组地址标志A3
/// </summary>
/// <param name="hexAList">地址域A集合</param>
/// <returns></returns>
public virtual A3? Analysis_A3(List<string> hexAList)
{
try
{
BaseHexMessage baseHexMessage = new BaseHexMessage
{
HexMessageList = hexAList.GetRange(4, 1) // 站地址和组地址标志A3 1个字节
};
baseHexMessage.HexMessageString = string.Join("", baseHexMessage.HexMessageList);
if (baseHexMessage.HexMessageList.Count == 0)
return null;
var binStr = baseHexMessage.HexMessageString.HexTo4BinZero();
A3 a3 = new A3
{
BaseHexMessage = baseHexMessage,
D0 = binStr.Substring(binStr.Length - 1, 1).BinToDec(),
D1_D7 = binStr.Substring(0, binStr.Length - 1).BinToDec()
};
return a3;
}
catch (Exception ex)
{
_logger.LogError($"解析Analysis_A3错误,报文:{string.Join("", hexAList)},异常:{ex.Message}");
}
return null;
}
/// <summary>
/// AFN_FC功能码
/// </summary>
/// <returns></returns>
public virtual AFN_FC? Analysis_AFN_FC(List<string> hexStringList)
{
try
{
BaseHexMessage baseHexMessage = new BaseHexMessage
{
HexMessageList = hexStringList.GetRange(12, 1) //AFN功能码 1个字节
};
baseHexMessage.HexMessageString = string.Join("", baseHexMessage.HexMessageList);
if (baseHexMessage.HexMessageList.Count == 0)
return null;
AFN_FC aFN_FC = new AFN_FC
{
BaseHexMessage = baseHexMessage,
AFN = baseHexMessage.HexMessageString.HexToDec(),
};
return aFN_FC;
}
catch (Exception ex)
{
_logger.LogError($"解析Analysis_AFN_FC错误,报文:{string.Join("", hexStringList)},异常:{ex.Message}");
}
return null;
}
/// <summary>
/// 解析帧序列域SEQ
/// </summary>
/// <returns></returns>
public virtual SEQ? Analysis_SEQ(List<string> hexStringList)
{
try
{
BaseHexMessage baseHexMessage = new BaseHexMessage
{
HexMessageList = hexStringList.GetRange(13, 1) //帧序列域 SEQ 1个字节
};
baseHexMessage.HexMessageString = string.Join("", baseHexMessage.HexMessageList);
if (baseHexMessage.HexMessageList.Count == 0)
return null;
var binStr = baseHexMessage.HexMessageString.HexTo4BinZero();
SEQ seq = new SEQ
{
PSEQ = binStr.Substring(binStr.Length - 4, 4).BinToDec(),
CON = binStr.Substring(3, 1).BinToDec(),
FIN = binStr.Substring(2, 1).BinToDec(),
FIR = binStr.Substring(1, 1).BinToDec(),
TpV = binStr.Substring(0, 1).BinToDec()
};
return seq;
}
catch (Exception ex)
{
_logger.LogError($"解析Analysis_SEQ错误,报文:{string.Join("", hexStringList)},异常:{ex.Message}");
}
return null;
}
/// <summary>
/// 数据单元标识及数据单元数据
/// </summary>
public virtual UnitData? Analysis_UnitData(List<string> hexStringList)
{
try
{
UnitData unitData = new UnitData
{
HexMessageList = hexStringList.GetRange(14, hexStringList.Count - 14 - 2) //总数字节数-固定长度报文头-控制域C-地址域A-校验和CS-结束字符16H
};
unitData.HexMessageString = string.Join("", unitData.HexMessageList);
if (unitData.HexMessageList.Count == 0)
return null;
return unitData;
}
catch (Exception ex)
{
_logger.LogError($"解析Analysis_UnitData错误,报文:{string.Join("", hexStringList)},异常:{ex.Message}");
}
return null;
}
/// <summary>
/// 信息点DA Pn
/// </summary>
/// <returns></returns>
public virtual DA? Analysis_DA(List<string> hexStringList)
{
try
{
BaseHexMessage baseHexMessage = new BaseHexMessage
{
HexMessageList = hexStringList.GetRange(14, 2) //信息点DA Pn 2个字节
};
baseHexMessage.HexMessageString = string.Join("", baseHexMessage.HexMessageList);
if (baseHexMessage.HexMessageList.Count == 0)
return null;
var da1 = baseHexMessage.HexMessageList[0];
var da2 = baseHexMessage.HexMessageList[1];
DA da = new DA()
{
BaseHexMessage = baseHexMessage,
Pn = CalculatePn(da1, da2)
};
return da;
}
catch (Exception ex)
{
_logger.LogError($"解析Analysis_DA错误,报文:{string.Join("", hexStringList)},异常:{ex.Message}");
}
return null;
}
/// <summary>
/// 信息类DT Fn
/// </summary>
/// <returns></returns>
public virtual DT? Analysis_DT(List<string> hexStringList)
{
try
{
BaseHexMessage baseHexMessage = new BaseHexMessage
{
HexMessageList = hexStringList.GetRange(16, 2) //信息类DT Fn 2个字节
};
baseHexMessage.HexMessageString = string.Join("", baseHexMessage.HexMessageList);
if (baseHexMessage.HexMessageList.Count == 0)
return null;
var dt1 = baseHexMessage.HexMessageList[0];
var dt2 = baseHexMessage.HexMessageList[1];
DT dt = new DT()
{
BaseHexMessage = baseHexMessage,
Fn = CalculateFn(dt1, dt2)
};
return dt;
}
catch (Exception ex)
{
_logger.LogError($"解析Analysis_DT错误,报文:{string.Join("", hexStringList)},异常:{ex.Message}");
}
return null;
}
/// <summary>
/// 计算Pn
/// </summary>
/// <param name="da1"></param>
/// <param name="da2"></param>
/// <returns></returns>
public int CalculatePn(string da1, string da2) => (da2.HexToDec() - 1) * 8 + (8 - da1.HexTo4BinZero().IndexOf(da1.Equals("00") ? "0" : "1"));
/// <summary>
/// 计算Fn
/// </summary>
/// <param name="dt1"></param>
/// <param name="dt2"></param>
/// <returns></returns>
public int CalculateFn(string dt1, string dt2) => dt2.HexToDec() * 8 + (8 - dt1.HexTo4BinZero().IndexOf("1"));
}
}

View File

@ -1,25 +0,0 @@
using System;
using System.Threading.Tasks;
using JiShe.CollectBus.Common.Models;
using JiShe.CollectBus.IotSystems.MessageReceiveds;
using JiShe.CollectBus.IotSystems.Protocols;
using JiShe.CollectBus.Protocol.Contracts.Models;
using TouchSocket.Sockets;
namespace JiShe.CollectBus.Protocol.Contracts.Interfaces
{
public interface IProtocolPlugin
{
Task<ProtocolInfo> GetAsync();
Task AddAsync();
Task<T> AnalyzeAsync<T>(ITcpSessionClient client, string messageReceived, Action<T>? sendAction = null) where T : class;
TB3761? Analysis3761(string messageReceived);
//Task LoginAsync(MessageReceivedLogin messageReceived);
//Task HeartbeatAsync(MessageReceivedHeartbeat messageReceived);
}
}

View File

@ -1,27 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Extensions\**" />
<EmbeddedResource Remove="Extensions\**" />
<None Remove="Extensions\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
<PackageReference Include="TouchSocket" Version="2.1.9" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\modules\JiShe.CollectBus.Kafka\JiShe.CollectBus.Kafka.csproj" />
<ProjectReference Include="..\..\services\JiShe.CollectBus.Domain\JiShe.CollectBus.Domain.csproj" />
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,6 @@
namespace JiShe.CollectBus.Protocol.Contracts.ProtocolPools
{
public interface IPluginContainer
{
}
}

View File

@ -0,0 +1,26 @@
using JiShe.CollectBus.Interfaces;
namespace JiShe.CollectBus.Protocol.Contracts.ProtocolPools
{
public class PluginContainer: IPluginContainer
{
public Dictionary<string, object> ProtocolPools;
public PluginContainer()
{
}
}
public static class ServiceProviderKeyedServiceExtensions
{
//public static Task AddKeyedSingleton<TImp>(this IServiceProvider provider, string key)
//{
// //var aa = Activator.CreateInstance<TImp>();
//}
}
}

View File

@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
<PackageReference Include="Volo.Abp.Core" Version="8.3.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\services\JiShe.CollectBus.Domain\JiShe.CollectBus.Domain.csproj" />
<ProjectReference Include="..\..\shared\JiShe.CollectBus.Common\JiShe.CollectBus.Common.csproj" />
<ProjectReference Include="..\JiShe.CollectBus.Protocol.T37612012\JiShe.CollectBus.Protocol.T37612012.csproj" />
</ItemGroup>
<Target Name="PostBuild" AfterTargets="PostBuildEvent">
<Exec Command="copy $(TargetDir)JiShe.CollectBus.Protocol.T1882018.dll $(ProjectDir)..\..\web\JiShe.CollectBus.Host\Plugins\" />
</Target>
</Project>

View File

@ -0,0 +1,28 @@
using JiShe.CollectBus.Protocol.Interfaces;
using Microsoft.Extensions.DependencyInjection;
using Volo.Abp;
using Volo.Abp.Modularity;
namespace JiShe.CollectBus.Protocol.T1882018
{
public class JiSheCollectBusProtocolT1882018Module : AbpModule
{
public override void ConfigureServices(ServiceConfigurationContext context)
{
context.Services.AddKeyedSingleton<IProtocolPlugin, T1882018ProtocolPlugin>(nameof(T1882018ProtocolPlugin));
}
public override async Task OnApplicationInitializationAsync(ApplicationInitializationContext context)
{
Console.WriteLine($"{nameof(T1882018ProtocolPlugin)} OnApplicationInitializationAsync");
var standardProtocol = context.ServiceProvider.GetRequiredKeyedService<IProtocolPlugin>(nameof(T1882018ProtocolPlugin));
await standardProtocol.LoadAsync();
}
public override void OnApplicationShutdown(ApplicationShutdownContext context)
{
Console.WriteLine($"{nameof(T1882018ProtocolPlugin)} OnApplicationShutdown");
base.OnApplicationShutdown(context);
}
}
}

View File

@ -0,0 +1,75 @@
using System.Reflection;
using JiShe.CollectBus.Common.BuildSendDatas;
namespace JiShe.CollectBus.Protocol.T1882018.SendData
{
/// <summary>
/// 构建188-2018下发报文
/// </summary>
public static class Telemetry1882018PacketBuilder
{
/// <summary>
/// 构建报文的委托
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public delegate Telemetry1882018PacketResponse T1882018Delegate(Telemetry1882018PacketRequest request);
/// <summary>
/// 编码与方法的映射表
/// </summary>
public static readonly Dictionary<string, T1882018Delegate> T1882018ControlHandlers = new();
static Telemetry1882018PacketBuilder()
{
// 初始化时自动注册所有符合命名规则的方法
var methods = typeof(Telemetry1882018PacketBuilder).GetMethods(BindingFlags.Static | BindingFlags.Public);
foreach (var method in methods)
{
if (method.Name.StartsWith("CTR") && method.Name.EndsWith("_Send"))
{
string code = method.Name;
var delegateInstance = (T1882018Delegate)Delegate.CreateDelegate(typeof(T1882018Delegate), method);
T1882018ControlHandlers[code] = delegateInstance;
}
}
}
#region
/// <summary>
/// 读取计量数据
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public static Telemetry1882018PacketResponse CTR_01_Send(Telemetry1882018PacketRequest request)
{
var itemCodeArr = request.ItemCode.Split('_');
var c_data = itemCodeArr[0];//01
var d_data = itemCodeArr[2];//91 或者 90
var dataUnit = new List<string>() { "1F", d_data, "00" };
var dataList = Build188SendData.Build188SendCommand(request.MeterAddress, c_data, dataUnit);
return new Telemetry1882018PacketResponse() { Data = dataList };
}
#endregion
#region
/// <summary>
/// 阀控
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public static Telemetry1882018PacketResponse CTR_04_Send(Telemetry1882018PacketRequest request)
{
var itemCodeArr = request.ItemCode.Split('_');
var c_data = itemCodeArr[0];//01
var d_data = itemCodeArr[2];//55 或者 99
var dataUnit = new List<string>() { "A0", "17", "00", d_data };
var dataList = Build188SendData.Build188SendCommand(request.MeterAddress, c_data, dataUnit);
return new Telemetry1882018PacketResponse() { Data = dataList };
}
#endregion
}
}

View File

@ -0,0 +1,23 @@
namespace JiShe.CollectBus.Protocol.T1882018.SendData
{
/// <summary>
/// 构建645报文参数
/// </summary>
public class Telemetry1882018PacketRequest
{
/// <summary>
/// 表地址
/// </summary>
public required string MeterAddress { get; set; }
/// <summary>
/// 密码
/// </summary>
public required string Password { get; set; }
/// <summary>
/// 操作码
/// </summary>
public required string ItemCode { get; set; }
}
}

View File

@ -0,0 +1,13 @@
namespace JiShe.CollectBus.Protocol.T1882018.SendData
{
/// <summary>
/// 返回645报文结果
/// </summary>
public class Telemetry1882018PacketResponse
{
/// <summary>
/// 报文体
/// </summary>
public List<string> Data { get; set; }
}
}

View File

@ -0,0 +1,142 @@
using JiShe.CollectBus.Common.Consts;
using JiShe.CollectBus.Common.Enums;
using JiShe.CollectBus.Common.Extensions;
using JiShe.CollectBus.IotSystems.Protocols;
using JiShe.CollectBus.Protocol.Models;
using JiShe.CollectBus.Protocol.T1882018.SendData;
using JiShe.CollectBus.Protocol.T37612012;
using JiShe.CollectBus.Protocol.T37612012.SendData;
using Mapster;
using Microsoft.Extensions.Logging;
using TouchSocket.Sockets;
namespace JiShe.CollectBus.Protocol.T1882018
{
/// <summary>
/// T1882018协议插件
/// </summary>
public class T1882018ProtocolPlugin : T37612012ProtocolPlugin
{
private readonly ILogger<T1882018ProtocolPlugin> _logger;
public readonly Dictionary<string, Telemetry1882018PacketBuilder.T1882018Delegate> T188ControlHandlers;
/// <summary>
/// Initializes a new instance of the <see cref="T1882018ProtocolPlugin"/> class.
/// </summary>
/// <param name="serviceProvider">The service provider.</param>
public T1882018ProtocolPlugin(IServiceProvider serviceProvider, ILogger<T1882018ProtocolPlugin> logger, ITcpService tcpService) : base(serviceProvider, logger, tcpService)
{
_logger = logger;
T188ControlHandlers = Telemetry1882018PacketBuilder.T1882018ControlHandlers;
}
public sealed override ProtocolInfo Info => new(nameof(T1882018ProtocolPlugin), "376.1/188-2018", "TCP", "376.1/188-2018协议", "HJ-LXS-15 DN15");
public override async Task<T> AnalyzeAsync<T>(ITcpSessionClient client, string messageReceived, Action<T>? sendAction = null)
{
//TB3761? tB3761 = Analysis3761(messageReceived);
//if (tB3761 != null)
//{
// if (tB3761.AFN_FC?.AFN == (int)AFN.链路接口检测)
// {
// if (tB3761.A == null || tB3761.A.Code.IsNullOrWhiteSpace() || tB3761.A.A3?.D1_D7 == null || tB3761.SEQ?.PSEQ == null)
// {
// _logger.LogError($"解析AFN.链路接口检测报文失败,报文:{messageReceived},TB3761:{tB3761.Serialize()}");
// }
// else
// {
// if (tB3761.DT?.Fn == (int)FN.登录)
// {
// // 登录回复
// if (tB3761.SEQ.CON == (int)CON.需要对该帧进行确认)
// await LoginAsync(client, messageReceived, tB3761.A.Code, tB3761.A.A3?.D1_D7, tB3761.SEQ?.PSEQ);
// }
// else if (tB3761.DT?.Fn == (int)FN.心跳)
// {
// // 心跳回复
// //心跳帧有两种情况:
// //1. 集中器先有登录帧,再有心跳帧
// //2. 集中器没有登录帧,只有心跳帧
// await HeartbeatAsync(client, messageReceived, tB3761.A.Code, tB3761.A.A3?.D1_D7, tB3761.SEQ?.PSEQ);
// }
// }
// }
// await OnTcpNormalReceived(client, tB3761);
//}
//return (tB3761 as T)!;
return null;
}
/// <summary>
/// 组装报文
/// </summary>
/// <param name="request">报文构建参数</param>
/// <returns></returns>
public override async Task<ProtocolBuildResponse> BuildAsync(ProtocolBuildRequest request)
{
if (request == null)
{
_logger.LogError($"{nameof(T1882018ProtocolPlugin)} 报文构建失败,参数为空");
return new ProtocolBuildResponse();
}
var itemCodeArr = request.ItemCode.Split('_');
var aFNStr = itemCodeArr[0];
var aFN = (AFN)aFNStr.HexToDec();
var fn = int.Parse(itemCodeArr[1]);
Telemetry3761PacketResponse builderResponse = null;
List<string> dataUnit = new List<string>();
//数据转发场景 10H_F1
if (request.ItemCode == T37612012PacketItemCodeConst.AFN10HFN01H && request.SubProtocolRequest != null && string.IsNullOrWhiteSpace(request.SubProtocolRequest.ItemCode) == false)
{
//var subItemCodeArr = request.SubProtocolRequest.ItemCode.Split("_");
var t188PacketHandlerName = $"{T1882018PacketItemCodeConst.BasicT1882018}_{request.SubProtocolRequest.ItemCode}_Send";
Telemetry1882018PacketResponse t645PacketResponse = null;
if (T188ControlHandlers != null && T188ControlHandlers.TryGetValue(t188PacketHandlerName
, out var t645PacketHandler))
{
t645PacketResponse = t645PacketHandler(new Telemetry1882018PacketRequest()
{
MeterAddress = request.SubProtocolRequest.MeterAddress,
Password = request.SubProtocolRequest.Password,
ItemCode = request.SubProtocolRequest.ItemCode,
});
}
if (t645PacketResponse != null)
{
dataUnit = t645PacketResponse.Data;
}
}
string afnMethonCode = $"AFN{aFNStr}_Fn_Send";
if (base.T3761AFNHandlers != null && base.T3761AFNHandlers.TryGetValue(afnMethonCode
, out var handler))
{
builderResponse = handler(new Telemetry3761PacketRequest()
{
FocusAddress = request.FocusAddress,
Fn = fn,
Pn = request.Pn,
DataUnit = dataUnit,
});
}
if (builderResponse == null)
{
return new ProtocolBuildResponse();
}
var result = builderResponse.Adapt<ProtocolBuildResponse>();
result.IsSuccess = true;
return await Task.FromResult(result);
}
}
}

Some files were not shown because too many files have changed in this diff Show More