[{"data":1,"prerenderedAt":326},["ShallowReactive",2],{"site-data":3,"article-119":143,"article-show-hot-119":171,"article-show-new-119":261},{"settings":4,"categorys":35,"tree":97,"models":118,"search_model_select":128,"nav_list":130},{"changefreq":5,"changyan_app_id":6,"changyan_app_key":6,"copy":7,"description":8,"editor":9,"file":6,"guest_feedback":10,"head_html":6,"icp":11,"index_banner":12,"index_banner_bg":13,"is_excel":14,"is_watermark":10,"keywords":15,"links":16,"logo":17,"lzcms_banner":6,"lzcms_banner_link":18,"member_register_enabled":14,"qq_app_id":6,"qq_app_key":6,"qr_code":6,"search_model":19,"site_closedreason":6,"site_idea":6,"site_idea1":20,"site_idea2":21,"site_idea3":22,"site_name":23,"site_statistice":6,"site_status":14,"site_url":18,"sitemap_model":19,"stationmaster_motto":24,"stationmaster_name":25,"stationmaster_occupation":26,"stationmaster_qq":27,"stationmaster_qqnet":28,"stationmaster_qqnet_code":29,"threshold":14,"title_add":30,"watermark":6,"watermark_alpha":31,"watermark_height":32,"watermark_locate":33,"watermark_width":34},"weekly","","版权所有 © \u003Ca class=\"site_url\" href=\"https://zhl123.com\">2026 zhl123.com\u003C/a>","linux、Python、mysql、docker、k8s技术交流","layedit","0","粤ICP备15054664号-1","/uploads/images/20181109/7a86191de8b8bb60e9c6b54d8b27c5cc.jpg","#xe604","1","linux、Python、mysql、docker、k8s","{\"1\":{\"id\":\"1\",\"link_url\":\"https://linux.org\",\"logo\":\"\",\"name\":\"linux\",\"sort\":\"0\",\"status\":\"1\"}}","/uploads/images/20181109/e7305012448aed257176dd591846f50a.png","https://zhl123.com","2","学无止境\n学习，探索，研究，从不了解到了解，从无知到掌握，到灵活运用，在不断的学习中加深认识。由浅入深，由表及里。","业精于勤\n“业精于勤荒于嬉”，精深的业技靠的是勤学、刻苦努力，靠的是争分夺秒的勤学苦练才会有精深的技术。得在认真，失在随便。","工匠精神\n精益求精，注重细节，追求完美和极致，不惜花费时间精力，孜孜不倦，反复改进产品，把99%提高到99.99%。","linux","业精于勤、学无止境、工匠精神","廖地金","高级Linux运维工程师","1256636645","592958303","\u003Ca target=\"_blank\" href=\"//shang.qq.com/wpa/qunwpa?idkey=09be7d1a682073783fd636102e666393169b8a8aac8f3393da1de57bcaa821a0\">\u003Cimg border=\"0\" src=\"//pub.idqqimg.com/wpa/images/group.png\" alt=\"Linux技术\" title=\"Linux技术\">\u003C/a>"," | Python | mysql | docker | k8s 技术交流","100","300","9","500",{"0":36,"17":51,"18":60,"19":63,"20":67,"21":71,"22":76,"25":80,"26":83,"27":89,"29":94},{"children":37},{"0":38,"25":43,"26":48},[39,40,41,42],25,21,22,26,[44,45,46,47],17,18,19,20,[49,50],27,29,{"create_time":6,"description":6,"id":44,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":23,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":53,"subtitle":6,"update_time":6,"url":59},"index",1,"list","article",2,"文章模型","show","/article/lists?category_id=17",{"create_time":6,"description":6,"id":45,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":61,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":56,"subtitle":6,"update_time":6,"url":62},"python","/article/lists?category_id=18",{"create_time":6,"description":6,"id":46,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":64,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":65,"subtitle":6,"update_time":6,"url":66},"容器技术",3,"/article/lists?category_id=19",{"create_time":6,"description":6,"id":47,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":68,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":69,"subtitle":6,"update_time":6,"url":70},"负载均衡",4,"/article/lists?category_id=20",{"create_time":6,"description":6,"id":40,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":72,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":74,"subtitle":6,"update_time":6,"url":75},"分享无价",0,5,"/article/lists?category_id=21",{"create_time":6,"description":6,"id":41,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":77,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":78,"subtitle":6,"update_time":6,"url":79},"随笔",6,"/article/lists?category_id=22",{"create_time":6,"description":6,"id":39,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":81,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":73,"subtitle":6,"update_time":6,"url":82},"学无止境","/article/lists?category_id=25",{"create_time":6,"description":6,"id":42,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":84,"model_id":65,"model_name":85,"name":86,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":87,"subtitle":6,"update_time":6,"url":88},"picture","图集模型","关于",8,"/picture/lists?category_id=26",{"create_time":6,"description":6,"id":49,"image_url":6,"index_template":52,"is_cover":53,"is_menu":73,"keywords":6,"list_template":54,"model_code":90,"model_id":53,"model_name":91,"name":92,"parent_id":42,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":47,"subtitle":6,"update_time":6,"url":93},"page","单页模型","关于博主","/page/27",{"create_time":6,"description":6,"id":50,"image_url":6,"index_template":52,"is_cover":73,"is_menu":73,"keywords":6,"list_template":54,"model_id":73,"name":95,"parent_id":42,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":47,"subtitle":6,"update_time":6,"url":96},"留言","/feedback",[98,108,110,112],{"children":99,"create_time":6,"description":6,"id":39,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":81,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":73,"subtitle":6,"update_time":6,"url":82},[100,102,104,106],{"children":101,"create_time":6,"description":6,"id":44,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":23,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":53,"subtitle":6,"update_time":6,"url":59},[],{"children":103,"create_time":6,"description":6,"id":45,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":61,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":56,"subtitle":6,"update_time":6,"url":62},[],{"children":105,"create_time":6,"description":6,"id":46,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":64,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":65,"subtitle":6,"update_time":6,"url":66},[],{"children":107,"create_time":6,"description":6,"id":47,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":68,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":69,"subtitle":6,"update_time":6,"url":70},[],{"children":109,"create_time":6,"description":6,"id":40,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":72,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":74,"subtitle":6,"update_time":6,"url":75},[],{"children":111,"create_time":6,"description":6,"id":41,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":77,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":78,"subtitle":6,"update_time":6,"url":79},[],{"children":113,"create_time":6,"description":6,"id":42,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":84,"model_id":65,"model_name":85,"name":86,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":87,"subtitle":6,"update_time":6,"url":88},[114,116],{"children":115,"create_time":6,"description":6,"id":49,"image_url":6,"index_template":52,"is_cover":53,"is_menu":73,"keywords":6,"list_template":54,"model_code":90,"model_id":53,"model_name":91,"name":92,"parent_id":42,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":47,"subtitle":6,"update_time":6,"url":93},[],{"children":117,"create_time":6,"description":6,"id":50,"image_url":6,"index_template":52,"is_cover":73,"is_menu":73,"keywords":6,"list_template":54,"model_id":73,"name":95,"parent_id":42,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":47,"subtitle":6,"update_time":6,"url":96},[],{"1":119,"2":120,"3":121,"4":122,"5":125},{"id":53,"index_template":52,"list_template":54,"name":91,"show_template":58,"status":73,"tablename":90},{"id":56,"index_template":52,"list_template":54,"name":57,"show_template":58,"status":73,"tablename":55},{"id":65,"index_template":52,"list_template":54,"name":85,"show_template":58,"status":73,"tablename":84},{"id":69,"index_template":52,"list_template":54,"name":123,"show_template":58,"status":73,"tablename":124},"链接模型","link",{"id":74,"index_template":52,"list_template":54,"name":126,"show_template":58,"status":73,"tablename":127},"下载模型","download",[129],{"id":56,"name":57,"tablename":55},[131,137,139,141],{"children":132,"create_time":6,"description":6,"id":39,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":81,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":73,"subtitle":6,"update_time":6,"url":82},[133,134,135,136],{"create_time":6,"description":6,"id":44,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":23,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":53,"subtitle":6,"update_time":6,"url":59},{"create_time":6,"description":6,"id":45,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":61,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":56,"subtitle":6,"update_time":6,"url":62},{"create_time":6,"description":6,"id":46,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":64,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":65,"subtitle":6,"update_time":6,"url":66},{"create_time":6,"description":6,"id":47,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":68,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":69,"subtitle":6,"update_time":6,"url":70},{"children":138,"create_time":6,"description":6,"id":40,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":72,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":74,"subtitle":6,"update_time":6,"url":75},[],{"children":140,"create_time":6,"description":6,"id":41,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":77,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":78,"subtitle":6,"update_time":6,"url":79},[],{"children":142,"create_time":6,"description":6,"id":42,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":84,"model_id":65,"model_name":85,"name":86,"parent_id":73,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":87,"subtitle":6,"update_time":6,"url":88},[],{"article":144,"breadcrumb":154,"category":158,"next":159,"prev":162,"second_categorys":165,"seo":170},{"id":145,"category_id":44,"title":146,"keywords":6,"description":147,"image_url":6,"content":148,"content_md":149,"url":150,"hits":151,"is_recommend":73,"is_top":73,"create_time":152,"update_time":153},119,"用 Prometheus Recording Rules 把告警噪声砍掉 70%(一)","一、概述1.1 背景介绍在大规模微服务架构下，Prometheus 告警系统往往会陷入一个尴尬的境地：告警太多，运维团队开始选择性忽略；告警太少，真正的故障又可能漏掉。我在某电商平台负责监控体系建设时，团队每天要处理超过 2000 条告警，其中 70% 以上是重复的、关联的或者短暂抖动产生的噪声。Recording Rules 是 Prometheus 提供的预计算机制，可以将复杂的查询表达式预先计算并存储为新的时间序列。通过合理设计 Recording Rules，我们不仅能显著降低 Prom","\u003Ch2 class=\"js_darkmode__bg__2\" style=\"text-align: center;\">\u003Cspan leaf=\"\">一、概述\u003C/span>\u003C/h2>\u003Ch3 class=\"js_darkmode__bg__3\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">1.1 背景介绍\u003C/span>\u003C/h3>\u003Cp class=\"js_darkmode__1\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">在大规模微服务架构下，Prometheus 告警系统往往会陷入一个尴尬的境地：告警太多，运维团队开始选择性忽略；告警太少，真正的故障又可能漏掉。我在某电商平台负责监控体系建设时，团队每天要处理超过 2000 条告警，其中 70% 以上是重复的、关联的或者短暂抖动产生的噪声。\u003C/span>\u003C/p>\u003Cp class=\"js_darkmode__2\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">Recording Rules 是 Prometheus 提供的预计算机制，可以将复杂的查询表达式预先计算并存储为新的时间序列。通过合理设计 Recording Rules，我们不仅能显著降低 PromQL 查询压力，更重要的是可以实现告警聚合、去重、平滑，从根本上减少告警噪声。\u003C/span>\u003C/p>\u003Ch3 class=\"js_darkmode__bg__4\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">1.2 技术特点\u003C/span>\u003C/h3>\u003Cul class=\"list-paddingleft-1 js_darkmode__3\" style=\"text-align: justify;\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>预计算优化\u003C/span>\u003Cspan leaf=\"\">：将复杂的聚合查询预先计算，避免告警规则重复计算相同的表达式，降低 Prometheus 负载\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>时间窗口平滑\u003C/span>\u003Cspan leaf=\"\">：通过 \u003C/span>\u003Ccode>\u003Cspan leaf=\"\">avg_over_time\u003C/span>\u003C/code>\u003Cspan leaf=\"\">、\u003C/span>\u003Ccode>\u003Cspan leaf=\"\">max_over_time\u003C/span>\u003C/code>\u003Cspan leaf=\"\"> 等函数消除瞬时抖动，避免单点毛刺触发告警\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>多维度聚合\u003C/span>\u003Cspan leaf=\"\">：支持按 namespace、service、pod 等维度聚合指标，实现告警收敛\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>层级化告警\u003C/span>\u003Cspan leaf=\"\">：基于 Recording Rules 构建指标层级，实现从基础指标到业务指标的逐层抽象\u003C/span>\u003C/section>\u003C/li>\u003C/ul>\u003Ch3 class=\"js_darkmode__bg__5\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">1.3 适用场景\u003C/span>\u003C/h3>\u003Cul class=\"list-paddingleft-1 js_darkmode__4\" style=\"text-align: justify;\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 场景一：大规模 Kubernetes 集群监控，Pod 数量超过 1000，原生指标产生的告警风暴需要收敛\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 场景二：微服务架构下，服务间依赖复杂，需要将底层告警聚合为服务级别的健康状态\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 场景三：业务高峰期流量抖动频繁，需要平滑处理避免误告警\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 场景四：多租户环境下，需要按租户/业务线聚合告警，避免一个故障触发上百条告警\u003C/span>\u003C/section>\u003C/li>\u003C/ul>\u003Ch3 class=\"js_darkmode__bg__6\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">1.4 环境要求\u003C/span>\u003C/h3>\u003Ctable class=\"js_darkmode__5 layui-table\" style=\"text-align: justify;\">\u003Cthead>\u003Ctr>\u003Cth class=\"js_darkmode__bg__7\" style=\"text-align: left;\">\u003Csection>\u003Cspan leaf=\"\">组件\u003C/span>\u003C/section>\u003C/th>\u003Cth class=\"js_darkmode__bg__8\" style=\"text-align: left;\">\u003Csection>\u003Cspan leaf=\"\">版本要求\u003C/span>\u003C/section>\u003C/th>\u003Cth class=\"js_darkmode__bg__9\" style=\"text-align: left;\">\u003Csection>\u003Cspan leaf=\"\">说明\u003C/span>\u003C/section>\u003C/th>\u003C/tr>\u003C/thead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"js_darkmode__6\">\u003Csection>\u003Cspan leaf=\"\">Prometheus\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__7\">\u003Csection>\u003Cspan leaf=\"\">2.28+\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__8\">\u003Csection>\u003Cspan leaf=\"\">支持 Recording Rules 分组和 limit 特性\u003C/span>\u003C/section>\u003C/td>\u003C/tr>\u003Ctr>\u003Ctd class=\"js_darkmode__9\">\u003Csection>\u003Cspan leaf=\"\">Alertmanager\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__10\">\u003Csection>\u003Cspan leaf=\"\">0.23+\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__11\">\u003Csection>\u003Cspan leaf=\"\">配合使用告警抑制和分组功能\u003C/span>\u003C/section>\u003C/td>\u003C/tr>\u003Ctr>\u003Ctd class=\"js_darkmode__12\">\u003Csection>\u003Cspan leaf=\"\">Kubernetes\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__13\">\u003Csection>\u003Cspan leaf=\"\">1.21+\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__14\">\u003Csection>\u003Cspan leaf=\"\">如果使用 kube-prometheus-stack\u003C/span>\u003C/section>\u003C/td>\u003C/tr>\u003Ctr>\u003Ctd class=\"js_darkmode__15\">\u003Csection>\u003Cspan leaf=\"\">内存配置\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__16\">\u003Csection>\u003Cspan leaf=\"\">建议 8GB+\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__17\">\u003Csection>\u003Cspan leaf=\"\">Recording Rules 会增加时序存储量\u003C/span>\u003C/section>\u003C/td>\u003C/tr>\u003C/tbody>\u003C/table>\u003Chr class=\"js_darkmode__bg__10 js_darkmode__18\" style=\"text-align: justify;\">\u003Ch2 class=\"js_darkmode__bg__11\" style=\"text-align: center;\">\u003Cspan leaf=\"\">二、详细步骤\u003C/span>\u003C/h2>\u003Ch3 class=\"js_darkmode__bg__12\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">2.1 准备工作\u003C/span>\u003C/h3>\u003Ch4 class=\"js_darkmode__bg__13\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 2.1.1 现状分析\u003C/span>\u003C/h4>\u003Cp class=\"js_darkmode__19\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">在动手之前，先要搞清楚当前告警噪声的分布。我习惯用下面这个查询来分析过去一周的告警触发情况：\u003C/span>\u003C/p>\u003Cpre class=\"js_darkmode__bg__14 js_darkmode__20\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 查看 Prometheus 配置\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">kubectl get configmap prometheus-server -n monitoring -o yaml\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 分析告警触发频次（在 Prometheus UI 执行）\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 统计过去7天各告警规则触发次数\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">count_over_time(ALERTS{alertstate=\u003C/span>\u003Cspan leaf=\"\">\"firing\"\u003C/span>\u003Cspan leaf=\"\">}[7d])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 查看当前活跃告警数量\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">count(ALERTS{alertstate=\u003C/span>\u003Cspan leaf=\"\">\"firing\"\u003C/span>\u003Cspan leaf=\"\">})\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch4 class=\"js_darkmode__bg__15\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 2.1.2 识别噪声来源\u003C/span>\u003C/h4>\u003Cp class=\"js_darkmode__21\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">根据我的经验，告警噪声主要来自以下几类：\u003C/span>\u003C/p>\u003Cpre class=\"js_darkmode__bg__16 js_darkmode__22\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 检查高频告警（每分钟触发超过10次的规则）\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 在 Prometheus 执行\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">topk(20, count by (alertname) (count_over_time(ALERTS{alertstate=\u003C/span>\u003Cspan leaf=\"\">\"firing\"\u003C/span>\u003Cspan leaf=\"\">}[1h])))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 检查短暂告警（持续时间小于5分钟的）\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 这类告警往往是抖动产生的噪声\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Cp class=\"js_darkmode__23\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">通常会发现以下几类高频噪声：\u003C/span>\u003C/p>\u003Cul class=\"list-paddingleft-1 js_darkmode__24\" style=\"text-align: justify;\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• CPU/内存使用率瞬时超阈值\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• Pod 重启计数器误报\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 网络延迟抖动\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 磁盘 IO 突增\u003C/span>\u003C/section>\u003C/li>\u003C/ul>\u003Ch3 class=\"js_darkmode__bg__17\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">2.2 核心配置\u003C/span>\u003C/h3>\u003Ch4 class=\"js_darkmode__bg__18\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 2.2.1 Recording Rules 基础结构\u003C/span>\u003C/h4>\u003Cpre class=\"js_darkmode__bg__19 js_darkmode__25\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 文件路径：/etc/prometheus/rules/recording_rules.yml\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">groups:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">node_recording_rules\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">interval:\u003C/span>\u003Cspan leaf=\"\">30s\u003C/span>\u003Cspan leaf=\"\"># 计算间隔，根据需求调整\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 规则定义\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">job:node_cpu_utilization:avg5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          1 - avg by (job, instance) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            rate(node_cpu_seconds_total{mode=\"idle\"}[5m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Cp class=\"js_darkmode__26\" style=\"text-align: justify;\">\u003Cspan>说明\u003C/span>\u003Cspan leaf=\"\">：\u003C/span>\u003Ccode>\u003Cspan leaf=\"\">interval\u003C/span>\u003C/code>\u003Cspan leaf=\"\"> 参数控制 Recording Rules 的计算频率。设置过小会增加计算压力，过大则可能错过关键变化。一般建议设置为告警评估间隔的一半。\u003C/span>\u003C/p>\u003Ch4 class=\"js_darkmode__bg__20\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 2.2.2 CPU 使用率平滑处理\u003C/span>\u003C/h4>\u003Cp class=\"js_darkmode__27\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">这是最常见的噪声来源。原始告警规则可能是这样的：\u003C/span>\u003C/p>\u003Cpre class=\"js_darkmode__bg__21 js_darkmode__28\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 原始告警规则（噪声大）\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">alert:\u003C/span>\u003Cspan leaf=\"\">HighCPUUsage\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">100\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">(avg\u003C/span>\u003Cspan leaf=\"\">by(instance)\u003C/span>\u003Cspan leaf=\"\">(rate(node_cpu_seconds_total{mode=\"idle\"}[1m]))\u003C/span>\u003Cspan leaf=\"\">*\u003C/span>\u003Cspan leaf=\"\">100\u003C/span>\u003Cspan leaf=\"\">)\u003C/span>\u003Cspan leaf=\"\">>\u003C/span>\u003Cspan leaf=\"\">80\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">for:\u003C/span>\u003Cspan leaf=\"\">1m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Cp class=\"js_darkmode__29\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">使用 Recording Rules 优化后：\u003C/span>\u003C/p>\u003Cpre class=\"js_darkmode__bg__22 js_darkmode__30\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># recording_rules.yml\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">groups:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">cpu_smoothing\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">interval:\u003C/span>\u003Cspan leaf=\"\">30s\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 第一层：5分钟平均 CPU 使用率\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_cpu_utilization:avg5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          1 - avg by (instance) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            rate(node_cpu_seconds_total{mode=\"idle\"}[5m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 第二层：基于5分钟均值的15分钟最大值\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 这样可以捕获持续性高负载，过滤掉短暂峰值\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_cpu_utilization:max15m_avg5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          max_over_time(instance:node_cpu_utilization:avg5m[15m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 第三层：按集群聚合的 CPU 使用率\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">cluster:node_cpu_utilization:avg\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          avg(instance:node_cpu_utilization:avg5m)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Cp class=\"js_darkmode__31\" style=\"text-align: justify;\">\u003Cspan>参数说明\u003C/span>\u003Cspan leaf=\"\">：\u003C/span>\u003C/p>\u003Cul class=\"list-paddingleft-1 js_darkmode__32\" style=\"text-align: justify;\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Ccode>\u003Cspan leaf=\"\">avg5m\u003C/span>\u003C/code>\u003Cspan leaf=\"\">：使用 5 分钟窗口计算平均值，消除秒级抖动\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Ccode>\u003Cspan leaf=\"\">max15m_avg5m\u003C/span>\u003C/code>\u003Cspan leaf=\"\">：在 5 分钟均值基础上取 15 分钟最大值，只有持续性高负载才会触发\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Ccode>\u003Cspan leaf=\"\">cluster:\u003C/span>\u003C/code>\u003Cspan leaf=\"\"> 前缀：表示集群级别聚合，便于识别层级\u003C/span>\u003C/section>\u003C/li>\u003C/ul>\u003Ch4 class=\"js_darkmode__bg__23\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 2.2.3 内存使用率分层聚合\u003C/span>\u003C/h4>\u003Cpre class=\"js_darkmode__bg__24 js_darkmode__33\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\">groups:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">memory_recording_rules\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">interval:\u003C/span>\u003Cspan leaf=\"\">30s\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 节点级别内存使用率\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_memory_utilization:ratio\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          1 - (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            node_memory_MemAvailable_bytes\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            / node_memory_MemTotal_bytes\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 应用级别内存使用（Kubernetes 环境）\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">namespace_pod:container_memory_usage:sum\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          sum by (namespace, pod) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 服务级别内存使用率\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">namespace_service:memory_utilization:avg5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          avg by (namespace, service) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            avg_over_time(\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">              container_memory_working_set_bytes{container!=\"\"}[5m]\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          ) / avg by (namespace, service) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            kube_pod_container_resource_limits{resource=\"memory\"}\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch4 class=\"js_darkmode__bg__25\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 2.2.4 请求错误率聚合\u003C/span>\u003C/h4>\u003Cp class=\"js_darkmode__34\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">对于微服务场景，HTTP 错误率告警是另一个噪声重灾区：\u003C/span>\u003C/p>\u003Cpre class=\"js_darkmode__bg__26 js_darkmode__35\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\">groups:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">http_error_rate_rules\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">interval:\u003C/span>\u003Cspan leaf=\"\">15s\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 服务级别错误率（5分钟窗口）\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:http_requests_total:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          sum by (namespace, service) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            rate(http_requests_total[5m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:http_requests_errors:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          sum by (namespace, service) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            rate(http_requests_total{status=~\"5..\"}[5m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 错误率计算\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:http_error_rate:ratio5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          service:http_requests_errors:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          / service:http_requests_total:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 加权错误率：流量越大权重越高\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 避免低流量服务的单个错误触发高错误率告警\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:http_error_rate:weighted5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          (service:http_requests_errors:rate5m + 1)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          / (service:http_requests_total:rate5m + 10)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch3 class=\"js_darkmode__bg__27\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">2.3 启动和验证\u003C/span>\u003C/h3>\u003Ch4 class=\"js_darkmode__bg__28\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 2.3.1 配置热加载\u003C/span>\u003C/h4>\u003Cpre class=\"js_darkmode__bg__29 js_darkmode__36\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 检查配置语法\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">promtool check rules /etc/prometheus/rules/recording_rules.yml\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 热加载 Prometheus 配置\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">curl -X POST http://localhost:9090/-/reload\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 或者发送 SIGHUP 信号\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">kill\u003C/span>\u003Cspan leaf=\"\"> -HUP $(pgrep prometheus)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch4 class=\"js_darkmode__bg__30\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 2.3.2 验证 Recording Rules 生效\u003C/span>\u003C/h4>\u003Cpre class=\"js_darkmode__bg__31 js_darkmode__37\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 在 Prometheus UI 查询新生成的指标\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 应该能看到数据\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">instance:node_cpu_utilization:avg5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 检查 Recording Rules 状态\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">curl -s http://localhost:9090/api/v1/rules | jq \u003C/span>\u003Cspan leaf=\"\">'.data.groups[] | select(.name==\"cpu_smoothing\")'\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 预期输出应显示规则状态为 \"health\": \"ok\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Chr class=\"js_darkmode__bg__32 js_darkmode__38\" style=\"text-align: justify;\">\u003Ch2 class=\"js_darkmode__bg__33\" style=\"text-align: center;\">\u003Cspan leaf=\"\">三、示例代码和配置\u003C/span>\u003C/h2>\u003Ch3 class=\"js_darkmode__bg__34\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">3.1 完整配置示例\u003C/span>\u003C/h3>\u003Ch4 class=\"js_darkmode__bg__35\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 3.1.1 完整的 Recording Rules 配置\u003C/span>\u003C/h4>\u003Cpre class=\"js_darkmode__bg__36 js_darkmode__39\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 文件路径：/etc/prometheus/rules/recording_rules.yml\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">groups:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># ============ 基础设施层 ============\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">infrastructure_recording_rules\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">interval:\u003C/span>\u003Cspan leaf=\"\">30s\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># CPU\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_cpu_utilization:avg5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">1\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">avg\u003C/span>\u003Cspan leaf=\"\">by\u003C/span>\u003Cspan leaf=\"\">(instance)\u003C/span>\u003Cspan leaf=\"\">(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_cpu_utilization:max_avg5m_over_15m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">max_over_time(instance:node_cpu_utilization:avg5m[15m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># Memory\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_memory_utilization:ratio\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">1\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">(node_memory_MemAvailable_bytes\u003C/span>\u003Cspan leaf=\"\">/\u003C/span>\u003Cspan leaf=\"\">node_memory_MemTotal_bytes)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_memory_utilization:avg5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">avg_over_time(instance:node_memory_utilization:ratio[5m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># Disk\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_disk_utilization:ratio\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          1 - (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            node_filesystem_avail_bytes{fstype=~\"ext4|xfs\"}\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            / node_filesystem_size_bytes{fstype=~\"ext4|xfs\"}\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># Network\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_network_receive:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">sum\u003C/span>\u003Cspan leaf=\"\">by\u003C/span>\u003Cspan leaf=\"\">(instance)\u003C/span>\u003Cspan leaf=\"\">(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|br.*\"}[5m]))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_network_transmit:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">sum\u003C/span>\u003Cspan leaf=\"\">by\u003C/span>\u003Cspan leaf=\"\">(instance)\u003C/span>\u003Cspan leaf=\"\">(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|br.*\"}[5m]))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># ============ Kubernetes 层 ============\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">kubernetes_recording_rules\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">interval:\u003C/span>\u003Cspan leaf=\"\">30s\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># Pod 资源使用聚合到 namespace\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">namespace:container_cpu_usage:sum\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          sum by (namespace) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">namespace:container_memory_usage:sum\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          sum by (namespace) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># Pod 重启率（1小时窗口，避免单次重启告警）\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">namespace_pod:container_restarts:increase1h\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          increase(kube_pod_container_status_restarts_total[1h])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># Deployment 可用性\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">namespace_deployment:replicas_unavailable:ratio\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          kube_deployment_status_replicas_unavailable\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          / kube_deployment_spec_replicas\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># ============ 应用层 ============\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">application_recording_rules\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">interval:\u003C/span>\u003Cspan leaf=\"\">15s\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># HTTP 请求速率\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:http_requests:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">sum\u003C/span>\u003Cspan leaf=\"\">by\u003C/span>\u003Cspan leaf=\"\">(namespace,\u003C/span>\u003Cspan leaf=\"\">service,\u003C/span>\u003Cspan leaf=\"\">method)\u003C/span>\u003Cspan leaf=\"\">(rate(http_requests_total[5m]))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># HTTP 错误率\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:http_errors:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">sum\u003C/span>\u003Cspan leaf=\"\">by\u003C/span>\u003Cspan leaf=\"\">(namespace,\u003C/span>\u003Cspan leaf=\"\">service)\u003C/span>\u003Cspan leaf=\"\">(rate(http_requests_total{status=~\"5..\"}[5m]))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># HTTP P99 延迟\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:http_latency_p99:5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          histogram_quantile(0.99,\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            sum by (namespace, service, le) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">              rate(http_request_duration_seconds_bucket[5m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 服务健康评分（综合指标）\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:health_score:5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            1 - clamp_max(service:http_errors:rate5m / service:http_requests:rate5m, 1)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          ) * 0.4\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          +\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            1 - clamp_max(service:http_latency_p99:5m / 2, 1)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          ) * 0.3\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          +\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            clamp_max(service:http_requests:rate5m / 100, 1)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          ) * 0.3\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch4 class=\"js_darkmode__bg__37\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 3.1.2 基于 Recording Rules 的告警规则\u003C/span>\u003C/h4>\u003Cpre class=\"js_darkmode__bg__38 js_darkmode__40\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 文件路径：/etc/prometheus/rules/alert_rules.yml\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">groups:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">infrastructure_alerts\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 使用预计算指标，更稳定\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">alert:\u003C/span>\u003Cspan leaf=\"\">NodeHighCPU\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">instance:node_cpu_utilization:max_avg5m_over_15m\u003C/span>\u003Cspan leaf=\"\">>\u003C/span>\u003Cspan leaf=\"\">0.85\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">for:\u003C/span>\u003Cspan leaf=\"\">5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">labels:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">severity:\u003C/span>\u003Cspan leaf=\"\">warning\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">annotations:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">summary:\u003C/span>\u003Cspan leaf=\"\">\"节点 \u003C/span>\u003Cspan leaf=\"\">{{ $labels.instance }}\u003C/span>\u003Cspan leaf=\"\"> CPU 持续高负载\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">description:\u003C/span>\u003Cspan leaf=\"\">\"15分钟内 CPU 使用率峰值超过 85%，当前值 \u003C/span>\u003Cspan leaf=\"\">{{ $value | humanizePercentage }}\u003C/span>\u003Cspan leaf=\"\">\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">alert:\u003C/span>\u003Cspan leaf=\"\">NodeHighMemory\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">instance:node_memory_utilization:avg5m\u003C/span>\u003Cspan leaf=\"\">>\u003C/span>\u003Cspan leaf=\"\">0.9\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">for:\u003C/span>\u003Cspan leaf=\"\">10m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">labels:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">severity:\u003C/span>\u003Cspan leaf=\"\">warning\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">annotations:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">summary:\u003C/span>\u003Cspan leaf=\"\">\"节点 \u003C/span>\u003Cspan leaf=\"\">{{ $labels.instance }}\u003C/span>\u003Cspan leaf=\"\"> 内存使用率过高\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">description:\u003C/span>\u003Cspan leaf=\"\">\"内存使用率持续超过 90%\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">application_alerts\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 基于聚合指标的服务健康告警\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">alert:\u003C/span>\u003Cspan leaf=\"\">ServiceUnhealthy\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">service:health_score:5m\u003C/span>\u003Cspan leaf=\"\"><\u003C/span>\u003Cspan leaf=\"\">0.6\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">for:\u003C/span>\u003Cspan leaf=\"\">5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">labels:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">severity:\u003C/span>\u003Cspan leaf=\"\">critical\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">annotations:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">summary:\u003C/span>\u003Cspan leaf=\"\">\"服务 \u003C/span>\u003Cspan leaf=\"\">{{ $labels.namespace }}\u003C/span>\u003Cspan leaf=\"\">/\u003C/span>\u003Cspan leaf=\"\">{{ $labels.service }}\u003C/span>\u003Cspan leaf=\"\"> 健康状态异常\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">description:\u003C/span>\u003Cspan leaf=\"\">\"服务健康评分低于 0.6，当前值 \u003C/span>\u003Cspan leaf=\"\">{{ $value }}\u003C/span>\u003Cspan leaf=\"\">\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># Pod 频繁重启（使用1小时窗口）\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">alert:\u003C/span>\u003Cspan leaf=\"\">PodFrequentRestart\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">namespace_pod:container_restarts:increase1h\u003C/span>\u003Cspan leaf=\"\">>\u003C/span>\u003Cspan leaf=\"\">3\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">for:\u003C/span>\u003Cspan leaf=\"\">0m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">labels:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">severity:\u003C/span>\u003Cspan leaf=\"\">warning\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">annotations:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">summary:\u003C/span>\u003Cspan leaf=\"\">\"Pod \u003C/span>\u003Cspan leaf=\"\">{{ $labels.namespace }}\u003C/span>\u003Cspan leaf=\"\">/\u003C/span>\u003Cspan leaf=\"\">{{ $labels.pod }}\u003C/span>\u003Cspan leaf=\"\"> 频繁重启\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">description:\u003C/span>\u003Cspan leaf=\"\">\"过去1小时重启 \u003C/span>\u003Cspan leaf=\"\">{{ $value }}\u003C/span>\u003Cspan leaf=\"\"> 次\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch3 class=\"js_darkmode__bg__39\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">3.2 实际应用案例\u003C/span>\u003C/h3>\u003Ch4 class=\"js_darkmode__bg__40\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 案例一：电商大促告警收敛\u003C/span>\u003C/h4>\u003Cp class=\"js_darkmode__41\" style=\"text-align: justify;\">\u003Cspan>场景描述\u003C/span>\u003Cspan leaf=\"\">：双11大促期间，订单服务 QPS 从日常 1000 飙升到 50000，原有的错误率告警（错误数/总请求数 > 1%）频繁触发，因为即使错误率只有 0.5%，绝对错误数也达到了 250/秒。\u003C/span>\u003C/p>\u003Cp class=\"js_darkmode__42\" style=\"text-align: justify;\">\u003Cspan>实现代码\u003C/span>\u003Cspan leaf=\"\">：\u003C/span>\u003C/p>\u003Cpre class=\"js_darkmode__bg__41 js_darkmode__43\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># Recording Rules\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">groups:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">ecommerce_rules\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 动态基线：使用过去1小时的错误率作为基线\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:http_error_rate:baseline1h\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          avg_over_time(\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">              sum by (service) (rate(http_requests_total{status=~\"5..\"}[5m]))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">              / sum by (service) (rate(http_requests_total[5m]))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            )[1h:]\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 当前错误率与基线的比值\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:http_error_rate:ratio_to_baseline\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            sum by (service) (rate(http_requests_total{status=~\"5..\"}[5m]))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            / sum by (service) (rate(http_requests_total[5m]))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          / service:http_error_rate:baseline1h\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 告警规则\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">alert:\u003C/span>\u003Cspan leaf=\"\">ServiceErrorRateSpike\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">service:http_error_rate:ratio_to_baseline\u003C/span>\u003Cspan leaf=\"\">>\u003C/span>\u003Cspan leaf=\"\">3\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">for:\u003C/span>\u003Cspan leaf=\"\">5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">annotations:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">summary:\u003C/span>\u003Cspan leaf=\"\">\"服务 \u003C/span>\u003Cspan leaf=\"\">{{ $labels.service }}\u003C/span>\u003Cspan leaf=\"\"> 错误率异常飙升\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">description:\u003C/span>\u003Cspan leaf=\"\">\"当前错误率是基线的 \u003C/span>\u003Cspan leaf=\"\">{{ $value | humanize }}\u003C/span>\u003Cspan leaf=\"\"> 倍\"\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Cp class=\"js_darkmode__44\" style=\"text-align: justify;\">\u003Cspan>运行结果\u003C/span>\u003Cspan leaf=\"\">：\u003C/span>\u003C/p>\u003Cpre class=\"js_darkmode__bg__42 js_darkmode__45\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\">大促期间告警数量对比：\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">- 优化前：每小时平均 150+ 条告警\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">- 优化后：每小时平均 5-10 条告警\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">- 误告警率：从 85% 降低到 15%\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch4 class=\"js_darkmode__bg__43\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 案例二：多租户环境告警聚合\u003C/span>\u003C/h4>\u003Cp class=\"js_darkmode__46\" style=\"text-align: justify;\">\u003Cspan>场景描述\u003C/span>\u003Cspan leaf=\"\">：SaaS 平台有 200+ 租户，每个租户有独立的命名空间。当某个底层节点故障时，原有配置会触发 200+ 条 Pod 不可用告警。\u003C/span>\u003C/p>\u003Cp class=\"js_darkmode__47\" style=\"text-align: justify;\">\u003Cspan>实现步骤\u003C/span>\u003Cspan leaf=\"\">：\u003C/span>\u003C/p>\u003Col class=\"list-paddingleft-1 js_darkmode__48\" style=\"text-align: justify;\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">1. 创建租户级别的聚合指标\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">2. 实现告警收敛，只告警受影响的节点，附带影响的租户列表\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">3. 配合 Alertmanager 的 group_by 实现进一步收敛\u003C/span>\u003C/section>\u003C/li>\u003C/ol>\u003Cpre class=\"js_darkmode__bg__44 js_darkmode__49\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># Recording Rules\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">groups:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">name:\u003C/span>\u003Cspan leaf=\"\">tenant_aggregation\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 按租户聚合不可用 Pod 数量\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">tenant:pods_unavailable:count\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          count by (tenant) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            kube_pod_status_phase{phase!=\"Running\", phase!=\"Succeeded\"}\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 租户服务可用性\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">tenant:service_availability:ratio\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          sum by (tenant) (kube_deployment_status_replicas_available)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          / sum by (tenant) (kube_deployment_spec_replicas)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 故障影响范围\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">node:affected_tenants:count\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          count by (node) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            count by (node, tenant) (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">              kube_pod_info * on(pod, namespace) group_left(tenant)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">              kube_namespace_labels\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">            )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">          )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Chr class=\"js_darkmode__bg__45 js_darkmode__50\" style=\"text-align: justify;\">\u003Ch2 class=\"js_darkmode__bg__46\" style=\"text-align: center;\">\u003Cspan leaf=\"\">四、最佳实践和注意事项\u003C/span>\u003C/h2>\u003Ch3 class=\"js_darkmode__bg__47\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">4.1 最佳实践\u003C/span>\u003C/h3>\u003Ch4 class=\"js_darkmode__bg__48\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 4.1.1 Recording Rules 命名规范\u003C/span>\u003C/h4>\u003Cp class=\"js_darkmode__51\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">遵循一致的命名规范对于维护大规模 Recording Rules 至关重要：\u003C/span>\u003C/p>\u003Cpre class=\"js_darkmode__bg__49 js_darkmode__52\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 命名格式：level:metric_name:aggregation_window\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># level: 聚合级别，如 instance, namespace, cluster\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># metric_name: 指标名称\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># aggregation_window: 时间窗口和聚合方式\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 好的命名示例\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">instance:node_cpu_utilization:avg5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">namespace:container_memory:\u003C/span>\u003Cspan leaf=\"\">sum\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">cluster:http_requests:rate15m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 避免的命名方式\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">cpu_high                    \u003C/span>\u003Cspan leaf=\"\"># 缺少层级和时间窗口信息\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">recording_rule_1            \u003C/span>\u003Cspan leaf=\"\"># 无意义的名称\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch4 class=\"js_darkmode__bg__50\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 4.1.2 分层设计\u003C/span>\u003C/h4>\u003Cpre class=\"js_darkmode__bg__51 js_darkmode__53\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># 三层架构设计\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 第一层：原始指标清洗和标准化\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_cpu_seconds:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">rate(node_cpu_seconds_total[5m])\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 第二层：指标聚合\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">instance:node_cpu_utilization:avg5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">1\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">avg\u003C/span>\u003Cspan leaf=\"\">by\u003C/span>\u003Cspan leaf=\"\">(instance)\u003C/span>\u003Cspan leaf=\"\">(instance:node_cpu_seconds:rate5m{mode=\"idle\"})\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 第三层：业务指标\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">service:availability:ratio\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">|\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">    (\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">      sum(instance:node_cpu_utilization:avg5m < 0.9)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">      / count(instance:node_cpu_utilization:avg5m)\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">    )\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch4 class=\"js_darkmode__bg__52\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 4.1.3 性能优化\u003C/span>\u003C/h4>\u003Cul class=\"list-paddingleft-1 js_darkmode__54\" style=\"text-align: justify;\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>控制 Recording Rules 数量\u003C/span>\u003Cspan leaf=\"\">：每增加一条规则，就会产生新的时间序列\u003C/span>\u003Cpre class=\"js_darkmode__bg__53 js_darkmode__55\">\u003Ccode>\u003Cspan leaf=\"\"># 检查 Recording Rules 产生的时序数量\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">prometheus_tsdb_head_series\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 建议：单个 Prometheus 实例的 Recording Rules 产生的时序不超过总时序的 20%\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>合理设置 interval\u003C/span>\u003Cspan leaf=\"\">：根据指标变化频率设置\u003C/span>\u003C/section>\u003C/li>\u003Cul class=\"list-paddingleft-1 js_darkmode__56\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 基础设施指标：30s\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 应用指标：15s\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 业务指标：60s\u003C/span>\u003C/section>\u003C/li>\u003C/ul>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>使用 limit 防止基数爆炸\u003C/span>\u003Cspan leaf=\"\">：\u003C/span>\u003Cpre class=\"js_darkmode__bg__54 js_darkmode__57\">\u003Ccode>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">record:\u003C/span>\u003Cspan leaf=\"\">topk_services:http_requests:rate5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">expr:\u003C/span>\u003Cspan leaf=\"\">topk(100,\u003C/span>\u003Cspan leaf=\"\">sum\u003C/span>\u003Cspan leaf=\"\">by\u003C/span>\u003Cspan leaf=\"\">(service)\u003C/span>\u003Cspan leaf=\"\">(rate(http_requests_total[5m])))\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003C/section>\u003C/li>\u003C/ul>\u003Ch4 class=\"js_darkmode__bg__55\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 4.1.4 高可用配置\u003C/span>\u003C/h4>\u003Cul class=\"list-paddingleft-1 js_darkmode__58\" style=\"text-align: justify;\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>多副本一致性\u003C/span>\u003Cspan leaf=\"\">：Recording Rules 在多个 Prometheus 副本上执行，确保配置完全相同\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>告警去重\u003C/span>\u003Cspan leaf=\"\">：配合 Alertmanager 的 \u003C/span>\u003Ccode>\u003Cspan leaf=\"\">group_by\u003C/span>\u003C/code>\u003Cspan leaf=\"\"> 和 \u003C/span>\u003Ccode>\u003Cspan leaf=\"\">inhibit_rules\u003C/span>\u003C/code>\u003Cspan leaf=\"\"> 进一步去重\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>联邦集群\u003C/span>\u003Cspan leaf=\"\">：在联邦架构中，Recording Rules 应该在被采集端执行，减少联邦查询压力\u003C/span>\u003C/section>\u003C/li>\u003C/ul>\u003Cpre class=\"js_darkmode__bg__56 js_darkmode__59\" style=\"text-align: justify;\">\u003Ccode>\u003Cspan leaf=\"\"># Alertmanager 配置配合\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">route:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">group_by:\u003C/span>\u003Cspan leaf=\"\"> [\u003C/span>\u003Cspan leaf=\"\">'alertname'\u003C/span>\u003Cspan leaf=\"\">, \u003C/span>\u003Cspan leaf=\"\">'cluster'\u003C/span>\u003Cspan leaf=\"\">, \u003C/span>\u003Cspan leaf=\"\">'service'\u003C/span>\u003Cspan leaf=\"\">]\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">group_wait:\u003C/span>\u003Cspan leaf=\"\">30s\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">group_interval:\u003C/span>\u003Cspan leaf=\"\">5m\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">repeat_interval:\u003C/span>\u003Cspan leaf=\"\">4h\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">inhibit_rules:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\"># 节点故障时抑制该节点上的所有 Pod 告警\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">-\u003C/span>\u003Cspan leaf=\"\">source_match:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">alertname:\u003C/span>\u003Cspan leaf=\"\">NodeDown\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">target_match_re:\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">alertname:\u003C/span>\u003Cspan leaf=\"\">Pod.*\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003Cspan leaf=\"\">equal:\u003C/span>\u003Cspan leaf=\"\"> [\u003C/span>\u003Cspan leaf=\"\">'node'\u003C/span>\u003Cspan leaf=\"\">]\u003C/span>\u003Cspan leaf=\"\">\u003Cbr>\u003C/span>\u003C/code>\u003C/pre>\u003Ch3 class=\"js_darkmode__bg__57\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">4.2 注意事项\u003C/span>\u003C/h3>\u003Ch4 class=\"js_darkmode__bg__58\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 4.2.1 配置注意事项\u003C/span>\u003C/h4>\u003Cp class=\"js_darkmode__60\" style=\"text-align: justify;\">\u003Cspan>警告\u003C/span>\u003Cspan leaf=\"\">：Recording Rules 配置错误可能导致 Prometheus 无法启动或产生错误数据\u003C/span>\u003C/p>\u003Cul class=\"list-paddingleft-1 js_darkmode__61\" style=\"text-align: justify;\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 注意事项一：Recording Rules 的 \u003C/span>\u003Ccode>\u003Cspan leaf=\"\">record\u003C/span>\u003C/code>\u003Cspan leaf=\"\"> 字段必须是合法的指标名称，不能包含特殊字符\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 注意事项二：避免 Recording Rules 之间的循环依赖，虽然 Prometheus 会检测，但可能导致计算延迟\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• 注意事项三：修改已有 Recording Rules 的表达式时，注意下游依赖的告警规则可能需要同步调整阈值\u003C/span>\u003C/section>\u003C/li>\u003C/ul>\u003Ch4 class=\"js_darkmode__bg__59\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 4.2.2 常见错误\u003C/span>\u003C/h4>\u003Ctable class=\"js_darkmode__62 layui-table\" style=\"text-align: justify;\">\u003Cthead>\u003Ctr>\u003Cth class=\"js_darkmode__bg__60\" style=\"text-align: left;\">\u003Csection>\u003Cspan leaf=\"\">错误现象\u003C/span>\u003C/section>\u003C/th>\u003Cth class=\"js_darkmode__bg__61\" style=\"text-align: left;\">\u003Csection>\u003Cspan leaf=\"\">原因分析\u003C/span>\u003C/section>\u003C/th>\u003Cth class=\"js_darkmode__bg__62\" style=\"text-align: left;\">\u003Csection>\u003Cspan leaf=\"\">解决方案\u003C/span>\u003C/section>\u003C/th>\u003C/tr>\u003C/thead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"js_darkmode__63\">\u003Csection>\u003Cspan leaf=\"\">Recording Rule 显示 health: unknown\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__64\">\u003Csection>\u003Cspan leaf=\"\">表达式语法错误或引用的指标不存在\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__65\">\u003Csection>\u003Cspan leaf=\"\">使用 promtool check rules 验证语法，确认源指标存在\u003C/span>\u003C/section>\u003C/td>\u003C/tr>\u003Ctr>\u003Ctd class=\"js_darkmode__66\">\u003Csection>\u003Cspan leaf=\"\">新指标无数据\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__67\">\u003Csection>\u003Cspan leaf=\"\">interval 设置过长或表达式结果为空\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__68\">\u003Csection>\u003Cspan leaf=\"\">检查 interval 配置，在 Prometheus UI 测试表达式\u003C/span>\u003C/section>\u003C/td>\u003C/tr>\u003Ctr>\u003Ctd class=\"js_darkmode__69\">\u003Csection>\u003Cspan leaf=\"\">Prometheus OOM\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__70\">\u003Csection>\u003Cspan leaf=\"\">Recording Rules 产生的时序过多\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__71\">\u003Csection>\u003Cspan leaf=\"\">添加 limit、优化标签基数、增加内存\u003C/span>\u003C/section>\u003C/td>\u003C/tr>\u003Ctr>\u003Ctd class=\"js_darkmode__72\">\u003Csection>\u003Cspan leaf=\"\">告警延迟增加\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__73\">\u003Csection>\u003Cspan leaf=\"\">Recording Rules 计算压力过大\u003C/span>\u003C/section>\u003C/td>\u003Ctd class=\"js_darkmode__74\">\u003Csection>\u003Cspan leaf=\"\">减少规则数量、增大 interval、升级硬件\u003C/span>\u003C/section>\u003C/td>\u003C/tr>\u003C/tbody>\u003C/table>\u003Ch4 class=\"js_darkmode__bg__63\" style=\"text-align: justify;\">\u003Cspan leaf=\"\">◆ 4.2.3 兼容性问题\u003C/span>\u003C/h4>\u003Cul class=\"list-paddingleft-1 js_darkmode__75\" style=\"text-align: justify;\">\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>版本兼容\u003C/span>\u003Cspan leaf=\"\">：\u003C/span>\u003Ccode>\u003Cspan leaf=\"\">limit\u003C/span>\u003C/code>\u003Cspan leaf=\"\"> 功能需要 Prometheus 2.28+，低版本需要在表达式中使用 \u003C/span>\u003Ccode>\u003Cspan leaf=\"\">topk\u003C/span>\u003C/code>\u003Cspan leaf=\"\"> 替代\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>平台兼容\u003C/span>\u003Cspan leaf=\"\">：使用 Prometheus Operator 时，Recording Rules 需要通过 PrometheusRule CRD 管理\u003C/span>\u003C/section>\u003C/li>\u003Cli>\u003Csection>\u003Cspan leaf=\"\">• \u003C/span>\u003Cspan>组件依赖\u003C/span>\u003Cspan leaf=\"\">：Recording Rules 依赖的源指标（如 node_exporter、kube-state-metrics）版本变更可能导致表达式失效\u003C/span>\u003C/section>\u003C/li>\u003C/ul>\u003Chr class=\"js_darkmode__bg__64 js_darkmode__76\" style=\"text-align: justify;\">\u003Ch2 class=\"js_darkmode__bg__65\" style=\"text-align: center;\">\u003Cbr>\u003C/h2>","## 一、概述\n\n### 1.1 背景介绍\n\n在大规模微服务架构下，Prometheus 告警系统往往会陷入一个尴尬的境地：告警太多，运维团队开始选择性忽略；告警太少，真正的故障又可能漏掉。我在某电商平台负责监控体系建设时，团队每天要处理超过 2000 条告警，其中 70% 以上是重复的、关联的或者短暂抖动产生的噪声。\n\nRecording Rules 是 Prometheus 提供的预计算机制，可以将复杂的查询表达式预先计算并存储为新的时间序列。通过合理设计 Recording Rules，我们不仅能显著降低 PromQL 查询压力，更重要的是可以实现告警聚合、去重、平滑，从根本上减少告警噪声。\n\n### 1.2 技术特点\n\n- • 预计算优化：将复杂的聚合查询预先计算，避免告警规则重复计算相同的表达式，降低 Prometheus 负载\n- • 时间窗口平滑：通过 avg_over_time、max_over_time 等函数消除瞬时抖动，避免单点毛刺触发告警\n- • 多维度聚合：支持按 namespace、service、pod 等维度聚合指标，实现告警收敛\n- • 层级化告警：基于 Recording Rules 构建指标层级，实现从基础指标到业务指标的逐层抽象\n\n### 1.3 适用场景\n\n- • 场景一：大规模 Kubernetes 集群监控，Pod 数量超过 1000，原生指标产生的告警风暴需要收敛\n- • 场景二：微服务架构下，服务间依赖复杂，需要将底层告警聚合为服务级别的健康状态\n- • 场景三：业务高峰期流量抖动频繁，需要平滑处理避免误告警\n- • 场景四：多租户环境下，需要按租户/业务线聚合告警，避免一个故障触发上百条告警\n\n### 1.4 环境要求\n\n组件\n\n版本要求\n\n说明\n\nPrometheus\n\n2.28+\n\n支持 Recording Rules 分组和 limit 特性\n\nAlertmanager\n\n0.23+\n\n配合使用告警抑制和分组功能\n\nKubernetes\n\n1.21+\n\n如果使用 kube-prometheus-stack\n\n内存配置\n\n建议 8GB+\n\nRecording Rules 会增加时序存储量\n\n---\n\n## 二、详细步骤\n\n### 2.1 准备工作\n\n#### ◆ 2.1.1 现状分析\n\n在动手之前，先要搞清楚当前告警噪声的分布。我习惯用下面这个查询来分析过去一周的告警触发情况：\n\n```\n# 查看 Prometheus 配置kubectl get configmap prometheus-server -n monitoring -o yaml# 分析告警触发频次（在 Prometheus UI 执行）# 统计过去7天各告警规则触发次数count_over_time(ALERTS{alertstate=\"firing\"}[7d])# 查看当前活跃告警数量count(ALERTS{alertstate=\"firing\"})\n```\n\n#### ◆ 2.1.2 识别噪声来源\n\n根据我的经验，告警噪声主要来自以下几类：\n\n```\n# 检查高频告警（每分钟触发超过10次的规则）# 在 Prometheus 执行topk(20, count by (alertname) (count_over_time(ALERTS{alertstate=\"firing\"}[1h])))# 检查短暂告警（持续时间小于5分钟的）# 这类告警往往是抖动产生的噪声\n```\n\n通常会发现以下几类高频噪声：\n\n- • CPU/内存使用率瞬时超阈值\n- • Pod 重启计数器误报\n- • 网络延迟抖动\n- • 磁盘 IO 突增\n\n### 2.2 核心配置\n\n#### ◆ 2.2.1 Recording Rules 基础结构\n\n```\n# 文件路径：/etc/prometheus/rules/recording_rules.ymlgroups:-name:node_recording_rulesinterval:30s# 计算间隔，根据需求调整rules:# 规则定义-record:job:node_cpu_utilization:avg5mexpr:|          1 - avg by (job, instance) (            rate(node_cpu_seconds_total{mode=\"idle\"}[5m])          )\n```\n\n说明\n\n：\n`\ninterval\n`\n 参数控制 Recording Rules 的计算频率。设置过小会增加计算压力，过大则可能错过关键变化。一般建议设置为告警评估间隔的一半。\n\n#### ◆ 2.2.2 CPU 使用率平滑处理\n\n这是最常见的噪声来源。原始告警规则可能是这样的：\n\n```\n# 原始告警规则（噪声大）-alert:HighCPUUsageexpr:100-(avgby(instance)(rate(node_cpu_seconds_total{mode=\"idle\"}[1m]))*100)>80for:1m\n```\n\n使用 Recording Rules 优化后：\n\n```\n# recording_rules.ymlgroups:-name:cpu_smoothinginterval:30srules:# 第一层：5分钟平均 CPU 使用率-record:instance:node_cpu_utilization:avg5mexpr:|          1 - avg by (instance) (            rate(node_cpu_seconds_total{mode=\"idle\"}[5m])          )# 第二层：基于5分钟均值的15分钟最大值# 这样可以捕获持续性高负载，过滤掉短暂峰值-record:instance:node_cpu_utilization:max15m_avg5mexpr:|          max_over_time(instance:node_cpu_utilization:avg5m[15m])# 第三层：按集群聚合的 CPU 使用率-record:cluster:node_cpu_utilization:avgexpr:|          avg(instance:node_cpu_utilization:avg5m)\n```\n\n参数说明\n\n：\n\n- • avg5m：使用 5 分钟窗口计算平均值，消除秒级抖动\n- • max15m_avg5m：在 5 分钟均值基础上取 15 分钟最大值，只有持续性高负载才会触发\n- • cluster: 前缀：表示集群级别聚合，便于识别层级\n\n#### ◆ 2.2.3 内存使用率分层聚合\n\n```\ngroups:-name:memory_recording_rulesinterval:30srules:# 节点级别内存使用率-record:instance:node_memory_utilization:ratioexpr:|          1 - (            node_memory_MemAvailable_bytes            / node_memory_MemTotal_bytes          )# 应用级别内存使用（Kubernetes 环境）-record:namespace_pod:container_memory_usage:sumexpr:|          sum by (namespace, pod) (            container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}          )# 服务级别内存使用率-record:namespace_service:memory_utilization:avg5mexpr:|          avg by (namespace, service) (            avg_over_time(              container_memory_working_set_bytes{container!=\"\"}[5m]            )          ) / avg by (namespace, service) (            kube_pod_container_resource_limits{resource=\"memory\"}          )\n```\n\n#### ◆ 2.2.4 请求错误率聚合\n\n对于微服务场景，HTTP 错误率告警是另一个噪声重灾区：\n\n```\ngroups:-name:http_error_rate_rulesinterval:15srules:# 服务级别错误率（5分钟窗口）-record:service:http_requests_total:rate5mexpr:|          sum by (namespace, service) (            rate(http_requests_total[5m])          )-record:service:http_requests_errors:rate5mexpr:|          sum by (namespace, service) (            rate(http_requests_total{status=~\"5..\"}[5m])          )# 错误率计算-record:service:http_error_rate:ratio5mexpr:|          service:http_requests_errors:rate5m          / service:http_requests_total:rate5m# 加权错误率：流量越大权重越高# 避免低流量服务的单个错误触发高错误率告警-record:service:http_error_rate:weighted5mexpr:|          (service:http_requests_errors:rate5m + 1)          / (service:http_requests_total:rate5m + 10)\n```\n\n### 2.3 启动和验证\n\n#### ◆ 2.3.1 配置热加载\n\n```\n# 检查配置语法promtool check rules /etc/prometheus/rules/recording_rules.yml# 热加载 Prometheus 配置curl -X POST http://localhost:9090/-/reload# 或者发送 SIGHUP 信号kill -HUP $(pgrep prometheus)\n```\n\n#### ◆ 2.3.2 验证 Recording Rules 生效\n\n```\n# 在 Prometheus UI 查询新生成的指标# 应该能看到数据instance:node_cpu_utilization:avg5m# 检查 Recording Rules 状态curl -s http://localhost:9090/api/v1/rules | jq '.data.groups[] | select(.name==\"cpu_smoothing\")'# 预期输出应显示规则状态为 \"health\": \"ok\"\n```\n\n---\n\n## 三、示例代码和配置\n\n### 3.1 完整配置示例\n\n#### ◆ 3.1.1 完整的 Recording Rules 配置\n\n```\n# 文件路径：/etc/prometheus/rules/recording_rules.ymlgroups:# ============ 基础设施层 ============-name:infrastructure_recording_rulesinterval:30srules:# CPU-record:instance:node_cpu_utilization:avg5mexpr:1-avgby(instance)(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))-record:instance:node_cpu_utilization:max_avg5m_over_15mexpr:max_over_time(instance:node_cpu_utilization:avg5m[15m])# Memory-record:instance:node_memory_utilization:ratioexpr:1-(node_memory_MemAvailable_bytes/node_memory_MemTotal_bytes)-record:instance:node_memory_utilization:avg5mexpr:avg_over_time(instance:node_memory_utilization:ratio[5m])# Disk-record:instance:node_disk_utilization:ratioexpr:|          1 - (            node_filesystem_avail_bytes{fstype=~\"ext4|xfs\"}            / node_filesystem_size_bytes{fstype=~\"ext4|xfs\"}          )# Network-record:instance:node_network_receive:rate5mexpr:sumby(instance)(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|br.*\"}[5m]))-record:instance:node_network_transmit:rate5mexpr:sumby(instance)(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|br.*\"}[5m]))# ============ Kubernetes 层 ============-name:kubernetes_recording_rulesinterval:30srules:# Pod 资源使用聚合到 namespace-record:namespace:container_cpu_usage:sumexpr:|          sum by (namespace) (            rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])          )-record:namespace:container_memory_usage:sumexpr:|          sum by (namespace) (            container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}          )# Pod 重启率（1小时窗口，避免单次重启告警）-record:namespace_pod:container_restarts:increase1hexpr:|          increase(kube_pod_container_status_restarts_total[1h])# Deployment 可用性-record:namespace_deployment:replicas_unavailable:ratioexpr:|          kube_deployment_status_replicas_unavailable          / kube_deployment_spec_replicas# ============ 应用层 ============-name:application_recording_rulesinterval:15srules:# HTTP 请求速率-record:service:http_requests:rate5mexpr:sumby(namespace,service,method)(rate(http_requests_total[5m]))# HTTP 错误率-record:service:http_errors:rate5mexpr:sumby(namespace,service)(rate(http_requests_total{status=~\"5..\"}[5m]))# HTTP P99 延迟-record:service:http_latency_p99:5mexpr:|          histogram_quantile(0.99,            sum by (namespace, service, le) (              rate(http_request_duration_seconds_bucket[5m])            )          )# 服务健康评分（综合指标）-record:service:health_score:5mexpr:|          (            1 - clamp_max(service:http_errors:rate5m / service:http_requests:rate5m, 1)          ) * 0.4          +          (            1 - clamp_max(service:http_latency_p99:5m / 2, 1)          ) * 0.3          +          (            clamp_max(service:http_requests:rate5m / 100, 1)          ) * 0.3\n```\n\n#### ◆ 3.1.2 基于 Recording Rules 的告警规则\n\n```\n# 文件路径：/etc/prometheus/rules/alert_rules.ymlgroups:-name:infrastructure_alertsrules:# 使用预计算指标，更稳定-alert:NodeHighCPUexpr:instance:node_cpu_utilization:max_avg5m_over_15m>0.85for:5mlabels:severity:warningannotations:summary:\"节点 {{ $labels.instance }} CPU 持续高负载\"description:\"15分钟内 CPU 使用率峰值超过 85%，当前值 {{ $value | humanizePercentage }}\"-alert:NodeHighMemoryexpr:instance:node_memory_utilization:avg5m>0.9for:10mlabels:severity:warningannotations:summary:\"节点 {{ $labels.instance }} 内存使用率过高\"description:\"内存使用率持续超过 90%\"-name:application_alertsrules:# 基于聚合指标的服务健康告警-alert:ServiceUnhealthyexpr:service:health_score:5m3for:0mlabels:severity:warningannotations:summary:\"Pod {{ $labels.namespace }}/{{ $labels.pod }} 频繁重启\"description:\"过去1小时重启 {{ $value }} 次\"\n```\n\n### 3.2 实际应用案例\n\n#### ◆ 案例一：电商大促告警收敛\n\n场景描述\n\n：双11大促期间，订单服务 QPS 从日常 1000 飙升到 50000，原有的错误率告警（错误数/总请求数 > 1%）频繁触发，因为即使错误率只有 0.5%，绝对错误数也达到了 250/秒。\n\n实现代码\n\n：\n\n```\n# Recording Rulesgroups:-name:ecommerce_rulesrules:# 动态基线：使用过去1小时的错误率作为基线-record:service:http_error_rate:baseline1hexpr:|          avg_over_time(            (              sum by (service) (rate(http_requests_total{status=~\"5..\"}[5m]))              / sum by (service) (rate(http_requests_total[5m]))            )[1h:]          )# 当前错误率与基线的比值-record:service:http_error_rate:ratio_to_baselineexpr:|          (            sum by (service) (rate(http_requests_total{status=~\"5..\"}[5m]))            / sum by (service) (rate(http_requests_total[5m]))          )          / service:http_error_rate:baseline1h# 告警规则-alert:ServiceErrorRateSpikeexpr:service:http_error_rate:ratio_to_baseline>3for:5mannotations:summary:\"服务 {{ $labels.service }} 错误率异常飙升\"description:\"当前错误率是基线的 {{ $value | humanize }} 倍\"\n```\n\n运行结果\n\n：\n\n```\n大促期间告警数量对比：- 优化前：每小时平均 150+ 条告警- 优化后：每小时平均 5-10 条告警- 误告警率：从 85% 降低到 15%\n```\n\n#### ◆ 案例二：多租户环境告警聚合\n\n场景描述\n\n：SaaS 平台有 200+ 租户，每个租户有独立的命名空间。当某个底层节点故障时，原有配置会触发 200+ 条 Pod 不可用告警。\n\n实现步骤\n\n：\n\n1. 1. 创建租户级别的聚合指标\n2. 2. 实现告警收敛，只告警受影响的节点，附带影响的租户列表\n3. 3. 配合 Alertmanager 的 group_by 实现进一步收敛\n\n```\n# Recording Rulesgroups:-name:tenant_aggregationrules:# 按租户聚合不可用 Pod 数量-record:tenant:pods_unavailable:countexpr:|          count by (tenant) (            kube_pod_status_phase{phase!=\"Running\", phase!=\"Succeeded\"}          )# 租户服务可用性-record:tenant:service_availability:ratioexpr:|          sum by (tenant) (kube_deployment_status_replicas_available)          / sum by (tenant) (kube_deployment_spec_replicas)# 故障影响范围-record:node:affected_tenants:countexpr:|          count by (node) (            count by (node, tenant) (              kube_pod_info * on(pod, namespace) group_left(tenant)              kube_namespace_labels            )          )\n```\n\n---\n\n## 四、最佳实践和注意事项\n\n### 4.1 最佳实践\n\n#### ◆ 4.1.1 Recording Rules 命名规范\n\n遵循一致的命名规范对于维护大规模 Recording Rules 至关重要：\n\n```\n# 命名格式：level:metric_name:aggregation_window# level: 聚合级别，如 instance, namespace, cluster# metric_name: 指标名称# aggregation_window: 时间窗口和聚合方式# 好的命名示例instance:node_cpu_utilization:avg5mnamespace:container_memory:sumcluster:http_requests:rate15m# 避免的命名方式cpu_high                    # 缺少层级和时间窗口信息recording_rule_1            # 无意义的名称\n```\n\n#### ◆ 4.1.2 分层设计\n\n```\n# 三层架构设计# 第一层：原始指标清洗和标准化-record:instance:node_cpu_seconds:rate5mexpr:rate(node_cpu_seconds_total[5m])# 第二层：指标聚合-record:instance:node_cpu_utilization:avg5mexpr:1-avgby(instance)(instance:node_cpu_seconds:rate5m{mode=\"idle\"})# 第三层：业务指标-record:service:availability:ratioexpr:|    (      sum(instance:node_cpu_utilization:avg5m \u003C 0.9)      / count(instance:node_cpu_utilization:avg5m)    )\n```\n\n#### ◆ 4.1.3 性能优化\n\n- • 控制 Recording Rules 数量：每增加一条规则，就会产生新的时间序列\n```\n# 检查 Recording Rules 产生的时序数量prometheus_tsdb_head_series# 建议：单个 Prometheus 实例的 Recording Rules 产生的时序不超过总时序的 20%\n```\n- • 合理设置 interval：根据指标变化频率设置\n- • 基础设施指标：30s\n- • 应用指标：15s\n- • 业务指标：60s\n\n• \n\n使用 limit 防止基数爆炸\n\n：\n\n```\n-record:topk_services:http_requests:rate5mexpr:topk(100,sumby(service)(rate(http_requests_total[5m])))\n```\n\n#### ◆ 4.1.4 高可用配置\n\n- • 多副本一致性：Recording Rules 在多个 Prometheus 副本上执行，确保配置完全相同\n- • 告警去重：配合 Alertmanager 的 group_by 和 inhibit_rules 进一步去重\n- • 联邦集群：在联邦架构中，Recording Rules 应该在被采集端执行，减少联邦查询压力\n\n```\n# Alertmanager 配置配合route:group_by: ['alertname', 'cluster', 'service']group_wait:30sgroup_interval:5mrepeat_interval:4hinhibit_rules:# 节点故障时抑制该节点上的所有 Pod 告警-source_match:alertname:NodeDowntarget_match_re:alertname:Pod.*equal: ['node']\n```\n\n### 4.2 注意事项\n\n#### ◆ 4.2.1 配置注意事项\n\n警告\n\n：Recording Rules 配置错误可能导致 Prometheus 无法启动或产生错误数据\n\n- • 注意事项一：Recording Rules 的 record 字段必须是合法的指标名称，不能包含特殊字符\n- • 注意事项二：避免 Recording Rules 之间的循环依赖，虽然 Prometheus 会检测，但可能导致计算延迟\n- • 注意事项三：修改已有 Recording Rules 的表达式时，注意下游依赖的告警规则可能需要同步调整阈值\n\n#### ◆ 4.2.2 常见错误\n\n错误现象\n\n原因分析\n\n解决方案\n\nRecording Rule 显示 health: unknown\n\n表达式语法错误或引用的指标不存在\n\n使用 promtool check rules 验证语法，确认源指标存在\n\n新指标无数据\n\ninterval 设置过长或表达式结果为空\n\n检查 interval 配置，在 Prometheus UI 测试表达式\n\nPrometheus OOM\n\nRecording Rules 产生的时序过多\n\n添加 limit、优化标签基数、增加内存\n\n告警延迟增加\n\nRecording Rules 计算压力过大\n\n减少规则数量、增大 interval、升级硬件\n\n#### ◆ 4.2.3 兼容性问题\n\n- • 版本兼容：limit 功能需要 Prometheus 2.28+，低版本需要在表达式中使用 topk 替代\n- • 平台兼容：使用 Prometheus Operator 时，Recording Rules 需要通过 PrometheusRule CRD 管理\n- • 组件依赖：Recording Rules 依赖的源指标（如 node_exporter、kube-state-metrics）版本变更可能导致表达式失效\n\n---\n\n##","/article/119",301,"2026-01-06 11:51:58","2026-01-06 11:53:45",[155,156,157],{"id":39,"name":81,"url":82},{"id":44,"name":23,"url":59},{"id":145,"name":146,"url":150},{"create_time":6,"description":6,"id":44,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":23,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":53,"subtitle":6,"update_time":6,"url":59},{"title":160,"url":161},"用 Prometheus Recording Rules 把告警噪声砍掉 70%(二)","/article/120",{"title":163,"url":164},"GitOps 落地实践：ArgoCD + Kustomize 实现声明式基础设施管理(二)","/article/118",[166,167,168,169],{"create_time":6,"description":6,"id":44,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":23,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":53,"subtitle":6,"update_time":6,"url":59},{"create_time":6,"description":6,"id":45,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":61,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":56,"subtitle":6,"update_time":6,"url":62},{"create_time":6,"description":6,"id":46,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":64,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":65,"subtitle":6,"update_time":6,"url":66},{"create_time":6,"description":6,"id":47,"image_url":6,"index_template":52,"is_cover":53,"is_menu":53,"keywords":6,"list_template":54,"model_code":55,"model_id":56,"model_name":57,"name":68,"parent_id":39,"seo_description":6,"seo_keywords":6,"seo_title":6,"show_template":58,"sort":69,"subtitle":6,"update_time":6,"url":70},{"description":147,"keywords":6,"title":146},[172,181,190,199,208,217,226,235,244,252],{"id":173,"category_id":44,"title":174,"keywords":175,"description":176,"image_url":6,"url":177,"hits":178,"is_recommend":73,"is_top":73,"create_time":179,"update_time":180},101,"开启 HTTPS 并获得 ssllabs 满分的过程","开启,获得,满分,过程","准备工作确保你要申请证书的域名都解析到了这台服务器上，且能直接通过域名访问。使用官网推荐的CertBot获取证书。在CertBot官网选择一下环境(比如我选Nginx on Ubuntu 17.04)就可以看到入门教程了。安装CertBot12345apt-get updateapt-get install software-properties-commonadd-apt-repository ppa:certbot/certbotapt-get updateapt-get","/article/101",18546,"2019-11-26 16:12:11","2019-11-26 16:23:16",{"id":182,"category_id":44,"title":183,"keywords":184,"description":185,"image_url":6,"url":186,"hits":187,"is_recommend":73,"is_top":73,"create_time":188,"update_time":189},113,"在CentOS 7中添加命令自动补全功能","命令自动补全,centos","在CentOS 7中，默认情况下并不会安装命令补全包，需要手动安装才能使用命令补全功能。以下是在CentOS 7中安装命令补全包的方法：1. bash-completion：这是一个针对Bash shell的命令补全软件包，可以提供对系统命令、用户自定义命令和文件路径的自动补全功能。可以通过以下命令安装：```sudo yum install bash-completion```安装完成后，需要在/etc/profile配置文件中添加以下内容：```if [ -f /etc/bash_compl","/article/113",14948,"2023-05-16 10:54:01","2023-05-16 10:57:28",{"id":191,"category_id":44,"title":192,"keywords":193,"description":194,"image_url":6,"url":195,"hits":196,"is_recommend":53,"is_top":73,"create_time":197,"update_time":198},94,"Centos7 安装 openvas ","openvas,开放式漏洞评估系统，installing openvas centos-7","一、描述OpenVAS，即开放式漏洞评估系统，是一个用于评估目标漏洞的杰出框架。功能十分强大，最重要的是，它是“开源”的——就是免费的意思啦～它与著名的Nessus“本是同根生”，在Nessus商业化之后仍然坚持开源，号称“当前最好用的开源漏洞扫描工具”。最新版的Kali Linux(kali 3.0)不再自带OpenVAS了，所以我们要自己部署OpenVAS漏洞检测系统。其核心部件是一个服务器，包括一套网络漏洞测试程序，可以检测远程系统和应用程序中的安全问题。但是它的最常用用途是检测目标网络或","/article/94",14690,"2019-01-14 17:43:42","2019-01-14 18:16:36",{"id":200,"category_id":44,"title":201,"keywords":202,"description":203,"image_url":6,"url":204,"hits":205,"is_recommend":73,"is_top":73,"create_time":206,"update_time":207},99,"Centos7 利用iptables防止nmap工具防端口扫描","iptables","一、Nmap介绍       Nmap（NetworkMapper）是一款开放源代码的网络探测和安全审核工具。它用于快速扫描一个网络和一台主机开放的端口，还能使用TCP/IP协议栈特征探测远程主机的操作系统类型。nmap支持很多扫描技术，例如：UDP、TCPconnect()、TCPSYN(半开扫描)、ftp代理(bounce攻击)、反向标志、ICMP、FIN、ACK扫描、圣诞树(XmasTree)、SYN扫描和null扫描。Nmap最初是用于Unix系统","/article/99",14415,"2019-07-09 21:27:00","2019-07-09 21:57:53",{"id":209,"category_id":46,"title":210,"keywords":211,"description":212,"image_url":6,"url":213,"hits":214,"is_recommend":73,"is_top":73,"create_time":215,"update_time":216},61,"Docker 推荐的启动方式","推荐,启动,方式","# cat DockerfileFROM openjdk:8-alpineWORKDIR /ADD ./target/*.jar app.jarEXPOSE 9999COPY docker-entrypoint.sh /RUN chmod +x /docker-entrypoint.shENTRYPOINT [“/docker-entrypoint.sh”]CMD [“java”,”-server”,”-Duser.timezone=GMT+08″,”-jar”,”/app.jar”]# cat","/article/61",14347,"2018-10-22 13:55:47","2018-10-22 13:56:10",{"id":218,"category_id":44,"title":219,"keywords":220,"description":221,"image_url":6,"url":222,"hits":223,"is_recommend":73,"is_top":73,"create_time":224,"update_time":225},107,"Acme.sh 给 SSL 证书自动续期失败的解决方法","HTTP/1.1 200 OK,Server: Bayou Tech Web Srv 1.0,Content-Encoding: none,Content-Length: 5,Content-Type","一、Acme.sh 自动续期失败的症状问题描述如下，续期的时候，提示如下错误：root@dc:~# \"/data/acme.sh\"/acme.sh --cron --home \"/data/acme.sh\" > /dev/null[Sun Nov 10 23:52:17 CST 2020] Error, can not get domain token entry example.com[Sun Nov 10 23:52:17 CST 2020] Please check log file","/article/107",12618,"2021-09-03 10:44:18","2021-09-03 10:46:23",{"id":227,"category_id":44,"title":228,"keywords":229,"description":230,"image_url":6,"url":231,"hits":232,"is_recommend":53,"is_top":73,"create_time":233,"update_time":234},93,"ELK+Filebeat+Kafka+ZooKeeper 构建海量日志分析平台","Filebeat,Kafka","一、说明1.Filebeat是一个日志文件托运工具，在你的服务器上安装客户端后，filebeat会监控日志目录或者指定的日志文件，追踪读取这些文件（追踪文件的变化，不停的读）2.Kafka是一种高吞吐量的分布式发布订阅消息系统，它可以处理消费者规模的网站中的所有动作流数据3.Logstash是一根具备实时数据传输能力的管道，负责将数据信息从管道的输入端传输到管道的输出端；与此同时这根管道还可以让你根据自己的需求在中间加上滤网，Logstash提供里很多功能强大的滤网以满足你的各种应用场景4.El","/article/93",12190,"2018-12-24 16:40:08","2018-12-26 11:53:38",{"id":236,"category_id":46,"title":237,"keywords":238,"description":239,"image_url":6,"url":240,"hits":241,"is_recommend":53,"is_top":73,"create_time":242,"update_time":243},81,"kubernetes 1.12.1 高可用安装之部署Dashboard","安装Dashboard","创建Dashboard需要CoreDNS部署成功之后再安装Dashboard。[root@master01 ~]# wget https://zhl123.com/download/k8s/Dashboard.tgz[root@master01 ~]# tar xf Dashboard.tgz[root@master01 ~]# kubectl create -f Dashboard/[root@master01 Dashboard]# kubectl get svc -n kube-syste","/article/81",12037,"2018-10-26 09:54:41","2018-10-26 16:59:19",{"id":245,"category_id":44,"title":246,"keywords":6,"description":247,"image_url":6,"url":248,"hits":249,"is_recommend":73,"is_top":73,"create_time":250,"update_time":251},100,"用 Nginx 给 Cookie 增加 Secure 和 HttpOnly","在 nginx 的 location 中配置12# 只支持 proxy 模式下设置，SameSite 不需要可删除，如果想更安全可以把 SameSite 设置为 Strictproxy_cookie_path / \"/; httponly; secure; SameSite=Lax\";示例1234567891011121314151617181920212223242526server {    listen 443 ssl http2;    server_name www.zhl123.cn","/article/100",11848,"2019-11-26 16:10:57","2019-11-26 16:23:45",{"id":253,"category_id":44,"title":254,"keywords":255,"description":256,"image_url":6,"url":257,"hits":258,"is_recommend":73,"is_top":73,"create_time":259,"update_time":260},41,"Tomcat 安全配置与性能优化","tomcat，性能优化","1. JVM 1.1. 使用 Server JRE 替代JDK。 服务器上不要安装JDK，请使用 Server JRE. 服务器上根本不需要编译器，代码应该在Release服务器上完成编译打包工作。 理由：一旦服务器被控制，可以防止在其服务器上编译其他恶意代码并植入到你的程序中。 1.2. JAVA_OPTS export JAVA_OPTS=\"-server -Xms512m -Xmx4096m  -XX:PermSize=64M -","/article/41",11623,"2016-09-01 17:06:36","2018-10-18 17:06:58",[262,270,278,286,292,293,299,307,316,325],{"id":263,"category_id":40,"title":264,"keywords":6,"description":265,"image_url":6,"url":266,"hits":267,"is_recommend":73,"is_top":73,"create_time":268,"update_time":269},123,"Agent Skill 精选集：最值得收藏的 Agent Skills Top 10","如果你正在用 Claude Code 或 Codex，一定对 Agent Skills 不陌生。通过安装 Agent Skills，你可以让这些 AI 助手变得更强——不用每次都解释你的需求，它们直接就知道该怎么做。最近有人在 GitHub 上做了一个采样调查，统计了哪些 Skills 的质量最佳和最受欢迎。我整理了这份 Top 10 榜单，加上使用场景和适合人群，帮你快速找到最有用的那几个。Top 10 最受欢迎的 Agent Skills1. Skil","/article/123",270,"2026-01-19 18:49:12","2026-01-19 18:52:01",{"id":271,"category_id":44,"title":272,"keywords":6,"description":273,"image_url":6,"url":274,"hits":275,"is_recommend":73,"is_top":73,"create_time":276,"update_time":277},122,"Nginx性能调优18条黄金法则：支撑10万并发的配置模板","一、概述1.1 背景介绍说实话，Nginx调优这事儿我踩过无数坑。记得2019年双11，我们电商平台流量暴涨，Nginx直接扛不住了，QPS从平时的2万飙升到8万，响应时间从50ms飙到了2秒，最后还是靠临时加机器扛过去的。那次事故之后，我花了大半年时间专门研究Nginx的性能极限，总结出了这20条黄金法则。Nginx作为目前最流行的Web服务器和反向代理，官方数据显示单机可以轻松处理10万+的并发连接。但实际生产环境中，很多同学拿到默认配置就直接上了，结果发现连1万并发都扛不住。问题不在Ngi","/article/122",337,"2026-01-12 11:11:50","2026-01-12 11:12:28",{"id":279,"category_id":46,"title":280,"keywords":6,"description":281,"image_url":6,"url":282,"hits":283,"is_recommend":73,"is_top":73,"create_time":284,"update_time":285},121,"Docker 镜像优化与安全扫描：将镜像体积压缩 70%","1. 适用场景 & 前置条件项目要求适用场景容器化应用镜像体积过大（> 500MB），构建时间长（> 10分钟），存在安全漏洞（CVE高危）OSRHEL/CentOS 7.9+ 或 Ubuntu 20.04+内核Linux Kernel 3.10+软件版本Docker 20.10+ 或 Podman 3.0+，Trivy 0.40+（安全扫描工具）资源规格2C4G（最小）/ 4C8G（推荐），磁盘 50GB+（存储镜像与缓存）网络可访问 Docker Hub/阿里云镜像仓库（","/article/121",309,"2026-01-06 11:54:35","2026-01-06 12:01:59",{"id":287,"category_id":44,"title":160,"keywords":6,"description":288,"image_url":6,"url":161,"hits":289,"is_recommend":73,"is_top":73,"create_time":290,"update_time":291},120,"五、故障排查和监控5.1 故障排查◆ 5.1.1 日志查看# 查看 Prometheus 日志中的规则评估错误journalctl -u prometheus | grep -i \"rule\" | tail -50# 查看规则评估耗时curl -s http://localhost:9090/api/v1/rules | jq '.data.groups[].rules[] | select(.health != \"ok\")'# Kubernet",282,"2026-01-06 11:53:50","2026-01-06 11:54:33",{"id":145,"category_id":44,"title":146,"keywords":6,"description":147,"image_url":6,"url":150,"hits":151,"is_recommend":73,"is_top":73,"create_time":152,"update_time":153},{"id":294,"category_id":44,"title":163,"keywords":6,"description":295,"image_url":6,"url":164,"hits":296,"is_recommend":73,"is_top":73,"create_time":297,"update_time":298},118,"四、最佳实践和注意事项4.1 最佳实践4.1.1 性能优化优化点一：减少 Git 轮询频率# argocd-cm ConfigMapapiVersion: v1kind: ConfigMapmetadata:  name: argocd-cm  namespace: argocddata:  timeout.reconciliation: 300s  # 默认 180",305,"2026-01-06 11:49:00","2026-01-06 11:49:34",{"id":300,"category_id":44,"title":301,"keywords":6,"description":302,"image_url":6,"url":303,"hits":304,"is_recommend":73,"is_top":73,"create_time":305,"update_time":306},117,"GitOps 落地实践：ArgoCD + Kustomize 实现声明式基础设施管理(一)","一、概述1.1 背景介绍GitOps 作为云原生时代的运维范式，将 Git 作为基础设施和应用配置的单一事实来源，通过声明式配置和自动化同步机制，实现了配置管理的版本控制、审计追溯和快速回滚。ArgoCD 作为 CNCF 毕业项目，提供了完整的 GitOps 工作流，支持多集群管理、RBAC 权限控制、SSO 集成等企业级特性。结合 Kustomize 的配置管理能力，能够优雅地解决多环境配置差异、敏感信息管理、配置复用等问题。在传统的 CI/CD 流程中，往往由 CI 工具直接执行 kubec","/article/117",310,"2026-01-06 11:45:03","2026-01-06 11:48:56",{"id":308,"category_id":40,"title":309,"keywords":310,"description":311,"image_url":6,"url":312,"hits":313,"is_recommend":73,"is_top":73,"create_time":314,"update_time":315},116,"运维部门年度2025工作总结与2026工作规划应该如何写？","运维部门年度2025工作总结,2026工作规划","2025年，运维部在公司“数字化转型深化”战略引领下，以“稳定为基、效率为纲、安全为盾、创新为翼”为核心导向，全面支撑核心业务系统运行、推动技术架构迭代、强化团队能力建设。 全年实现核心业务系统可用性99.985%，较2024年提升0.02个百分点；故障平均恢复时间（MTTR）从42分钟压缩至29分钟，下降31%；云资源成本同比降低16.8%，自动化运维覆盖率从65%提升至83%，未发生重大生产安全事故，圆满完成年度目标。现将全年工作及2026年规划汇报如下：2025年核心工作成果（数","/article/116",297,"2026-01-06 11:20:58","2026-01-06 11:44:04",{"id":317,"category_id":40,"title":318,"keywords":319,"description":320,"image_url":6,"url":321,"hits":322,"is_recommend":73,"is_top":73,"create_time":323,"update_time":324},115,"Kubernetes 100个常用命令","100 个 Kubectl 命令","这篇文章是关于使用 Kubectl 进行 Kubernetes 诊断的指南。列出了 100 个 Kubectl 命令，这些命令对于诊断 Kubernetes 集群中的问题非常有用。这些问题包括但不限于：• 集群信息• Pod 诊断• 服务诊断• 部署诊断• 网络诊断• 持久卷和持久卷声明诊断• 资源使用情况• 安全和授权• 节点故障排除• 其他诊断命令：文章还提到了许多其他命令，如资源扩展和自动扩","/article/115",3642,"2023-11-02 14:09:30","2023-11-02 14:10:05",{"id":182,"category_id":44,"title":183,"keywords":184,"description":185,"image_url":6,"url":186,"hits":187,"is_recommend":73,"is_top":73,"create_time":188,"update_time":189},1784716028275]