{"host": "db2184.codfw.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3066, "only_in_self": ["Class[Passwords::Nagios::Mysql]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]", "Mariadb::Monitor_replication[backup1-codfw]", "Monitoring::Exported_nagios_service[db2184 mariadb_replica_io_state_backup1-codfw]", "Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_lag_backup1-codfw]", "Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_state_backup1-codfw]", "Monitoring::Service[mariadb_replica_io_state_backup1-codfw]", "Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]", "Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw]", "Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c]", "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8]", "Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f]", "Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e]", "Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d]", "Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw]", "Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]"], "only_in_other": ["Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0]", "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1]", "Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e]"], "resource_diffs": [{"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\"\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]\n\n-    refreshonly => True\n-    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Monitoring::Service[mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_replica_io_state_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_replica_io_state_backup1-codfw]\n\n-    notes_url      => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    critical       => False\n-    migration_task => T315866\n-    contact_group  => admins\n-    retries        => 3\n-    config_dir     => /etc/nagios\n-    check_command  => nrpe_check!check_mariadb_replica_io_state_backup1-codfw!10\n-    description    => MariaDB Replica IO: backup1-codfw\n-    retry_interval => 1\n-    passive        => False\n-    host           => db2184\n-    freshness      => 36000\n-    check_interval => 1\n-    ensure         => present\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg\n@@ -1,2 +0,0 @@\n-# File generated by puppet. DO NOT edit by hand\n-command[check_mariadb_replica_io_state_backup1-codfw]=/usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_io_state", "parameters": "--- File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg].orig\n+++ File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]\n\n-    tag     => nrpe::check\n-    notify  => Service[nagios-nrpe-server]\n-    group   => root\n-    mode    => 0444\n-    require => Package[nagios-nrpe-server]\n-    ensure  => present\n-    owner   => root\n"}, {"resource": "Nrpe::Check[check_mariadb_read_only_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_read_only_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_read_only_backup1-codfw]\n\n@@\n-    command => db-check-health --port=3306 --icinga --check_read_only=true --process\n+    command => db-check-health --port=3306 --icinga --check_read_only=false --process\n"}, {"resource": "Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw].orig\n+++ Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]\n\n-    accuracy           => 15sec\n-    splay              => 60\n-    unit_name          => nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n-    timer_intervals    => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}, {'interval': '1s', 'start': 'OnActiveSec'}]\n-    fixed_random_delay => True\n-    ensure             => absent\n"}, {"resource": "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]\n\n-    accuracy           => 15sec\n-    splay              => 60\n-    unit_name          => nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n-    timer_intervals    => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}, {'interval': '1s', 'start': 'OnActiveSec'}]\n-    fixed_random_delay => True\n-    ensure             => absent\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mysqld.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mysqld.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mysqld.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mysqld\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\"\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mysqld]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mysqld].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mysqld]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\"\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page\n"}, {"resource": "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]", "content": "--- /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf.orig\n+++ /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf\n@@ -1,10 +0,0 @@\n-# SPDX-License-Identifier: Apache-2.0\n-if $programname contains \"nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw\" then {\n-    if ($msg contains \"\\\"ecs.version\\\": \\\"1.7.0\\\"\") then {\n-        # Send logs to kafka\n-        set $.log_outputs = \"kafka ecs_170 local\";\n-    } else {\n-        # Filter out non-relevant nrpe2nodexp messages\n-        stop\n-    }\n-}", "parameters": "--- File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf].orig\n+++ File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]\n\n-    notify => Service[rsyslog]\n-    group  => root\n-    mode   => 0444\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw]\n\n-    notes_url                   => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    critical                    => False\n-    migration_task              => T315866\n-    enable_icinga_check         => True\n-    contact_group               => admins\n-    nrpe2nodexp_parse_perf_data => False\n-    timeout                     => 10\n-    retries                     => 3\n-    description                 => MariaDB Replica SQL: backup1-codfw\n-    nrpe_command                => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_state\n-    retry_interval              => 1\n-    alertmanager_team           => observability\n-    check_interval              => 1\n-    ensure                      => present\n-    enable_nrpe2nodexp          => False\n"}, {"resource": "Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw]\n\n-    notes_url                   => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    critical                    => False\n-    migration_task              => T315866\n-    enable_icinga_check         => True\n-    contact_group               => admins\n-    nrpe2nodexp_parse_perf_data => False\n-    timeout                     => 10\n-    retries                     => 3\n-    description                 => MariaDB Replica IO: backup1-codfw\n-    nrpe_command                => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_io_state\n-    retry_interval              => 1\n-    alertmanager_team           => observability\n-    check_interval              => 1\n-    ensure                      => present\n-    enable_nrpe2nodexp          => False\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mysqld.service]"}, {"resource": "Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "parameters": "--- Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer].orig\n+++ Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]\n\n-    provider => systemd\n-    enable   => False\n-    ensure   => stopped\n-    before   => ['Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]']\n"}, {"resource": "Monitoring::Exported_nagios_service[db2184 mariadb_read_only_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2184 mariadb_read_only_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2184 mariadb_read_only_backup1-codfw]\n\n@@\n-    contact_groups        => admins\n+    contact_groups        => admins,sms,admins\n@@\n-    notification_interval => 0\n+    notification_interval => 240\n@@\n-    service_description   => MariaDB read only backup1-codfw\n+    service_description   => MariaDB read only backup1-codfw #page\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg\n@@ -1,2 +1,2 @@\n # File generated by puppet. DO NOT edit by hand\n-command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=true --process\n+command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=false --process"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1].orig\n+++ Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1]\n\n+    for                => 3m\n+    group              => nrpechecks\n+    summary            => NRPE CHECK: MariaDB read only backup1-codfw #page\n+    instance           => ops\n+    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting#Master_comes_back_in_read_only\n+    team               => observability\n+    description        => NRPE CHECK: MariaDB read only backup1-codfw\n+    def_label_whitelst => ['team', 'severity']\n+    site               => codfw\n+    alert_name         => nrpe_MariaDB_read_only_backup1_codfw\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"c754855176474e5a67d2bb5c504a51c1\",check_name=\"check_mariadb_read_only_backup1-codfw\", status=\"CRITICAL\", severity=\"page\"} > 0) * on (instance) group_left (team) role_owner\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_read_only_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    dashboard          => TODO\n+    ensure             => absent\n+    severity           => info\n"}, {"resource": "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw].orig\n+++ Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]\n\n-    accuracy           => 15sec\n-    splay              => 60\n-    unit_name          => nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n-    timer_intervals    => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}, {'interval': '1s', 'start': 'OnActiveSec'}]\n-    fixed_random_delay => True\n-    ensure             => absent\n"}, {"resource": "Systemd::Service[pt-heartbeat-wikimedia]", "parameters": "--- Systemd::Service[pt-heartbeat-wikimedia].orig\n+++ Systemd::Service[pt-heartbeat-wikimedia]\n\n@@\n-    service_params => {'ensure': False}\n+    service_params => {'ensure': True}\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer\n@@ -1,14 +0,0 @@\n-[Unit]\n-Description=Periodic execution of nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n-\n-[Timer]\n-Unit=nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n-# Accuracy sets the maximum time interval around the execution time we want to allow\n-AccuracySec=15sec\n-OnUnitInactiveSec=1min\n-OnActiveSec=1s\n-RandomizedDelaySec=60\n-FixedRandomDelay=true\n-\n-[Install]\n-WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]\n\n-    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]\n-    group  => root\n-    mode   => 0444\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]\n\n-    refreshonly => True\n-    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw].orig\n+++ Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]\n\n-    unit_type                => timer\n-    monitoring_enabled       => False\n-    monitoring_critical      => False\n-    migration_task           => T407130\n-    monitoring_contact_group => admins\n-    restart                  => False\n-    service_params           => {}\n-    require                  => Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]\n-    ensure                   => absent\n-    override                 => False\n"}, {"resource": "Class[Profile::Mariadb::Mysql_role]", "parameters": "--- Class[Profile::Mariadb::Mysql_role].orig\n+++ Class[Profile::Mariadb::Mysql_role]\n\n@@\n-    role => slave\n+    role => master\n"}, {"resource": "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]", "content": "--- /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf.orig\n+++ /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf\n@@ -1,10 +0,0 @@\n-# SPDX-License-Identifier: Apache-2.0\n-if $programname contains \"nrpe2nodexp-mariadb_replica_io_state_backup1-codfw\" then {\n-    if ($msg contains \"\\\"ecs.version\\\": \\\"1.7.0\\\"\") then {\n-        # Send logs to kafka\n-        set $.log_outputs = \"kafka ecs_170 local\";\n-    } else {\n-        # Filter out non-relevant nrpe2nodexp messages\n-        stop\n-    }\n-}", "parameters": "--- File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf].orig\n+++ File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]\n\n-    notify => Service[rsyslog]\n-    group  => root\n-    mode   => 0444\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw].orig\n+++ Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]\n\n-    priority => 25\n-    mode     => 0444\n-    ensure   => absent\n"}, {"resource": "Class[Mariadb::Config]", "parameters": "--- Class[Mariadb::Config].orig\n+++ Class[Mariadb::Config]\n\n@@\n-    read_only => 1\n+    read_only => 0\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]\n\n-    refreshonly => True\n-    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw].orig\n+++ Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]\n\n-    priority => 25\n-    mode     => 0444\n-    ensure   => absent\n"}, {"resource": "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]\n\n-    priority => 25\n-    mode     => 0444\n-    ensure   => absent\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_read_only_backup1-codfw.service]"}, {"resource": "Nrpe::Monitor_service[mariadb_disk_space]", "parameters": "--- Nrpe::Monitor_service[mariadb_disk_space].orig\n+++ Nrpe::Monitor_service[mariadb_disk_space]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "Monitoring::Exported_nagios_service[db2184 mariadb_disk_space]", "parameters": "--- Monitoring::Exported_nagios_service[db2184 mariadb_disk_space].orig\n+++ Monitoring::Exported_nagios_service[db2184 mariadb_disk_space]\n\n@@\n-    contact_groups        => admins\n+    contact_groups        => admins,sms,admins\n@@\n-    notification_interval => 0\n+    notification_interval => 240\n@@\n-    service_description   => MariaDB disk space\n+    service_description   => MariaDB disk space #page\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8].orig\n+++ Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8]\n\n-    for                => 3m\n-    group              => nrpechecks\n-    summary            => NRPE CHECK: MariaDB read only backup1-codfw\n-    instance           => ops\n-    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting#Master_comes_back_in_read_only\n-    team               => observability\n-    description        => NRPE CHECK: MariaDB read only backup1-codfw\n-    def_label_whitelst => ['team', 'severity']\n-    site               => codfw\n-    alert_name         => nrpe_MariaDB_read_only_backup1_codfw\n-    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"8aa8e94c48d271ddfcbf96b11f04e0e8\",check_name=\"check_mariadb_read_only_backup1-codfw\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n-    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_read_only_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n-    dashboard          => TODO\n-    ensure             => absent\n-    severity           => info\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]\n\n-    restart           => False\n-    override          => False\n-    require           => ['Class[Systemd]']\n-    override_filename => puppet-override.conf\n-    ensure            => absent\n-    unit              => nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n"}, {"resource": "Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "parameters": "--- Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer].orig\n+++ Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]\n\n-    provider => systemd\n-    enable   => False\n-    ensure   => stopped\n-    before   => ['Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]']\n"}, {"resource": "Class[Mariadb::Monitor_process]", "parameters": "--- Class[Mariadb::Monitor_process].orig\n+++ Class[Mariadb::Monitor_process]\n\n@@\n-    is_critical => False\n+    is_critical => True\n"}, {"resource": "Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw]\n\n-    notes_url                   => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    critical                    => False\n-    migration_task              => T315866\n-    enable_icinga_check         => True\n-    contact_group               => admins\n-    nrpe2nodexp_parse_perf_data => False\n-    timeout                     => 10\n-    retries                     => 10\n-    description                 => MariaDB Replica Lag: backup1-codfw\n-    nrpe_command                => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_lag                            --shard=backup1-codfw --datacenter=codfw                            --sql-lag-warn=60                            --sql-lag-crit=300\n-    retry_interval              => 1\n-    alertmanager_team           => observability\n-    check_interval              => 1\n-    ensure                      => present\n-    enable_nrpe2nodexp          => False\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_disk_space\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\"\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page"}, {"resource": "Monitoring::Service[mysqld]", "parameters": "--- Monitoring::Service[mysqld].orig\n+++ Monitoring::Service[mysqld]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer\n@@ -1,14 +0,0 @@\n-[Unit]\n-Description=Periodic execution of nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n-\n-[Timer]\n-Unit=nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n-# Accuracy sets the maximum time interval around the execution time we want to allow\n-AccuracySec=15sec\n-OnUnitInactiveSec=1min\n-OnActiveSec=1s\n-RandomizedDelaySec=60\n-FixedRandomDelay=true\n-\n-[Install]\n-WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]\n\n-    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]\n-    group  => root\n-    mode   => 0444\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "Monitoring::Exported_nagios_service[db2184 mysqld]", "parameters": "--- Monitoring::Exported_nagios_service[db2184 mysqld].orig\n+++ Monitoring::Exported_nagios_service[db2184 mysqld]\n\n@@\n-    contact_groups        => admins\n+    contact_groups        => admins,sms,admins\n@@\n-    notification_interval => 0\n+    notification_interval => 240\n@@\n-    service_description   => mysqld processes\n+    service_description   => mysqld processes #page\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c].orig\n+++ Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c]\n\n-    for                => 3m\n-    group              => nrpechecks\n-    summary            => NRPE CHECK: MariaDB disk space\n-    instance           => ops\n-    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting\n-    team               => observability\n-    description        => NRPE CHECK: MariaDB disk space\n-    def_label_whitelst => ['team', 'severity']\n-    site               => codfw\n-    alert_name         => nrpe_MariaDB_disk_space\n-    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"d2aae7bfec5549be1a8b9413215bd86c\",check_name=\"check_mariadb_disk_space\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n-    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_disk_space))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n-    dashboard          => TODO\n-    ensure             => absent\n-    severity           => info\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg\n@@ -1,2 +0,0 @@\n-# File generated by puppet. DO NOT edit by hand\n-command[check_mariadb_replica_sql_state_backup1-codfw]=/usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_state", "parameters": "--- File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg].orig\n+++ File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]\n\n-    tag     => nrpe::check\n-    notify  => Service[nagios-nrpe-server]\n-    group   => root\n-    mode    => 0444\n-    require => Package[nagios-nrpe-server]\n-    ensure  => present\n-    owner   => root\n"}, {"resource": "Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw]\n\n-    command => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_io_state\n-    ensure  => present\n-    before  => Monitoring::Service[mariadb_replica_io_state_backup1-codfw]\n"}, {"resource": "Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw].orig\n+++ Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw]\n\n-    privileges => []\n-    user       => nagios\n-    tag        => nrpe::check\n-    require    => ['Class[Sudo]']\n-    ensure     => absent\n"}, {"resource": "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]", "parameters": "--- File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom].orig\n+++ File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]\n\n-    group  => root\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\"\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page\n"}, {"resource": "Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "parameters": "--- Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer].orig\n+++ Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]\n\n-    provider => systemd\n-    enable   => False\n-    ensure   => stopped\n-    before   => ['Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]']\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0].orig\n+++ Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0]\n\n+    for                => 3m\n+    group              => nrpechecks\n+    summary            => NRPE CHECK: MariaDB disk space #page\n+    instance           => ops\n+    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting\n+    team               => observability\n+    description        => NRPE CHECK: MariaDB disk space\n+    def_label_whitelst => ['team', 'severity']\n+    site               => codfw\n+    alert_name         => nrpe_MariaDB_disk_space\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"40ee06f86a277811ae30c96774f3ddc0\",check_name=\"check_mariadb_disk_space\", status=\"CRITICAL\", severity=\"page\"} > 0) * on (instance) group_left (team) role_owner\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_disk_space))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    dashboard          => TODO\n+    ensure             => absent\n+    severity           => info\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n@@ -1,11 +0,0 @@\n-[Unit]\n-Description=execution of nrpe2nodexp for the check_mariadb_replica_sql_lag_backup1-codfw command.\n-Documentation=https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n-\n-[Service]\n-Type=oneshot\n-User=nagios\n-\n-Group=prometheus-node-exporter\n-SyslogIdentifier=nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"8688c7f4c31f8bd2553cec605fe5ef5e\" --timeout 10 --check-command \"check_mariadb_replica_sql_lag_backup1-codfw\"", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]\n\n-    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]\n-    group  => root\n-    mode   => 0444\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "Monitoring::Service[mariadb_disk_space]", "parameters": "--- Monitoring::Service[mariadb_disk_space].orig\n+++ Monitoring::Service[mariadb_disk_space]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "Monitoring::Service[mariadb_read_only_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_read_only_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_read_only_backup1-codfw]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer\n@@ -1,14 +0,0 @@\n-[Unit]\n-Description=Periodic execution of nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n-\n-[Timer]\n-Unit=nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n-# Accuracy sets the maximum time interval around the execution time we want to allow\n-AccuracySec=15sec\n-OnUnitInactiveSec=1min\n-OnActiveSec=1s\n-RandomizedDelaySec=60\n-FixedRandomDelay=true\n-\n-[Install]\n-WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]\n\n-    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]\n-    group  => root\n-    mode   => 0444\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]", "parameters": "--- File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw].orig\n+++ File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]\n\n-    group   => root\n-    require => Package[nagios-nrpe-server]\n-    ensure  => absent\n-    owner   => root\n"}, {"resource": "Nrpe::Monitor_service[mysqld]", "parameters": "--- Nrpe::Monitor_service[mysqld].orig\n+++ Nrpe::Monitor_service[mysqld]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "Mariadb::Monitor_replication[backup1-codfw]", "parameters": "--- Mariadb::Monitor_replication[backup1-codfw].orig\n+++ Mariadb::Monitor_replication[backup1-codfw]\n\n-    source_dc     => codfw\n-    lag_warn      => 60\n-    contact_group => admins\n-    lag_crit      => 300\n-    socket        => /run/mysqld/mysqld.sock\n-    multisource   => False\n-    is_critical   => False\n-    warn_stopped  => True\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d].orig\n+++ Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d]\n\n-    for                => 3m\n-    group              => nrpechecks\n-    summary            => NRPE CHECK: MariaDB Replica SQL: backup1-codfw\n-    instance           => ops\n-    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    team               => observability\n-    description        => NRPE CHECK: MariaDB Replica SQL: backup1-codfw\n-    def_label_whitelst => ['team', 'severity']\n-    site               => codfw\n-    alert_name         => nrpe_MariaDB_Replica_SQL__backup1_codfw\n-    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"872d6618b9a5af3d398dc5cd99b2733d\",check_name=\"check_mariadb_replica_sql_state_backup1-codfw\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n-    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_replica_sql_state_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n-    dashboard          => TODO\n-    ensure             => absent\n-    severity           => info\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]\n\n-    ignore_errors             => True\n-    monitoring_enabled        => False\n-    logfile_basedir           => /var/log\n-    success_exit_status       => []\n-    syslog_identifier         => nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw\n-    logfile_perms             => all\n-    logfile_group             => root\n-    syslog_force_stop         => True\n-    send_mail_only_on_error   => True\n-    logging_enabled           => False\n-    description               => execution of nrpe2nodexp for the check_mariadb_replica_sql_lag_backup1-codfw command.\n-    logfile_name              => syslog.log\n-    send_mail_to              => root@db2184.codfw.wmnet\n-    command                   => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"8688c7f4c31f8bd2553cec605fe5ef5e\" --timeout 10 --check-command \"check_mariadb_replica_sql_lag_backup1-codfw\"\n-    interval                  => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}]\n-    syslog_match_startswith   => True\n-    user                      => nagios\n-    group                     => prometheus-node-exporter\n-    private_tmp               => False\n-    environment               => {}\n-    send_mail                 => False\n-    splay                     => 60\n-    monitoring_contact_groups => admins\n-    fixed_random_delay        => True\n-    monitoring_notes_url      => https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n-    ensure                    => absent\n"}, {"resource": "Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116]", "parameters": "--- Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116].orig\n+++ Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116]\n\n-    for                => 3m\n-    group              => nrpechecks\n-    summary            => NRPE CHECK: mysqld processes\n-    instance           => ops\n-    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting\n-    team               => observability\n-    description        => NRPE CHECK: mysqld processes\n-    def_label_whitelst => ['team', 'severity']\n-    site               => codfw\n-    alert_name         => nrpe_mysqld_processes\n-    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"ca81db2c9e98d408f9f65c79817da116\",check_name=\"check_mysqld\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n-    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mysqld))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n-    dashboard          => TODO\n-    ensure             => absent\n-    severity           => info\n"}, {"resource": "Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw]\n\n-    privileges => []\n-    user       => nagios\n-    tag        => nrpe::check\n-    require    => ['Class[Sudo]']\n-    ensure     => absent\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]\n\n-    refreshonly => True\n-    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]\n\n-    override_filename => puppet-override.conf\n-    override          => False\n-    require           => ['Class[Systemd]']\n-    restart           => False\n-    ensure            => absent\n-    unit              => nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer\n"}, {"resource": "Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e]", "parameters": "--- Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e].orig\n+++ Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e]\n\n+    for                => 3m\n+    group              => nrpechecks\n+    summary            => NRPE CHECK: mysqld processes #page\n+    instance           => ops\n+    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting\n+    team               => observability\n+    description        => NRPE CHECK: mysqld processes\n+    def_label_whitelst => ['team', 'severity']\n+    site               => codfw\n+    alert_name         => nrpe_mysqld_processes\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"fc384e29e203c0920c26ca72c745692e\",check_name=\"check_mysqld\", status=\"CRITICAL\", severity=\"page\"} > 0) * on (instance) group_left (team) role_owner\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mysqld))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    dashboard          => TODO\n+    ensure             => absent\n+    severity           => info\n"}, {"resource": "Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_state_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_state_backup1-codfw]\n\n-    servicegroups          => mysql_codfw\n-    notes_url              => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    host_name              => db2184\n-    check_command          => nrpe_check!check_mariadb_replica_sql_state_backup1-codfw!10\n-    contact_groups         => admins\n-    check_period           => 24x7\n-    check_freshness        => 0\n-    notification_options   => c,r,f\n-    is_volatile            => 0\n-    notification_period    => 24x7\n-    retry_interval         => 1\n-    passive_checks_enabled => 1\n-    active_checks_enabled  => 1\n-    notification_interval  => 0\n-    service_description    => MariaDB Replica SQL: backup1-codfw\n-    check_interval         => 1\n-    notifications_enabled  => 1\n-    max_check_attempts     => 3\n-    ensure                 => present\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n@@ -1,11 +0,0 @@\n-[Unit]\n-Description=execution of nrpe2nodexp for the check_mariadb_replica_io_state_backup1-codfw command.\n-Documentation=https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n-\n-[Service]\n-Type=oneshot\n-User=nagios\n-\n-Group=prometheus-node-exporter\n-SyslogIdentifier=nrpe2nodexp-mariadb_replica_io_state_backup1-codfw\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca85d57665ee50d48a78933a6514ca3f\" --timeout 10 --check-command \"check_mariadb_replica_io_state_backup1-codfw\"", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]\n\n-    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]\n-    group  => root\n-    mode   => 0444\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_read_only_backup1-codfw\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\"\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page"}, {"resource": "Class[Mariadb::Heartbeat]", "parameters": "--- Class[Mariadb::Heartbeat].orig\n+++ Class[Mariadb::Heartbeat]\n\n@@\n-    enabled => False\n+    enabled => True\n"}, {"resource": "File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]", "parameters": "--- File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom].orig\n+++ File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]\n\n-    group  => root\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]\n\n-    ignore_errors             => True\n-    monitoring_enabled        => False\n-    logfile_basedir           => /var/log\n-    success_exit_status       => []\n-    syslog_identifier         => nrpe2nodexp-mariadb_replica_io_state_backup1-codfw\n-    logfile_perms             => all\n-    logfile_group             => root\n-    syslog_force_stop         => True\n-    send_mail_only_on_error   => True\n-    logging_enabled           => False\n-    description               => execution of nrpe2nodexp for the check_mariadb_replica_io_state_backup1-codfw command.\n-    logfile_name              => syslog.log\n-    send_mail_to              => root@db2184.codfw.wmnet\n-    command                   => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca85d57665ee50d48a78933a6514ca3f\" --timeout 10 --check-command \"check_mariadb_replica_io_state_backup1-codfw\"\n-    interval                  => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}]\n-    syslog_match_startswith   => True\n-    user                      => nagios\n-    group                     => prometheus-node-exporter\n-    private_tmp               => False\n-    environment               => {}\n-    send_mail                 => False\n-    splay                     => 60\n-    monitoring_contact_groups => admins\n-    fixed_random_delay        => True\n-    monitoring_notes_url      => https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n-    ensure                    => absent\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]\n\n-    restart           => False\n-    override          => False\n-    require           => ['Class[Systemd]']\n-    override_filename => puppet-override.conf\n-    ensure            => absent\n-    unit              => nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg\n@@ -1,2 +0,0 @@\n-# File generated by puppet. DO NOT edit by hand\n-command[check_mariadb_replica_sql_lag_backup1-codfw]=/usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_lag                            --shard=backup1-codfw --datacenter=codfw                            --sql-lag-warn=60                            --sql-lag-crit=300", "parameters": "--- File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg].orig\n+++ File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]\n\n-    tag     => nrpe::check\n-    notify  => Service[nagios-nrpe-server]\n-    group   => root\n-    mode    => 0444\n-    require => Package[nagios-nrpe-server]\n-    ensure  => present\n-    owner   => root\n"}, {"resource": "Monitoring::Exported_nagios_service[db2184 mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2184 mariadb_replica_io_state_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2184 mariadb_replica_io_state_backup1-codfw]\n\n-    servicegroups          => mysql_codfw\n-    notes_url              => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    host_name              => db2184\n-    check_command          => nrpe_check!check_mariadb_replica_io_state_backup1-codfw!10\n-    contact_groups         => admins\n-    check_period           => 24x7\n-    check_freshness        => 0\n-    notification_options   => c,r,f\n-    is_volatile            => 0\n-    notification_period    => 24x7\n-    retry_interval         => 1\n-    passive_checks_enabled => 1\n-    active_checks_enabled  => 1\n-    notification_interval  => 0\n-    service_description    => MariaDB Replica IO: backup1-codfw\n-    check_interval         => 1\n-    notifications_enabled  => 1\n-    max_check_attempts     => 3\n-    ensure                 => present\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]\n\n-    override_filename => puppet-override.conf\n-    override          => False\n-    require           => ['Class[Systemd]']\n-    restart           => False\n-    ensure            => absent\n-    unit              => nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer\n"}, {"resource": "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw].orig\n+++ File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]\n\n-    group   => root\n-    require => Package[nagios-nrpe-server]\n-    ensure  => absent\n-    owner   => root\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]\n\n-    refreshonly => True\n-    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw]\n\n-    command => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_lag                            --shard=backup1-codfw --datacenter=codfw                            --sql-lag-warn=60                            --sql-lag-crit=300\n-    ensure  => present\n-    before  => Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]\n"}, {"resource": "Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]\n\n-    notes_url      => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    critical       => False\n-    migration_task => T315866\n-    contact_group  => admins\n-    retries        => 10\n-    config_dir     => /etc/nagios\n-    check_command  => nrpe_check!check_mariadb_replica_sql_lag_backup1-codfw!10\n-    description    => MariaDB Replica Lag: backup1-codfw\n-    retry_interval => 1\n-    passive        => False\n-    host           => db2184\n-    freshness      => 36000\n-    check_interval => 1\n-    ensure         => present\n"}, {"resource": "File[/etc/my.cnf]", "content": "--- /etc/my.cnf.orig\n+++ /etc/my.cnf\n@@ -16,7 +16,7 @@\n server_id          = 180362264\n report_host        = db2184.codfw.wmnet\n wsrep_cluster_name = codfw # For orchestrator. T266635\n-read_only          = 1\n+read_only          = 0\n \n log_bin\n log_slave_updates"}, {"resource": "Class[Mariadb::Monitor_disk]", "parameters": "--- Class[Mariadb::Monitor_disk].orig\n+++ Class[Mariadb::Monitor_disk]\n\n@@\n-    is_critical => False\n+    is_critical => True\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]\n\n-    restart           => False\n-    override          => False\n-    require           => ['Class[Systemd]']\n-    override_filename => puppet-override.conf\n-    ensure            => absent\n-    unit              => nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n"}, {"resource": "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]", "content": "--- /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf.orig\n+++ /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf\n@@ -1,10 +0,0 @@\n-# SPDX-License-Identifier: Apache-2.0\n-if $programname contains \"nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw\" then {\n-    if ($msg contains \"\\\"ecs.version\\\": \\\"1.7.0\\\"\") then {\n-        # Send logs to kafka\n-        set $.log_outputs = \"kafka ecs_170 local\";\n-    } else {\n-        # Filter out non-relevant nrpe2nodexp messages\n-        stop\n-    }\n-}", "parameters": "--- File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf].orig\n+++ File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]\n\n-    notify => Service[rsyslog]\n-    group  => root\n-    mode   => 0444\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]", "parameters": "--- File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom].orig\n+++ File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]\n\n-    group  => root\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]\n\n-    unit_type                => timer\n-    monitoring_enabled       => False\n-    monitoring_critical      => False\n-    migration_task           => T407130\n-    monitoring_contact_group => admins\n-    restart                  => False\n-    service_params           => {}\n-    require                  => Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]\n-    ensure                   => absent\n-    override                 => False\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_disk_space.service]"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]\n\n-    refreshonly => True\n-    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Nrpe::Monitor_service[mariadb_read_only_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_read_only_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_read_only_backup1-codfw]\n\n@@\n-    critical     => False\n+    critical     => True\n@@\n-    nrpe_command => db-check-health --port=3306 --icinga --check_read_only=true --process\n+    nrpe_command => db-check-health --port=3306 --icinga --check_read_only=false --process\n"}, {"resource": "Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw].orig\n+++ Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw]\n\n-    privileges => []\n-    user       => nagios\n-    tag        => nrpe::check\n-    require    => ['Class[Sudo]']\n-    ensure     => absent\n"}, {"resource": "Service[pt-heartbeat-wikimedia]", "parameters": "--- Service[pt-heartbeat-wikimedia].orig\n+++ Service[pt-heartbeat-wikimedia]\n\n@@\n-    ensure => False\n+    ensure => True\n"}, {"resource": "Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw]\n\n-    command => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_state\n-    ensure  => present\n-    before  => Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]\n"}, {"resource": "Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_replica_sql_state_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]\n\n-    notes_url      => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    critical       => False\n-    migration_task => T315866\n-    contact_group  => admins\n-    retries        => 3\n-    config_dir     => /etc/nagios\n-    check_command  => nrpe_check!check_mariadb_replica_sql_state_backup1-codfw!10\n-    description    => MariaDB Replica SQL: backup1-codfw\n-    retry_interval => 1\n-    passive        => False\n-    host           => db2184\n-    freshness      => 36000\n-    check_interval => 1\n-    ensure         => present\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f].orig\n+++ Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f]\n\n-    for                => 3m\n-    group              => nrpechecks\n-    summary            => NRPE CHECK: MariaDB Replica IO: backup1-codfw\n-    instance           => ops\n-    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    team               => observability\n-    description        => NRPE CHECK: MariaDB Replica IO: backup1-codfw\n-    def_label_whitelst => ['team', 'severity']\n-    site               => codfw\n-    alert_name         => nrpe_MariaDB_Replica_IO__backup1_codfw\n-    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"ca85d57665ee50d48a78933a6514ca3f\",check_name=\"check_mariadb_replica_io_state_backup1-codfw\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n-    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_replica_io_state_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n-    dashboard          => TODO\n-    ensure             => absent\n-    severity           => info\n"}, {"resource": "Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_lag_backup1-codfw]\n\n-    servicegroups          => mysql_codfw\n-    notes_url              => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    host_name              => db2184\n-    check_command          => nrpe_check!check_mariadb_replica_sql_lag_backup1-codfw!10\n-    contact_groups         => admins\n-    check_period           => 24x7\n-    check_freshness        => 0\n-    notification_options   => c,r,f\n-    is_volatile            => 0\n-    notification_period    => 24x7\n-    retry_interval         => 1\n-    passive_checks_enabled => 1\n-    active_checks_enabled  => 1\n-    notification_interval  => 0\n-    service_description    => MariaDB Replica Lag: backup1-codfw\n-    check_interval         => 1\n-    notifications_enabled  => 1\n-    max_check_attempts     => 10\n-    ensure                 => present\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n@@ -1,11 +0,0 @@\n-[Unit]\n-Description=execution of nrpe2nodexp for the check_mariadb_replica_sql_state_backup1-codfw command.\n-Documentation=https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n-\n-[Service]\n-Type=oneshot\n-User=nagios\n-\n-Group=prometheus-node-exporter\n-SyslogIdentifier=nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"872d6618b9a5af3d398dc5cd99b2733d\" --timeout 10 --check-command \"check_mariadb_replica_sql_state_backup1-codfw\"", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]\n\n-    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]\n-    group  => root\n-    mode   => 0444\n-    ensure => absent\n-    owner  => root\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e].orig\n+++ Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e]\n\n-    for                => 10m\n-    group              => nrpechecks\n-    summary            => NRPE CHECK: MariaDB Replica Lag: backup1-codfw\n-    instance           => ops\n-    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n-    team               => observability\n-    description        => NRPE CHECK: MariaDB Replica Lag: backup1-codfw\n-    def_label_whitelst => ['team', 'severity']\n-    site               => codfw\n-    alert_name         => nrpe_MariaDB_Replica_Lag__backup1_codfw\n-    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"8688c7f4c31f8bd2553cec605fe5ef5e\",check_name=\"check_mariadb_replica_sql_lag_backup1-codfw\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n-    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_replica_sql_lag_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n-    dashboard          => TODO\n-    ensure             => absent\n-    severity           => info\n"}, {"resource": "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw].orig\n+++ Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]\n\n-    unit_type                => timer\n-    monitoring_enabled       => False\n-    monitoring_critical      => False\n-    migration_task           => T407130\n-    monitoring_contact_group => admins\n-    restart                  => False\n-    service_params           => {}\n-    require                  => Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]\n-    ensure                   => absent\n-    override                 => False\n"}, {"resource": "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw].orig\n+++ File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]\n\n-    group   => root\n-    require => Package[nagios-nrpe-server]\n-    ensure  => absent\n-    owner   => root\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]\n\n-    ignore_errors             => True\n-    monitoring_enabled        => False\n-    logfile_basedir           => /var/log\n-    success_exit_status       => []\n-    syslog_identifier         => nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw\n-    logfile_perms             => all\n-    logfile_group             => root\n-    syslog_force_stop         => True\n-    send_mail_only_on_error   => True\n-    logging_enabled           => False\n-    description               => execution of nrpe2nodexp for the check_mariadb_replica_sql_state_backup1-codfw command.\n-    logfile_name              => syslog.log\n-    send_mail_to              => root@db2184.codfw.wmnet\n-    command                   => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"872d6618b9a5af3d398dc5cd99b2733d\" --timeout 10 --check-command \"check_mariadb_replica_sql_state_backup1-codfw\"\n-    interval                  => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}]\n-    syslog_match_startswith   => True\n-    user                      => nagios\n-    group                     => prometheus-node-exporter\n-    private_tmp               => False\n-    environment               => {}\n-    send_mail                 => False\n-    splay                     => 60\n-    monitoring_contact_groups => admins\n-    fixed_random_delay        => True\n-    monitoring_notes_url      => https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n-    ensure                    => absent\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]\n\n-    override_filename => puppet-override.conf\n-    override          => False\n-    require           => ['Class[Systemd]']\n-    restart           => False\n-    ensure            => absent\n-    unit              => nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer\n"}, {"resource": "Mariadb::Monitor_readonly[backup1-codfw]", "parameters": "--- Mariadb::Monitor_readonly[backup1-codfw].orig\n+++ Mariadb::Monitor_readonly[backup1-codfw]\n\n@@\n-    is_critical => False\n+    is_critical => True\n@@\n-    read_only   => True\n+    read_only   => False\n"}], "perc_changed": "5.54%"}, "core": {"total": 3066, "only_in_self": ["Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]", "Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]"], "only_in_other": [], "resource_diffs": [{"resource": "File[/lib/systemd/system/nrpe2nodexp-mysqld.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mysqld.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mysqld.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mysqld\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\"\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_read_only_backup1-codfw\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\"\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg\n@@ -1,2 +1,2 @@\n # File generated by puppet. DO NOT edit by hand\n-command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=true --process\n+command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=false --process"}, {"resource": "File[/etc/my.cnf]", "content": "--- /etc/my.cnf.orig\n+++ /etc/my.cnf\n@@ -16,7 +16,7 @@\n server_id          = 180362264\n report_host        = db2184.codfw.wmnet\n wsrep_cluster_name = codfw # For orchestrator. T266635\n-read_only          = 1\n+read_only          = 0\n \n log_bin\n log_slave_updates"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_disk_space\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\"\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page"}, {"resource": "Service[pt-heartbeat-wikimedia]", "parameters": "--- Service[pt-heartbeat-wikimedia].orig\n+++ Service[pt-heartbeat-wikimedia]\n\n@@\n-    ensure => False\n+    ensure => True\n"}], "perc_changed": "1.08%"}, "main": {"total": 3066, "only_in_self": ["Class[Passwords::Nagios::Mysql]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]", "Mariadb::Monitor_replication[backup1-codfw]", "Monitoring::Exported_nagios_service[db2184 mariadb_replica_io_state_backup1-codfw]", "Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_lag_backup1-codfw]", "Monitoring::Exported_nagios_service[db2184 mariadb_replica_sql_state_backup1-codfw]", "Monitoring::Service[mariadb_replica_io_state_backup1-codfw]", "Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]", "Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw]", "Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c]", "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8]", "Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f]", "Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e]", "Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d]", "Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw]", "Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]"], "only_in_other": ["Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0]", "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1]", "Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e]"], "resource_diffs": [{"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\"\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page\n"}, {"resource": "Nrpe::Check[check_mariadb_read_only_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_read_only_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_read_only_backup1-codfw]\n\n@@\n-    command => db-check-health --port=3306 --icinga --check_read_only=true --process\n+    command => db-check-health --port=3306 --icinga --check_read_only=false --process\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mysqld.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mysqld.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mysqld.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mysqld\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\"\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mysqld]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mysqld].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mysqld]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\"\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_read_only_backup1-codfw\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\"\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page"}, {"resource": "Class[Mariadb::Heartbeat]", "parameters": "--- Class[Mariadb::Heartbeat].orig\n+++ Class[Mariadb::Heartbeat]\n\n@@\n-    enabled => False\n+    enabled => True\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mysqld.service]"}, {"resource": "Monitoring::Exported_nagios_service[db2184 mariadb_read_only_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2184 mariadb_read_only_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2184 mariadb_read_only_backup1-codfw]\n\n@@\n-    contact_groups        => admins\n+    contact_groups        => admins,sms,admins\n@@\n-    notification_interval => 0\n+    notification_interval => 240\n@@\n-    service_description   => MariaDB read only backup1-codfw\n+    service_description   => MariaDB read only backup1-codfw #page\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg\n@@ -1,2 +1,2 @@\n # File generated by puppet. DO NOT edit by hand\n-command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=true --process\n+command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=false --process"}, {"resource": "Systemd::Service[pt-heartbeat-wikimedia]", "parameters": "--- Systemd::Service[pt-heartbeat-wikimedia].orig\n+++ Systemd::Service[pt-heartbeat-wikimedia]\n\n@@\n-    service_params => {'ensure': False}\n+    service_params => {'ensure': True}\n"}, {"resource": "Class[Profile::Mariadb::Mysql_role]", "parameters": "--- Class[Profile::Mariadb::Mysql_role].orig\n+++ Class[Profile::Mariadb::Mysql_role]\n\n@@\n-    role => slave\n+    role => master\n"}, {"resource": "Class[Mariadb::Config]", "parameters": "--- Class[Mariadb::Config].orig\n+++ Class[Mariadb::Config]\n\n@@\n-    read_only => 1\n+    read_only => 0\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_read_only_backup1-codfw.service]"}, {"resource": "Nrpe::Monitor_service[mariadb_disk_space]", "parameters": "--- Nrpe::Monitor_service[mariadb_disk_space].orig\n+++ Nrpe::Monitor_service[mariadb_disk_space]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "File[/etc/my.cnf]", "content": "--- /etc/my.cnf.orig\n+++ /etc/my.cnf\n@@ -16,7 +16,7 @@\n server_id          = 180362264\n report_host        = db2184.codfw.wmnet\n wsrep_cluster_name = codfw # For orchestrator. T266635\n-read_only          = 1\n+read_only          = 0\n \n log_bin\n log_slave_updates"}, {"resource": "Class[Mariadb::Monitor_disk]", "parameters": "--- Class[Mariadb::Monitor_disk].orig\n+++ Class[Mariadb::Monitor_disk]\n\n@@\n-    is_critical => False\n+    is_critical => True\n"}, {"resource": "Monitoring::Exported_nagios_service[db2184 mariadb_disk_space]", "parameters": "--- Monitoring::Exported_nagios_service[db2184 mariadb_disk_space].orig\n+++ Monitoring::Exported_nagios_service[db2184 mariadb_disk_space]\n\n@@\n-    contact_groups        => admins\n+    contact_groups        => admins,sms,admins\n@@\n-    notification_interval => 0\n+    notification_interval => 240\n@@\n-    service_description   => MariaDB disk space\n+    service_description   => MariaDB disk space #page\n"}, {"resource": "Class[Mariadb::Monitor_process]", "parameters": "--- Class[Mariadb::Monitor_process].orig\n+++ Class[Mariadb::Monitor_process]\n\n@@\n-    is_critical => False\n+    is_critical => True\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_disk_space\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\"\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page"}, {"resource": "Monitoring::Service[mysqld]", "parameters": "--- Monitoring::Service[mysqld].orig\n+++ Monitoring::Service[mysqld]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_disk_space.service]"}, {"resource": "Nrpe::Monitor_service[mariadb_read_only_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_read_only_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_read_only_backup1-codfw]\n\n@@\n-    critical     => False\n+    critical     => True\n@@\n-    nrpe_command => db-check-health --port=3306 --icinga --check_read_only=true --process\n+    nrpe_command => db-check-health --port=3306 --icinga --check_read_only=false --process\n"}, {"resource": "Monitoring::Exported_nagios_service[db2184 mysqld]", "parameters": "--- Monitoring::Exported_nagios_service[db2184 mysqld].orig\n+++ Monitoring::Exported_nagios_service[db2184 mysqld]\n\n@@\n-    contact_groups        => admins\n+    contact_groups        => admins,sms,admins\n@@\n-    notification_interval => 0\n+    notification_interval => 240\n@@\n-    service_description   => mysqld processes\n+    service_description   => mysqld processes #page\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\"\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page\n"}, {"resource": "Service[pt-heartbeat-wikimedia]", "parameters": "--- Service[pt-heartbeat-wikimedia].orig\n+++ Service[pt-heartbeat-wikimedia]\n\n@@\n-    ensure => False\n+    ensure => True\n"}, {"resource": "Monitoring::Service[mariadb_disk_space]", "parameters": "--- Monitoring::Service[mariadb_disk_space].orig\n+++ Monitoring::Service[mariadb_disk_space]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "Monitoring::Service[mariadb_read_only_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_read_only_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_read_only_backup1-codfw]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "Nrpe::Monitor_service[mysqld]", "parameters": "--- Nrpe::Monitor_service[mysqld].orig\n+++ Nrpe::Monitor_service[mysqld]\n\n@@\n-    critical => False\n+    critical => True\n"}, {"resource": "Mariadb::Monitor_readonly[backup1-codfw]", "parameters": "--- Mariadb::Monitor_readonly[backup1-codfw].orig\n+++ Mariadb::Monitor_readonly[backup1-codfw]\n\n@@\n-    is_critical => False\n+    is_critical => True\n@@\n-    read_only   => True\n+    read_only   => False\n"}], "perc_changed": "3.26%"}}}