{"host": "db2183.codfw.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3001, "only_in_self": ["Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0]", "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1]", "Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e]"], "only_in_other": ["Class[Passwords::Nagios::Mysql]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]", "Mariadb::Monitor_replication[backup1-codfw]", "Monitoring::Exported_nagios_service[db2183 mariadb_replica_io_state_backup1-codfw]", "Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_lag_backup1-codfw]", "Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_state_backup1-codfw]", "Monitoring::Service[mariadb_replica_io_state_backup1-codfw]", "Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]", "Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw]", "Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c]", "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8]", "Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f]", "Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e]", "Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d]", "Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw]", "Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]"], "resource_diffs": [{"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\"\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Monitoring::Service[mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_replica_io_state_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_replica_io_state_backup1-codfw]\n\n+    notes_url      => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    critical       => False\n+    migration_task => T315866\n+    contact_group  => admins\n+    retries        => 3\n+    config_dir     => /etc/nagios\n+    check_command  => nrpe_check!check_mariadb_replica_io_state_backup1-codfw!10\n+    description    => MariaDB Replica IO: backup1-codfw\n+    retry_interval => 1\n+    passive        => False\n+    host           => db2183\n+    freshness      => 36000\n+    check_interval => 1\n+    ensure         => present\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg\n@@ -0,0 +1,2 @@\n+# File generated by puppet. DO NOT edit by hand\n+command[check_mariadb_replica_io_state_backup1-codfw]=/usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_io_state", "parameters": "--- File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg].orig\n+++ File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]\n\n+    tag     => nrpe::check\n+    notify  => Service[nagios-nrpe-server]\n+    group   => root\n+    mode    => 0444\n+    require => Package[nagios-nrpe-server]\n+    ensure  => present\n+    owner   => root\n"}, {"resource": "Nrpe::Check[check_mariadb_read_only_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_read_only_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_read_only_backup1-codfw]\n\n@@\n-    command => db-check-health --port=3306 --icinga --check_read_only=false --process\n+    command => db-check-health --port=3306 --icinga --check_read_only=true --process\n"}, {"resource": "Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw].orig\n+++ Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]\n\n+    accuracy           => 15sec\n+    splay              => 60\n+    unit_name          => nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n+    timer_intervals    => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}, {'interval': '1s', 'start': 'OnActiveSec'}]\n+    fixed_random_delay => True\n+    ensure             => absent\n"}, {"resource": "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]\n\n+    accuracy           => 15sec\n+    splay              => 60\n+    unit_name          => nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n+    timer_intervals    => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}, {'interval': '1s', 'start': 'OnActiveSec'}]\n+    fixed_random_delay => True\n+    ensure             => absent\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mysqld.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mysqld.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mysqld.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mysqld\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\""}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mysqld]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mysqld].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mysqld]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\"\n"}, {"resource": "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]", "content": "--- /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf.orig\n+++ /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf\n@@ -0,0 +1,10 @@\n+# SPDX-License-Identifier: Apache-2.0\n+if $programname contains \"nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw\" then {\n+    if ($msg contains \"\\\"ecs.version\\\": \\\"1.7.0\\\"\") then {\n+        # Send logs to kafka\n+        set $.log_outputs = \"kafka ecs_170 local\";\n+    } else {\n+        # Filter out non-relevant nrpe2nodexp messages\n+        stop\n+    }\n+}", "parameters": "--- File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf].orig\n+++ File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]\n\n+    notify => Service[rsyslog]\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw]\n\n+    notes_url                   => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    critical                    => False\n+    migration_task              => T315866\n+    enable_icinga_check         => True\n+    contact_group               => admins\n+    nrpe2nodexp_parse_perf_data => False\n+    timeout                     => 10\n+    retries                     => 3\n+    description                 => MariaDB Replica SQL: backup1-codfw\n+    nrpe_command                => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_state\n+    retry_interval              => 1\n+    alertmanager_team           => observability\n+    check_interval              => 1\n+    ensure                      => present\n+    enable_nrpe2nodexp          => False\n"}, {"resource": "Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw]\n\n+    notes_url                   => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    critical                    => False\n+    migration_task              => T315866\n+    enable_icinga_check         => True\n+    contact_group               => admins\n+    nrpe2nodexp_parse_perf_data => False\n+    timeout                     => 10\n+    retries                     => 3\n+    description                 => MariaDB Replica IO: backup1-codfw\n+    nrpe_command                => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_io_state\n+    retry_interval              => 1\n+    alertmanager_team           => observability\n+    check_interval              => 1\n+    ensure                      => present\n+    enable_nrpe2nodexp          => False\n"}, {"resource": "Monitoring::Exported_nagios_service[db2183 mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2183 mariadb_replica_io_state_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2183 mariadb_replica_io_state_backup1-codfw]\n\n+    servicegroups          => mysql_codfw\n+    notes_url              => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    host_name              => db2183\n+    check_command          => nrpe_check!check_mariadb_replica_io_state_backup1-codfw!10\n+    contact_groups         => admins\n+    check_period           => 24x7\n+    check_freshness        => 0\n+    notification_options   => c,r,f\n+    is_volatile            => 0\n+    notification_period    => 24x7\n+    retry_interval         => 1\n+    passive_checks_enabled => 1\n+    active_checks_enabled  => 1\n+    notification_interval  => 0\n+    service_description    => MariaDB Replica IO: backup1-codfw\n+    check_interval         => 1\n+    notifications_enabled  => 1\n+    max_check_attempts     => 3\n+    ensure                 => present\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mysqld.service]"}, {"resource": "Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "parameters": "--- Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer].orig\n+++ Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]\n\n+    provider => systemd\n+    enable   => False\n+    ensure   => stopped\n+    before   => ['Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]']\n"}, {"resource": "Monitoring::Exported_nagios_service[db2183 mysqld]", "parameters": "--- Monitoring::Exported_nagios_service[db2183 mysqld].orig\n+++ Monitoring::Exported_nagios_service[db2183 mysqld]\n\n@@\n-    contact_groups        => admins,sms,admins\n+    contact_groups        => admins\n@@\n-    notification_interval => 240\n+    notification_interval => 0\n@@\n-    service_description   => mysqld processes #page\n+    service_description   => mysqld processes\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg\n@@ -1,2 +1,2 @@\n # File generated by puppet. DO NOT edit by hand\n-command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=false --process\n+command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=true --process"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1].orig\n+++ Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1]\n\n-    for                => 3m\n-    group              => nrpechecks\n-    summary            => NRPE CHECK: MariaDB read only backup1-codfw #page\n-    instance           => ops\n-    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting#Master_comes_back_in_read_only\n-    team               => observability\n-    description        => NRPE CHECK: MariaDB read only backup1-codfw\n-    def_label_whitelst => ['team', 'severity']\n-    site               => codfw\n-    alert_name         => nrpe_MariaDB_read_only_backup1_codfw\n-    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"c754855176474e5a67d2bb5c504a51c1\",check_name=\"check_mariadb_read_only_backup1-codfw\", status=\"CRITICAL\", severity=\"page\"} > 0) * on (instance) group_left (team) role_owner\n-    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_read_only_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n-    dashboard          => TODO\n-    ensure             => absent\n-    severity           => info\n"}, {"resource": "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw].orig\n+++ Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]\n\n+    accuracy           => 15sec\n+    splay              => 60\n+    unit_name          => nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n+    timer_intervals    => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}, {'interval': '1s', 'start': 'OnActiveSec'}]\n+    fixed_random_delay => True\n+    ensure             => absent\n"}, {"resource": "Systemd::Service[pt-heartbeat-wikimedia]", "parameters": "--- Systemd::Service[pt-heartbeat-wikimedia].orig\n+++ Systemd::Service[pt-heartbeat-wikimedia]\n\n@@\n-    service_params => {'ensure': True}\n+    service_params => {'ensure': False}\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer\n@@ -0,0 +1,14 @@\n+[Unit]\n+Description=Periodic execution of nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n+\n+[Timer]\n+Unit=nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n+# Accuracy sets the maximum time interval around the execution time we want to allow\n+AccuracySec=15sec\n+OnUnitInactiveSec=1min\n+OnActiveSec=1s\n+RandomizedDelaySec=60\n+FixedRandomDelay=true\n+\n+[Install]\n+WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]\n\n+    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw].orig\n+++ Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]\n\n+    unit_type                => timer\n+    monitoring_enabled       => False\n+    monitoring_critical      => False\n+    migration_task           => T407130\n+    monitoring_contact_group => admins\n+    restart                  => False\n+    service_params           => {}\n+    require                  => Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]\n+    ensure                   => absent\n+    override                 => False\n"}, {"resource": "Class[Profile::Mariadb::Mysql_role]", "parameters": "--- Class[Profile::Mariadb::Mysql_role].orig\n+++ Class[Profile::Mariadb::Mysql_role]\n\n@@\n-    role => master\n+    role => slave\n"}, {"resource": "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]", "content": "--- /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf.orig\n+++ /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf\n@@ -0,0 +1,10 @@\n+# SPDX-License-Identifier: Apache-2.0\n+if $programname contains \"nrpe2nodexp-mariadb_replica_io_state_backup1-codfw\" then {\n+    if ($msg contains \"\\\"ecs.version\\\": \\\"1.7.0\\\"\") then {\n+        # Send logs to kafka\n+        set $.log_outputs = \"kafka ecs_170 local\";\n+    } else {\n+        # Filter out non-relevant nrpe2nodexp messages\n+        stop\n+    }\n+}", "parameters": "--- File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf].orig\n+++ File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]\n\n+    notify => Service[rsyslog]\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw].orig\n+++ Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]\n\n+    priority => 25\n+    mode     => 0444\n+    ensure   => absent\n"}, {"resource": "Class[Mariadb::Config]", "parameters": "--- Class[Mariadb::Config].orig\n+++ Class[Mariadb::Config]\n\n@@\n-    read_only => 0\n+    read_only => 1\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw].orig\n+++ Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]\n\n+    priority => 25\n+    mode     => 0444\n+    ensure   => absent\n"}, {"resource": "Monitoring::Exported_nagios_service[db2183 mariadb_disk_space]", "parameters": "--- Monitoring::Exported_nagios_service[db2183 mariadb_disk_space].orig\n+++ Monitoring::Exported_nagios_service[db2183 mariadb_disk_space]\n\n@@\n-    contact_groups        => admins,sms,admins\n+    contact_groups        => admins\n@@\n-    notification_interval => 240\n+    notification_interval => 0\n@@\n-    service_description   => MariaDB disk space #page\n+    service_description   => MariaDB disk space\n"}, {"resource": "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]\n\n+    priority => 25\n+    mode     => 0444\n+    ensure   => absent\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_read_only_backup1-codfw.service]"}, {"resource": "Nrpe::Monitor_service[mariadb_disk_space]", "parameters": "--- Nrpe::Monitor_service[mariadb_disk_space].orig\n+++ Nrpe::Monitor_service[mariadb_disk_space]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8].orig\n+++ Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8]\n\n+    for                => 3m\n+    group              => nrpechecks\n+    summary            => NRPE CHECK: MariaDB read only backup1-codfw\n+    instance           => ops\n+    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting#Master_comes_back_in_read_only\n+    team               => observability\n+    description        => NRPE CHECK: MariaDB read only backup1-codfw\n+    def_label_whitelst => ['team', 'severity']\n+    site               => codfw\n+    alert_name         => nrpe_MariaDB_read_only_backup1_codfw\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"8aa8e94c48d271ddfcbf96b11f04e0e8\",check_name=\"check_mariadb_read_only_backup1-codfw\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_read_only_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    dashboard          => TODO\n+    ensure             => absent\n+    severity           => info\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]\n\n+    restart           => False\n+    override          => False\n+    require           => ['Class[Systemd]']\n+    override_filename => puppet-override.conf\n+    ensure            => absent\n+    unit              => nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n"}, {"resource": "Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "parameters": "--- Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer].orig\n+++ Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]\n\n+    provider => systemd\n+    enable   => False\n+    ensure   => stopped\n+    before   => ['Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]']\n"}, {"resource": "Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_lag_backup1-codfw]\n\n+    servicegroups          => mysql_codfw\n+    notes_url              => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    host_name              => db2183\n+    check_command          => nrpe_check!check_mariadb_replica_sql_lag_backup1-codfw!10\n+    contact_groups         => admins\n+    check_period           => 24x7\n+    check_freshness        => 0\n+    notification_options   => c,r,f\n+    is_volatile            => 0\n+    notification_period    => 24x7\n+    retry_interval         => 1\n+    passive_checks_enabled => 1\n+    active_checks_enabled  => 1\n+    notification_interval  => 0\n+    service_description    => MariaDB Replica Lag: backup1-codfw\n+    check_interval         => 1\n+    notifications_enabled  => 1\n+    max_check_attempts     => 10\n+    ensure                 => present\n"}, {"resource": "Class[Mariadb::Monitor_process]", "parameters": "--- Class[Mariadb::Monitor_process].orig\n+++ Class[Mariadb::Monitor_process]\n\n@@\n-    is_critical => True\n+    is_critical => False\n"}, {"resource": "Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw]\n\n+    notes_url                   => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    critical                    => False\n+    migration_task              => T315866\n+    enable_icinga_check         => True\n+    contact_group               => admins\n+    nrpe2nodexp_parse_perf_data => False\n+    timeout                     => 10\n+    retries                     => 10\n+    description                 => MariaDB Replica Lag: backup1-codfw\n+    nrpe_command                => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_lag                            --shard=backup1-codfw --datacenter=codfw                            --sql-lag-warn=60                            --sql-lag-crit=300\n+    retry_interval              => 1\n+    alertmanager_team           => observability\n+    check_interval              => 1\n+    ensure                      => present\n+    enable_nrpe2nodexp          => False\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_disk_space\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\""}, {"resource": "Monitoring::Service[mysqld]", "parameters": "--- Monitoring::Service[mysqld].orig\n+++ Monitoring::Service[mysqld]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer\n@@ -0,0 +1,14 @@\n+[Unit]\n+Description=Periodic execution of nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n+\n+[Timer]\n+Unit=nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n+# Accuracy sets the maximum time interval around the execution time we want to allow\n+AccuracySec=15sec\n+OnUnitInactiveSec=1min\n+OnActiveSec=1s\n+RandomizedDelaySec=60\n+FixedRandomDelay=true\n+\n+[Install]\n+WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]\n\n+    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c].orig\n+++ Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c]\n\n+    for                => 3m\n+    group              => nrpechecks\n+    summary            => NRPE CHECK: MariaDB disk space\n+    instance           => ops\n+    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting\n+    team               => observability\n+    description        => NRPE CHECK: MariaDB disk space\n+    def_label_whitelst => ['team', 'severity']\n+    site               => codfw\n+    alert_name         => nrpe_MariaDB_disk_space\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"d2aae7bfec5549be1a8b9413215bd86c\",check_name=\"check_mariadb_disk_space\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_disk_space))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    dashboard          => TODO\n+    ensure             => absent\n+    severity           => info\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg\n@@ -0,0 +1,2 @@\n+# File generated by puppet. DO NOT edit by hand\n+command[check_mariadb_replica_sql_state_backup1-codfw]=/usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_state", "parameters": "--- File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg].orig\n+++ File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]\n\n+    tag     => nrpe::check\n+    notify  => Service[nagios-nrpe-server]\n+    group   => root\n+    mode    => 0444\n+    require => Package[nagios-nrpe-server]\n+    ensure  => present\n+    owner   => root\n"}, {"resource": "Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw]\n\n+    command => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_io_state\n+    ensure  => present\n+    before  => Monitoring::Service[mariadb_replica_io_state_backup1-codfw]\n"}, {"resource": "Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw].orig\n+++ Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw]\n\n+    privileges => []\n+    user       => nagios\n+    tag        => nrpe::check\n+    require    => ['Class[Sudo]']\n+    ensure     => absent\n"}, {"resource": "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]", "parameters": "--- File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom].orig\n+++ File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]\n\n+    group  => root\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\"\n"}, {"resource": "Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "parameters": "--- Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer].orig\n+++ Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]\n\n+    provider => systemd\n+    enable   => False\n+    ensure   => stopped\n+    before   => ['Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]']\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0].orig\n+++ Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0]\n\n-    for                => 3m\n-    group              => nrpechecks\n-    summary            => NRPE CHECK: MariaDB disk space #page\n-    instance           => ops\n-    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting\n-    team               => observability\n-    description        => NRPE CHECK: MariaDB disk space\n-    def_label_whitelst => ['team', 'severity']\n-    site               => codfw\n-    alert_name         => nrpe_MariaDB_disk_space\n-    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"40ee06f86a277811ae30c96774f3ddc0\",check_name=\"check_mariadb_disk_space\", status=\"CRITICAL\", severity=\"page\"} > 0) * on (instance) group_left (team) role_owner\n-    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_disk_space))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n-    dashboard          => TODO\n-    ensure             => absent\n-    severity           => info\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n@@ -0,0 +1,11 @@\n+[Unit]\n+Description=execution of nrpe2nodexp for the check_mariadb_replica_sql_lag_backup1-codfw command.\n+Documentation=https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+\n+[Service]\n+Type=oneshot\n+User=nagios\n+\n+Group=prometheus-node-exporter\n+SyslogIdentifier=nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"8688c7f4c31f8bd2553cec605fe5ef5e\" --timeout 10 --check-command \"check_mariadb_replica_sql_lag_backup1-codfw\"", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]\n\n+    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Monitoring::Service[mariadb_disk_space]", "parameters": "--- Monitoring::Service[mariadb_disk_space].orig\n+++ Monitoring::Service[mariadb_disk_space]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "Monitoring::Service[mariadb_read_only_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_read_only_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_read_only_backup1-codfw]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_state_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_state_backup1-codfw]\n\n+    servicegroups          => mysql_codfw\n+    notes_url              => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    host_name              => db2183\n+    check_command          => nrpe_check!check_mariadb_replica_sql_state_backup1-codfw!10\n+    contact_groups         => admins\n+    check_period           => 24x7\n+    check_freshness        => 0\n+    notification_options   => c,r,f\n+    is_volatile            => 0\n+    notification_period    => 24x7\n+    retry_interval         => 1\n+    passive_checks_enabled => 1\n+    active_checks_enabled  => 1\n+    notification_interval  => 0\n+    service_description    => MariaDB Replica SQL: backup1-codfw\n+    check_interval         => 1\n+    notifications_enabled  => 1\n+    max_check_attempts     => 3\n+    ensure                 => present\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer\n@@ -0,0 +1,14 @@\n+[Unit]\n+Description=Periodic execution of nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n+\n+[Timer]\n+Unit=nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n+# Accuracy sets the maximum time interval around the execution time we want to allow\n+AccuracySec=15sec\n+OnUnitInactiveSec=1min\n+OnActiveSec=1s\n+RandomizedDelaySec=60\n+FixedRandomDelay=true\n+\n+[Install]\n+WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]\n\n+    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]", "parameters": "--- File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw].orig\n+++ File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]\n\n+    group   => root\n+    require => Package[nagios-nrpe-server]\n+    ensure  => absent\n+    owner   => root\n"}, {"resource": "Nrpe::Monitor_service[mysqld]", "parameters": "--- Nrpe::Monitor_service[mysqld].orig\n+++ Nrpe::Monitor_service[mysqld]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "Mariadb::Monitor_replication[backup1-codfw]", "parameters": "--- Mariadb::Monitor_replication[backup1-codfw].orig\n+++ Mariadb::Monitor_replication[backup1-codfw]\n\n+    source_dc     => codfw\n+    lag_warn      => 60\n+    contact_group => admins\n+    lag_crit      => 300\n+    socket        => /run/mysqld/mysqld.sock\n+    multisource   => False\n+    is_critical   => False\n+    warn_stopped  => True\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d].orig\n+++ Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d]\n\n+    for                => 3m\n+    group              => nrpechecks\n+    summary            => NRPE CHECK: MariaDB Replica SQL: backup1-codfw\n+    instance           => ops\n+    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    team               => observability\n+    description        => NRPE CHECK: MariaDB Replica SQL: backup1-codfw\n+    def_label_whitelst => ['team', 'severity']\n+    site               => codfw\n+    alert_name         => nrpe_MariaDB_Replica_SQL__backup1_codfw\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"872d6618b9a5af3d398dc5cd99b2733d\",check_name=\"check_mariadb_replica_sql_state_backup1-codfw\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_replica_sql_state_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    dashboard          => TODO\n+    ensure             => absent\n+    severity           => info\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]\n\n+    ignore_errors             => True\n+    monitoring_enabled        => False\n+    logfile_basedir           => /var/log\n+    success_exit_status       => []\n+    syslog_identifier         => nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw\n+    logfile_perms             => all\n+    logfile_group             => root\n+    syslog_force_stop         => True\n+    send_mail_only_on_error   => True\n+    logging_enabled           => False\n+    description               => execution of nrpe2nodexp for the check_mariadb_replica_sql_lag_backup1-codfw command.\n+    logfile_name              => syslog.log\n+    send_mail_to              => root@db2183.codfw.wmnet\n+    command                   => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"8688c7f4c31f8bd2553cec605fe5ef5e\" --timeout 10 --check-command \"check_mariadb_replica_sql_lag_backup1-codfw\"\n+    interval                  => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}]\n+    syslog_match_startswith   => True\n+    user                      => nagios\n+    group                     => prometheus-node-exporter\n+    private_tmp               => False\n+    environment               => {}\n+    send_mail                 => False\n+    splay                     => 60\n+    monitoring_contact_groups => admins\n+    fixed_random_delay        => True\n+    monitoring_notes_url      => https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+    ensure                    => absent\n"}, {"resource": "Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116]", "parameters": "--- Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116].orig\n+++ Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116]\n\n+    for                => 3m\n+    group              => nrpechecks\n+    summary            => NRPE CHECK: mysqld processes\n+    instance           => ops\n+    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting\n+    team               => observability\n+    description        => NRPE CHECK: mysqld processes\n+    def_label_whitelst => ['team', 'severity']\n+    site               => codfw\n+    alert_name         => nrpe_mysqld_processes\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"ca81db2c9e98d408f9f65c79817da116\",check_name=\"check_mysqld\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mysqld))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    dashboard          => TODO\n+    ensure             => absent\n+    severity           => info\n"}, {"resource": "Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw]\n\n+    privileges => []\n+    user       => nagios\n+    tag        => nrpe::check\n+    require    => ['Class[Sudo]']\n+    ensure     => absent\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]\n\n+    override_filename => puppet-override.conf\n+    override          => False\n+    require           => ['Class[Systemd]']\n+    restart           => False\n+    ensure            => absent\n+    unit              => nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer\n"}, {"resource": "Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e]", "parameters": "--- Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e].orig\n+++ Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e]\n\n-    for                => 3m\n-    group              => nrpechecks\n-    summary            => NRPE CHECK: mysqld processes #page\n-    instance           => ops\n-    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/troubleshooting\n-    team               => observability\n-    description        => NRPE CHECK: mysqld processes\n-    def_label_whitelst => ['team', 'severity']\n-    site               => codfw\n-    alert_name         => nrpe_mysqld_processes\n-    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"fc384e29e203c0920c26ca72c745692e\",check_name=\"check_mysqld\", status=\"CRITICAL\", severity=\"page\"} > 0) * on (instance) group_left (team) role_owner\n-    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mysqld))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n-    dashboard          => TODO\n-    ensure             => absent\n-    severity           => info\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n@@ -0,0 +1,11 @@\n+[Unit]\n+Description=execution of nrpe2nodexp for the check_mariadb_replica_io_state_backup1-codfw command.\n+Documentation=https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+\n+[Service]\n+Type=oneshot\n+User=nagios\n+\n+Group=prometheus-node-exporter\n+SyslogIdentifier=nrpe2nodexp-mariadb_replica_io_state_backup1-codfw\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca85d57665ee50d48a78933a6514ca3f\" --timeout 10 --check-command \"check_mariadb_replica_io_state_backup1-codfw\"", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]\n\n+    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_read_only_backup1-codfw\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\""}, {"resource": "Class[Mariadb::Heartbeat]", "parameters": "--- Class[Mariadb::Heartbeat].orig\n+++ Class[Mariadb::Heartbeat]\n\n@@\n-    enabled => True\n+    enabled => False\n"}, {"resource": "File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]", "parameters": "--- File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom].orig\n+++ File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]\n\n+    group  => root\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]\n\n+    ignore_errors             => True\n+    monitoring_enabled        => False\n+    logfile_basedir           => /var/log\n+    success_exit_status       => []\n+    syslog_identifier         => nrpe2nodexp-mariadb_replica_io_state_backup1-codfw\n+    logfile_perms             => all\n+    logfile_group             => root\n+    syslog_force_stop         => True\n+    send_mail_only_on_error   => True\n+    logging_enabled           => False\n+    description               => execution of nrpe2nodexp for the check_mariadb_replica_io_state_backup1-codfw command.\n+    logfile_name              => syslog.log\n+    send_mail_to              => root@db2183.codfw.wmnet\n+    command                   => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca85d57665ee50d48a78933a6514ca3f\" --timeout 10 --check-command \"check_mariadb_replica_io_state_backup1-codfw\"\n+    interval                  => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}]\n+    syslog_match_startswith   => True\n+    user                      => nagios\n+    group                     => prometheus-node-exporter\n+    private_tmp               => False\n+    environment               => {}\n+    send_mail                 => False\n+    splay                     => 60\n+    monitoring_contact_groups => admins\n+    fixed_random_delay        => True\n+    monitoring_notes_url      => https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+    ensure                    => absent\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]\n\n+    restart           => False\n+    override          => False\n+    require           => ['Class[Systemd]']\n+    override_filename => puppet-override.conf\n+    ensure            => absent\n+    unit              => nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg\n@@ -0,0 +1,2 @@\n+# File generated by puppet. DO NOT edit by hand\n+command[check_mariadb_replica_sql_lag_backup1-codfw]=/usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_lag                            --shard=backup1-codfw --datacenter=codfw                            --sql-lag-warn=60                            --sql-lag-crit=300", "parameters": "--- File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg].orig\n+++ File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]\n\n+    tag     => nrpe::check\n+    notify  => Service[nagios-nrpe-server]\n+    group   => root\n+    mode    => 0444\n+    require => Package[nagios-nrpe-server]\n+    ensure  => present\n+    owner   => root\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]\n\n+    override_filename => puppet-override.conf\n+    override          => False\n+    require           => ['Class[Systemd]']\n+    restart           => False\n+    ensure            => absent\n+    unit              => nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer\n"}, {"resource": "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw].orig\n+++ File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]\n\n+    group   => root\n+    require => Package[nagios-nrpe-server]\n+    ensure  => absent\n+    owner   => root\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw]\n\n+    command => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_lag                            --shard=backup1-codfw --datacenter=codfw                            --sql-lag-warn=60                            --sql-lag-crit=300\n+    ensure  => present\n+    before  => Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]\n"}, {"resource": "Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]\n\n+    notes_url      => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    critical       => False\n+    migration_task => T315866\n+    contact_group  => admins\n+    retries        => 10\n+    config_dir     => /etc/nagios\n+    check_command  => nrpe_check!check_mariadb_replica_sql_lag_backup1-codfw!10\n+    description    => MariaDB Replica Lag: backup1-codfw\n+    retry_interval => 1\n+    passive        => False\n+    host           => db2183\n+    freshness      => 36000\n+    check_interval => 1\n+    ensure         => present\n"}, {"resource": "File[/etc/my.cnf]", "content": "--- /etc/my.cnf.orig\n+++ /etc/my.cnf\n@@ -16,7 +16,7 @@\n server_id          = 180356633\n report_host        = db2183.codfw.wmnet\n wsrep_cluster_name = codfw # For orchestrator. T266635\n-read_only          = 0\n+read_only          = 1\n \n log_bin\n log_slave_updates"}, {"resource": "Class[Mariadb::Monitor_disk]", "parameters": "--- Class[Mariadb::Monitor_disk].orig\n+++ Class[Mariadb::Monitor_disk]\n\n@@\n-    is_critical => True\n+    is_critical => False\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]\n\n+    restart           => False\n+    override          => False\n+    require           => ['Class[Systemd]']\n+    override_filename => puppet-override.conf\n+    ensure            => absent\n+    unit              => nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service\n"}, {"resource": "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]", "content": "--- /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf.orig\n+++ /etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf\n@@ -0,0 +1,10 @@\n+# SPDX-License-Identifier: Apache-2.0\n+if $programname contains \"nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw\" then {\n+    if ($msg contains \"\\\"ecs.version\\\": \\\"1.7.0\\\"\") then {\n+        # Send logs to kafka\n+        set $.log_outputs = \"kafka ecs_170 local\";\n+    } else {\n+        # Filter out non-relevant nrpe2nodexp messages\n+        stop\n+    }\n+}", "parameters": "--- File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf].orig\n+++ File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]\n\n+    notify => Service[rsyslog]\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]", "parameters": "--- File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom].orig\n+++ File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]\n\n+    group  => root\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Monitoring::Exported_nagios_service[db2183 mariadb_read_only_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2183 mariadb_read_only_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2183 mariadb_read_only_backup1-codfw]\n\n@@\n-    contact_groups        => admins,sms,admins\n+    contact_groups        => admins\n@@\n-    notification_interval => 240\n+    notification_interval => 0\n@@\n-    service_description   => MariaDB read only backup1-codfw #page\n+    service_description   => MariaDB read only backup1-codfw\n"}, {"resource": "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw].orig\n+++ Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]\n\n+    unit_type                => timer\n+    monitoring_enabled       => False\n+    monitoring_critical      => False\n+    migration_task           => T407130\n+    monitoring_contact_group => admins\n+    restart                  => False\n+    service_params           => {}\n+    require                  => Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]\n+    ensure                   => absent\n+    override                 => False\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_disk_space.service]"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Nrpe::Monitor_service[mariadb_read_only_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_read_only_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_read_only_backup1-codfw]\n\n@@\n-    critical     => True\n+    critical     => False\n@@\n-    nrpe_command => db-check-health --port=3306 --icinga --check_read_only=false --process\n+    nrpe_command => db-check-health --port=3306 --icinga --check_read_only=true --process\n"}, {"resource": "Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw].orig\n+++ Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw]\n\n+    privileges => []\n+    user       => nagios\n+    tag        => nrpe::check\n+    require    => ['Class[Sudo]']\n+    ensure     => absent\n"}, {"resource": "Service[pt-heartbeat-wikimedia]", "parameters": "--- Service[pt-heartbeat-wikimedia].orig\n+++ Service[pt-heartbeat-wikimedia]\n\n@@\n-    ensure => True\n+    ensure => False\n"}, {"resource": "Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw]\n\n+    command => /usr/local/lib/nagios/plugins/check_mariadb --sock=/run/mysqld/mysqld.sock --user=nagios --pass=  --warn-stopped --check=slave_sql_state\n+    ensure  => present\n+    before  => Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]\n"}, {"resource": "Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_replica_sql_state_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]\n\n+    notes_url      => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    critical       => False\n+    migration_task => T315866\n+    contact_group  => admins\n+    retries        => 3\n+    config_dir     => /etc/nagios\n+    check_command  => nrpe_check!check_mariadb_replica_sql_state_backup1-codfw!10\n+    description    => MariaDB Replica SQL: backup1-codfw\n+    retry_interval => 1\n+    passive        => False\n+    host           => db2183\n+    freshness      => 36000\n+    check_interval => 1\n+    ensure         => present\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f].orig\n+++ Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f]\n\n+    for                => 3m\n+    group              => nrpechecks\n+    summary            => NRPE CHECK: MariaDB Replica IO: backup1-codfw\n+    instance           => ops\n+    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    team               => observability\n+    description        => NRPE CHECK: MariaDB Replica IO: backup1-codfw\n+    def_label_whitelst => ['team', 'severity']\n+    site               => codfw\n+    alert_name         => nrpe_MariaDB_Replica_IO__backup1_codfw\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"ca85d57665ee50d48a78933a6514ca3f\",check_name=\"check_mariadb_replica_io_state_backup1-codfw\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_replica_io_state_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    dashboard          => TODO\n+    ensure             => absent\n+    severity           => info\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service\n@@ -0,0 +1,11 @@\n+[Unit]\n+Description=execution of nrpe2nodexp for the check_mariadb_replica_sql_state_backup1-codfw command.\n+Documentation=https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+\n+[Service]\n+Type=oneshot\n+User=nagios\n+\n+Group=prometheus-node-exporter\n+SyslogIdentifier=nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"872d6618b9a5af3d398dc5cd99b2733d\" --timeout 10 --check-command \"check_mariadb_replica_sql_state_backup1-codfw\"", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]\n\n+    notify => Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e]", "parameters": "--- Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e].orig\n+++ Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e]\n\n+    for                => 10m\n+    group              => nrpechecks\n+    summary            => NRPE CHECK: MariaDB Replica Lag: backup1-codfw\n+    instance           => ops\n+    runbook            => https://wikitech.wikimedia.org/wiki/MariaDB/Troubleshooting#Incident_Response\n+    team               => observability\n+    description        => NRPE CHECK: MariaDB Replica Lag: backup1-codfw\n+    def_label_whitelst => ['team', 'severity']\n+    site               => codfw\n+    alert_name         => nrpe_MariaDB_Replica_Lag__backup1_codfw\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"8688c7f4c31f8bd2553cec605fe5ef5e\",check_name=\"check_mariadb_replica_sql_lag_backup1-codfw\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_mariadb_replica_sql_lag_backup1-codfw))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    dashboard          => TODO\n+    ensure             => absent\n+    severity           => info\n"}, {"resource": "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw].orig\n+++ Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]\n\n+    unit_type                => timer\n+    monitoring_enabled       => False\n+    monitoring_critical      => False\n+    migration_task           => T407130\n+    monitoring_contact_group => admins\n+    restart                  => False\n+    service_params           => {}\n+    require                  => Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]\n+    ensure                   => absent\n+    override                 => False\n"}, {"resource": "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "parameters": "--- File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw].orig\n+++ File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]\n\n+    group   => root\n+    require => Package[nagios-nrpe-server]\n+    ensure  => absent\n+    owner   => root\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]\n\n+    ignore_errors             => True\n+    monitoring_enabled        => False\n+    logfile_basedir           => /var/log\n+    success_exit_status       => []\n+    syslog_identifier         => nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw\n+    logfile_perms             => all\n+    logfile_group             => root\n+    syslog_force_stop         => True\n+    send_mail_only_on_error   => True\n+    logging_enabled           => False\n+    description               => execution of nrpe2nodexp for the check_mariadb_replica_sql_state_backup1-codfw command.\n+    logfile_name              => syslog.log\n+    send_mail_to              => root@db2183.codfw.wmnet\n+    command                   => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"872d6618b9a5af3d398dc5cd99b2733d\" --timeout 10 --check-command \"check_mariadb_replica_sql_state_backup1-codfw\"\n+    interval                  => [{'start': 'OnUnitInactiveSec', 'interval': '1min'}]\n+    syslog_match_startswith   => True\n+    user                      => nagios\n+    group                     => prometheus-node-exporter\n+    private_tmp               => False\n+    environment               => {}\n+    send_mail                 => False\n+    splay                     => 60\n+    monitoring_contact_groups => admins\n+    fixed_random_delay        => True\n+    monitoring_notes_url      => https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+    ensure                    => absent\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "parameters": "--- Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer].orig\n+++ Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]\n\n+    override_filename => puppet-override.conf\n+    override          => False\n+    require           => ['Class[Systemd]']\n+    restart           => False\n+    ensure            => absent\n+    unit              => nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer\n"}, {"resource": "Mariadb::Monitor_readonly[backup1-codfw]", "parameters": "--- Mariadb::Monitor_readonly[backup1-codfw].orig\n+++ Mariadb::Monitor_readonly[backup1-codfw]\n\n@@\n-    is_critical => True\n+    is_critical => False\n@@\n-    read_only   => False\n+    read_only   => True\n"}], "perc_changed": "5.66%"}, "core": {"total": 3001, "only_in_self": [], "only_in_other": ["Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]", "Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]"], "resource_diffs": [{"resource": "File[/lib/systemd/system/nrpe2nodexp-mysqld.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mysqld.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mysqld.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mysqld\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\""}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_read_only_backup1-codfw\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\""}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg\n@@ -1,2 +1,2 @@\n # File generated by puppet. DO NOT edit by hand\n-command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=false --process\n+command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=true --process"}, {"resource": "File[/etc/my.cnf]", "content": "--- /etc/my.cnf.orig\n+++ /etc/my.cnf\n@@ -16,7 +16,7 @@\n server_id          = 180356633\n report_host        = db2183.codfw.wmnet\n wsrep_cluster_name = codfw # For orchestrator. T266635\n-read_only          = 0\n+read_only          = 1\n \n log_bin\n log_slave_updates"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_disk_space\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\""}, {"resource": "Service[pt-heartbeat-wikimedia]", "parameters": "--- Service[pt-heartbeat-wikimedia].orig\n+++ Service[pt-heartbeat-wikimedia]\n\n@@\n-    ensure => True\n+    ensure => False\n"}], "perc_changed": "1.10%"}, "main": {"total": 3001, "only_in_self": ["Prometheus::Alert::Rule[check_mariadb_disk_space_40ee06f86a277811ae30c96774f3ddc0]", "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_c754855176474e5a67d2bb5c504a51c1]", "Prometheus::Alert::Rule[check_mysqld_fc384e29e203c0920c26ca72c745692e]"], "only_in_other": ["Class[Passwords::Nagios::Mysql]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer (nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer)]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_io_state_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_lag_backup1-codfw.cfg]", "File[/etc/nagios/nrpe.d/check_mariadb_replica_sql_state_backup1-codfw.cfg]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-io-state-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-lag-backup1-codfw.conf]", "File[/etc/rsyslog.d/25-nrpe2nodexp-mariadb-replica-sql-state-backup1-codfw.conf]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_io_state_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "File[/etc/sudoers.d/nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "File[/lib/systemd/system/nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_io_state_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_lag_backup1-codfw.prom]", "File[/var/lib/prometheus/node.d/check_mariadb_replica_sql_state_backup1-codfw.prom]", "Mariadb::Monitor_replication[backup1-codfw]", "Monitoring::Exported_nagios_service[db2183 mariadb_replica_io_state_backup1-codfw]", "Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_lag_backup1-codfw]", "Monitoring::Exported_nagios_service[db2183 mariadb_replica_sql_state_backup1-codfw]", "Monitoring::Service[mariadb_replica_io_state_backup1-codfw]", "Monitoring::Service[mariadb_replica_sql_lag_backup1-codfw]", "Monitoring::Service[mariadb_replica_sql_state_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_io_state_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_sql_lag_backup1-codfw]", "Nrpe::Check[check_mariadb_replica_sql_state_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_io_state_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_sql_lag_backup1-codfw]", "Nrpe::Monitor_service[mariadb_replica_sql_state_backup1-codfw]", "Prometheus::Alert::Rule[check_mariadb_disk_space_d2aae7bfec5549be1a8b9413215bd86c]", "Prometheus::Alert::Rule[check_mariadb_read_only_backup1-codfw_8aa8e94c48d271ddfcbf96b11f04e0e8]", "Prometheus::Alert::Rule[check_mariadb_replica_io_state_backup1-codfw_ca85d57665ee50d48a78933a6514ca3f]", "Prometheus::Alert::Rule[check_mariadb_replica_sql_lag_backup1-codfw_8688c7f4c31f8bd2553cec605fe5ef5e]", "Prometheus::Alert::Rule[check_mariadb_replica_sql_state_backup1-codfw_872d6618b9a5af3d398dc5cd99b2733d]", "Prometheus::Alert::Rule[check_mysqld_ca81db2c9e98d408f9f65c79817da116]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Rsyslog::Conf[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]", "Sudo::User[nrpe-check_mariadb_replica_io_state_backup1-codfw]", "Sudo::User[nrpe-check_mariadb_replica_sql_lag_backup1-codfw]", "Sudo::User[nrpe-check_mariadb_replica_sql_state_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Service[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Timer::Job[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw]", "Systemd::Timer[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_io_state_backup1-codfw.timer]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_lag_backup1-codfw.timer]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.service]", "Systemd::Unit[nrpe2nodexp-mariadb_replica_sql_state_backup1-codfw.timer]"], "resource_diffs": [{"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_read_only_backup1-codfw]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\"\n"}, {"resource": "Nrpe::Check[check_mariadb_read_only_backup1-codfw]", "parameters": "--- Nrpe::Check[check_mariadb_read_only_backup1-codfw].orig\n+++ Nrpe::Check[check_mariadb_read_only_backup1-codfw]\n\n@@\n-    command => db-check-health --port=3306 --icinga --check_read_only=false --process\n+    command => db-check-health --port=3306 --icinga --check_read_only=true --process\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mysqld.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mysqld.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mysqld.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mysqld\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\""}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mysqld]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mysqld].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mysqld]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"fc384e29e203c0920c26ca72c745692e\" --timeout 10 --check-command \"check_mysqld\" --page\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"ca81db2c9e98d408f9f65c79817da116\" --timeout 10 --check-command \"check_mysqld\"\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_read_only_backup1-codfw.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_read_only_backup1-codfw\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"c754855176474e5a67d2bb5c504a51c1\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\" --page\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"8aa8e94c48d271ddfcbf96b11f04e0e8\" --timeout 10 --check-command \"check_mariadb_read_only_backup1-codfw\""}, {"resource": "Class[Mariadb::Heartbeat]", "parameters": "--- Class[Mariadb::Heartbeat].orig\n+++ Class[Mariadb::Heartbeat]\n\n@@\n-    enabled => True\n+    enabled => False\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mysqld.service]"}, {"resource": "Monitoring::Exported_nagios_service[db2183 mysqld]", "parameters": "--- Monitoring::Exported_nagios_service[db2183 mysqld].orig\n+++ Monitoring::Exported_nagios_service[db2183 mysqld]\n\n@@\n-    contact_groups        => admins,sms,admins\n+    contact_groups        => admins\n@@\n-    notification_interval => 240\n+    notification_interval => 0\n@@\n-    service_description   => mysqld processes #page\n+    service_description   => mysqld processes\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg]", "content": "--- /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg.orig\n+++ /etc/nagios/nrpe.d/check_mariadb_read_only_backup1-codfw.cfg\n@@ -1,2 +1,2 @@\n # File generated by puppet. DO NOT edit by hand\n-command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=false --process\n+command[check_mariadb_read_only_backup1-codfw]=db-check-health --port=3306 --icinga --check_read_only=true --process"}, {"resource": "Systemd::Service[pt-heartbeat-wikimedia]", "parameters": "--- Systemd::Service[pt-heartbeat-wikimedia].orig\n+++ Systemd::Service[pt-heartbeat-wikimedia]\n\n@@\n-    service_params => {'ensure': True}\n+    service_params => {'ensure': False}\n"}, {"resource": "Class[Profile::Mariadb::Mysql_role]", "parameters": "--- Class[Profile::Mariadb::Mysql_role].orig\n+++ Class[Profile::Mariadb::Mysql_role]\n\n@@\n-    role => master\n+    role => slave\n"}, {"resource": "Class[Mariadb::Config]", "parameters": "--- Class[Mariadb::Config].orig\n+++ Class[Mariadb::Config]\n\n@@\n-    read_only => 0\n+    read_only => 1\n"}, {"resource": "Monitoring::Exported_nagios_service[db2183 mariadb_disk_space]", "parameters": "--- Monitoring::Exported_nagios_service[db2183 mariadb_disk_space].orig\n+++ Monitoring::Exported_nagios_service[db2183 mariadb_disk_space]\n\n@@\n-    contact_groups        => admins,sms,admins\n+    contact_groups        => admins\n@@\n-    notification_interval => 240\n+    notification_interval => 0\n@@\n-    service_description   => MariaDB disk space #page\n+    service_description   => MariaDB disk space\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_read_only_backup1-codfw.service]"}, {"resource": "Nrpe::Monitor_service[mariadb_disk_space]", "parameters": "--- Nrpe::Monitor_service[mariadb_disk_space].orig\n+++ Nrpe::Monitor_service[mariadb_disk_space]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "File[/etc/my.cnf]", "content": "--- /etc/my.cnf.orig\n+++ /etc/my.cnf\n@@ -16,7 +16,7 @@\n server_id          = 180356633\n report_host        = db2183.codfw.wmnet\n wsrep_cluster_name = codfw # For orchestrator. T266635\n-read_only          = 0\n+read_only          = 1\n \n log_bin\n log_slave_updates"}, {"resource": "Class[Mariadb::Monitor_disk]", "parameters": "--- Class[Mariadb::Monitor_disk].orig\n+++ Class[Mariadb::Monitor_disk]\n\n@@\n-    is_critical => True\n+    is_critical => False\n"}, {"resource": "Monitoring::Exported_nagios_service[db2183 mariadb_read_only_backup1-codfw]", "parameters": "--- Monitoring::Exported_nagios_service[db2183 mariadb_read_only_backup1-codfw].orig\n+++ Monitoring::Exported_nagios_service[db2183 mariadb_read_only_backup1-codfw]\n\n@@\n-    contact_groups        => admins,sms,admins\n+    contact_groups        => admins\n@@\n-    notification_interval => 240\n+    notification_interval => 0\n@@\n-    service_description   => MariaDB read only backup1-codfw #page\n+    service_description   => MariaDB read only backup1-codfw\n"}, {"resource": "Class[Mariadb::Monitor_process]", "parameters": "--- Class[Mariadb::Monitor_process].orig\n+++ Class[Mariadb::Monitor_process]\n\n@@\n-    is_critical => True\n+    is_critical => False\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-mariadb_disk_space.service\n@@ -8,4 +8,4 @@\n \n Group=prometheus-node-exporter\n SyslogIdentifier=nrpe2nodexp-mariadb_disk_space\n-ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\""}, {"resource": "Monitoring::Service[mysqld]", "parameters": "--- Monitoring::Service[mysqld].orig\n+++ Monitoring::Service[mysqld]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-mariadb_disk_space.service]"}, {"resource": "Nrpe::Monitor_service[mariadb_read_only_backup1-codfw]", "parameters": "--- Nrpe::Monitor_service[mariadb_read_only_backup1-codfw].orig\n+++ Nrpe::Monitor_service[mariadb_read_only_backup1-codfw]\n\n@@\n-    critical     => True\n+    critical     => False\n@@\n-    nrpe_command => db-check-health --port=3306 --icinga --check_read_only=false --process\n+    nrpe_command => db-check-health --port=3306 --icinga --check_read_only=true --process\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-mariadb_disk_space]\n\n@@\n-    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"40ee06f86a277811ae30c96774f3ddc0\" --timeout 10 --check-command \"check_mariadb_disk_space\" --page\n+    command => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"d2aae7bfec5549be1a8b9413215bd86c\" --timeout 10 --check-command \"check_mariadb_disk_space\"\n"}, {"resource": "Service[pt-heartbeat-wikimedia]", "parameters": "--- Service[pt-heartbeat-wikimedia].orig\n+++ Service[pt-heartbeat-wikimedia]\n\n@@\n-    ensure => True\n+    ensure => False\n"}, {"resource": "Monitoring::Service[mariadb_disk_space]", "parameters": "--- Monitoring::Service[mariadb_disk_space].orig\n+++ Monitoring::Service[mariadb_disk_space]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "Monitoring::Service[mariadb_read_only_backup1-codfw]", "parameters": "--- Monitoring::Service[mariadb_read_only_backup1-codfw].orig\n+++ Monitoring::Service[mariadb_read_only_backup1-codfw]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "Nrpe::Monitor_service[mysqld]", "parameters": "--- Nrpe::Monitor_service[mysqld].orig\n+++ Nrpe::Monitor_service[mysqld]\n\n@@\n-    critical => True\n+    critical => False\n"}, {"resource": "Mariadb::Monitor_readonly[backup1-codfw]", "parameters": "--- Mariadb::Monitor_readonly[backup1-codfw].orig\n+++ Mariadb::Monitor_readonly[backup1-codfw]\n\n@@\n-    is_critical => True\n+    is_critical => False\n@@\n-    read_only   => False\n+    read_only   => True\n"}], "perc_changed": "3.33%"}}}