{"host": "contint1002.wikimedia.org", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3697, "only_in_self": [], "only_in_other": ["Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.service (nrpe2nodexp-check_jenkins_status.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.timer (nrpe2nodexp-check_jenkins_status.timer)]", "File[/etc/nagios/nrpe.d/check_check_jenkins_status.cfg]", "File[/etc/rsyslog.d/25-nrpe2nodexp-check-jenkins-status.conf]", "File[/etc/sudoers.d/nrpe-check_check_jenkins_status]", "File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.service]", "File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.timer]", "File[/usr/local/lib/nagios/plugins/check_systemd_unit_status]", "File[/var/lib/prometheus/node.d/check_check_jenkins_status.prom]", "Monitoring::Exported_nagios_service[contint1002 check_jenkins_status]", "Monitoring::Service[check_jenkins_status]", "Nrpe::Check[check_check_jenkins_status]", "Nrpe::Monitor_service[check_jenkins_status]", "Nrpe::Plugin[check_systemd_unit_status]", "Prometheus::Alert::Rule[check_check_jenkins_status_0566fc363c3a4922cf93444b3603f43c]", "Rsyslog::Conf[nrpe2nodexp-check_jenkins_status]", "Service[nrpe2nodexp-check_jenkins_status.timer]", "Sudo::User[nrpe-check_check_jenkins_status]", "Systemd::Monitor[jenkins]", "Systemd::Service[nrpe2nodexp-check_jenkins_status]", "Systemd::Timer::Job[nrpe2nodexp-check_jenkins_status]", "Systemd::Timer[nrpe2nodexp-check_jenkins_status]", "Systemd::Unit[nrpe2nodexp-check_jenkins_status.service]", "Systemd::Unit[nrpe2nodexp-check_jenkins_status.timer]"], "resource_diffs": [{"resource": "File[/etc/nagios/nrpe.d/check_check_jenkins_status.cfg]", "content": "--- /etc/nagios/nrpe.d/check_check_jenkins_status.cfg.orig\n+++ /etc/nagios/nrpe.d/check_check_jenkins_status.cfg\n@@ -0,0 +1,2 @@\n+# File generated by puppet. DO NOT edit by hand\n+command[check_check_jenkins_status]=/usr/local/lib/nagios/plugins/check_systemd_unit_status jenkins", "parameters": "--- File[/etc/nagios/nrpe.d/check_check_jenkins_status.cfg].orig\n+++ File[/etc/nagios/nrpe.d/check_check_jenkins_status.cfg]\n\n+    ensure  => present\n+    tag     => nrpe::check\n+    require => Package[nagios-nrpe-server]\n+    owner   => root\n+    notify  => Service[nagios-nrpe-server]\n+    mode    => 0444\n+    group   => root\n"}, {"resource": "Systemd::Timer::Job[nrpe2nodexp-check_jenkins_status]", "parameters": "--- Systemd::Timer::Job[nrpe2nodexp-check_jenkins_status].orig\n+++ Systemd::Timer::Job[nrpe2nodexp-check_jenkins_status]\n\n+    ensure                    => absent\n+    ignore_errors             => True\n+    syslog_identifier         => nrpe2nodexp-check_jenkins_status\n+    syslog_match_startswith   => True\n+    logfile_group             => root\n+    syslog_force_stop         => True\n+    logfile_basedir           => /var/log\n+    monitoring_notes_url      => https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+    interval                  => [{'start': 'OnUnitInactiveSec', 'interval': '5min'}]\n+    description               => execution of nrpe2nodexp for the check_check_jenkins_status command.\n+    user                      => nagios\n+    send_mail_to              => root@contint1002.wikimedia.org\n+    send_mail_only_on_error   => True\n+    logging_enabled           => False\n+    logfile_name              => syslog.log\n+    logfile_perms             => all\n+    environment               => {}\n+    monitoring_contact_groups => admins\n+    splay                     => 300\n+    success_exit_status       => []\n+    send_mail                 => False\n+    private_tmp               => False\n+    monitoring_enabled        => False\n+    fixed_random_delay        => True\n+    command                   => /usr/local/bin/nrpe2nodexp --alert-rule-hash \"0566fc363c3a4922cf93444b3603f43c\" --timeout 10 --check-command \"check_check_jenkins_status\"\n+    group                     => prometheus-node-exporter\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.service (nrpe2nodexp-check_jenkins_status.service)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.service (nrpe2nodexp-check_jenkins_status.service)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.service (nrpe2nodexp-check_jenkins_status.service)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Sudo::User[nrpe-check_check_jenkins_status]", "parameters": "--- Sudo::User[nrpe-check_check_jenkins_status].orig\n+++ Sudo::User[nrpe-check_check_jenkins_status]\n\n+    ensure     => absent\n+    require    => ['Class[Sudo]']\n+    tag        => nrpe::check\n+    user       => nagios\n+    privileges => []\n"}, {"resource": "Systemd::Service[jenkins]", "parameters": "--- Systemd::Service[jenkins].orig\n+++ Systemd::Service[jenkins]\n\n+    monitoring_notes_url => https://wikitech.wikimedia.org/wiki/Jenkins#How_to\n@@\n-    monitoring_enabled   => False\n+    monitoring_enabled   => True\n"}, {"resource": "Service[nrpe2nodexp-check_jenkins_status.timer]", "parameters": "--- Service[nrpe2nodexp-check_jenkins_status.timer].orig\n+++ Service[nrpe2nodexp-check_jenkins_status.timer]\n\n+    ensure   => stopped\n+    enable   => False\n+    before   => ['Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.timer (nrpe2nodexp-check_jenkins_status.timer)]']\n+    provider => systemd\n"}, {"resource": "Prometheus::Alert::Rule[check_check_jenkins_status_0566fc363c3a4922cf93444b3603f43c]", "parameters": "--- Prometheus::Alert::Rule[check_check_jenkins_status_0566fc363c3a4922cf93444b3603f43c].orig\n+++ Prometheus::Alert::Rule[check_check_jenkins_status_0566fc363c3a4922cf93444b3603f43c]\n\n+    ensure             => absent\n+    for                => 11m\n+    runbook            => https://wikitech.wikimedia.org/wiki/Jenkins#How_to\n+    logs               => https://logstash.wikimedia.org/app/dashboards#/view/2d343ac0-6df8-11f0-8e08-7fab0da52b33?_g=(filters:!((query:(match_phrase:(event.module:check_check_jenkins_status))),(query:(match_phrase:(host.name:{{$labels.instance|stripPort}})))))\n+    alert_name         => nrpe_Check_unit_status_of_jenkins\n+    summary            => NRPE CHECK: Check unit status of jenkins\n+    group              => nrpechecks\n+    severity           => info\n+    site               => eqiad\n+    def_label_whitelst => ['team', 'severity']\n+    description        => NRPE CHECK: Check unit status of jenkins\n+    dashboard          => TODO\n+    team               => observability\n+    instance           => ops\n+    expr               => (nagios_nrpe_check_result{alert_rule_hash=\"0566fc363c3a4922cf93444b3603f43c\",check_name=\"check_check_jenkins_status\", status=~\"(WARNING|CRITICAL)\", severity=~\"(warning|critical)\"} > 0) * on (instance) group_left (team) role_owner\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.timer]", "content": "--- /lib/systemd/system/nrpe2nodexp-check_jenkins_status.timer.orig\n+++ /lib/systemd/system/nrpe2nodexp-check_jenkins_status.timer\n@@ -0,0 +1,14 @@\n+[Unit]\n+Description=Periodic execution of nrpe2nodexp-check_jenkins_status.service\n+\n+[Timer]\n+Unit=nrpe2nodexp-check_jenkins_status.service\n+# Accuracy sets the maximum time interval around the execution time we want to allow\n+AccuracySec=15sec\n+OnUnitInactiveSec=5min\n+OnActiveSec=1s\n+RandomizedDelaySec=300\n+FixedRandomDelay=true\n+\n+[Install]\n+WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.timer].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.timer]\n\n+    ensure => absent\n+    owner  => root\n+    notify => Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.timer (nrpe2nodexp-check_jenkins_status.timer)]\n+    mode   => 0444\n+    group  => root\n"}, {"resource": "Systemd::Monitor[jenkins]", "parameters": "--- Systemd::Monitor[jenkins].orig\n+++ Systemd::Monitor[jenkins]\n\n+    ensure         => present\n+    contact_group  => admins\n+    migration_task => T407130\n+    check_interval => 10\n+    notes_url      => https://wikitech.wikimedia.org/wiki/Jenkins#How_to\n+    critical       => False\n+    retries        => 2\n"}, {"resource": "File[/etc/rsyslog.d/25-nrpe2nodexp-check-jenkins-status.conf]", "content": "--- /etc/rsyslog.d/25-nrpe2nodexp-check-jenkins-status.conf.orig\n+++ /etc/rsyslog.d/25-nrpe2nodexp-check-jenkins-status.conf\n@@ -0,0 +1,10 @@\n+# SPDX-License-Identifier: Apache-2.0\n+if $programname contains \"nrpe2nodexp-check_jenkins_status\" then {\n+    if ($msg contains \"\\\"ecs.version\\\": \\\"1.7.0\\\"\") then {\n+        # Send logs to kafka\n+        set $.log_outputs = \"kafka ecs_170 local\";\n+    } else {\n+        # Filter out non-relevant nrpe2nodexp messages\n+        stop\n+    }\n+}", "parameters": "--- File[/etc/rsyslog.d/25-nrpe2nodexp-check-jenkins-status.conf].orig\n+++ File[/etc/rsyslog.d/25-nrpe2nodexp-check-jenkins-status.conf]\n\n+    ensure => absent\n+    owner  => root\n+    notify => Service[rsyslog]\n+    mode   => 0444\n+    group  => root\n"}, {"resource": "File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.service]", "content": "--- /lib/systemd/system/nrpe2nodexp-check_jenkins_status.service.orig\n+++ /lib/systemd/system/nrpe2nodexp-check_jenkins_status.service\n@@ -0,0 +1,11 @@\n+[Unit]\n+Description=execution of nrpe2nodexp for the check_check_jenkins_status command.\n+Documentation=https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+\n+[Service]\n+Type=oneshot\n+User=nagios\n+\n+Group=prometheus-node-exporter\n+SyslogIdentifier=nrpe2nodexp-check_jenkins_status\n+ExecStart=-/usr/local/bin/nrpe2nodexp --alert-rule-hash \"0566fc363c3a4922cf93444b3603f43c\" --timeout 10 --check-command \"check_check_jenkins_status\"", "parameters": "--- File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.service].orig\n+++ File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.service]\n\n+    ensure => absent\n+    owner  => root\n+    notify => Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.service (nrpe2nodexp-check_jenkins_status.service)]\n+    mode   => 0444\n+    group  => root\n"}, {"resource": "Monitoring::Service[check_jenkins_status]", "parameters": "--- Monitoring::Service[check_jenkins_status].orig\n+++ Monitoring::Service[check_jenkins_status]\n\n+    ensure         => present\n+    retry_interval => 1\n+    freshness      => 36000\n+    check_interval => 10\n+    config_dir     => /etc/nagios\n+    critical       => False\n+    passive        => False\n+    contact_group  => admins\n+    check_command  => nrpe_check!check_check_jenkins_status!10\n+    description    => Check unit status of jenkins\n+    migration_task => T407130\n+    host           => contint1002\n+    notes_url      => https://wikitech.wikimedia.org/wiki/Jenkins#How_to\n+    retries        => 2\n"}, {"resource": "Nrpe::Check[check_check_jenkins_status]", "parameters": "--- Nrpe::Check[check_check_jenkins_status].orig\n+++ Nrpe::Check[check_check_jenkins_status]\n\n+    ensure  => present\n+    command => /usr/local/lib/nagios/plugins/check_systemd_unit_status jenkins\n+    before  => Monitoring::Service[check_jenkins_status]\n"}, {"resource": "Monitoring::Exported_nagios_service[contint1002 check_jenkins_status]", "parameters": "--- Monitoring::Exported_nagios_service[contint1002 check_jenkins_status].orig\n+++ Monitoring::Exported_nagios_service[contint1002 check_jenkins_status]\n\n+    ensure                 => present\n+    host_name              => contint1002\n+    retry_interval         => 1\n+    passive_checks_enabled => 1\n+    servicegroups          => ci_eqiad\n+    is_volatile            => 0\n+    service_description    => Check unit status of jenkins\n+    notification_period    => 24x7\n+    notification_interval  => 0\n+    active_checks_enabled  => 1\n+    check_command          => nrpe_check!check_check_jenkins_status!10\n+    notifications_enabled  => 1\n+    check_freshness        => 0\n+    check_interval         => 10\n+    contact_groups         => admins\n+    notification_options   => c,r,f\n+    max_check_attempts     => 2\n+    check_period           => 24x7\n+    notes_url              => https://wikitech.wikimedia.org/wiki/Jenkins#How_to\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-check_jenkins_status.service]", "parameters": "--- Systemd::Unit[nrpe2nodexp-check_jenkins_status.service].orig\n+++ Systemd::Unit[nrpe2nodexp-check_jenkins_status.service]\n\n+    ensure            => absent\n+    require           => ['Class[Systemd]']\n+    override          => False\n+    override_filename => puppet-override.conf\n+    restart           => False\n+    unit              => nrpe2nodexp-check_jenkins_status.service\n"}, {"resource": "Systemd::Unit[nrpe2nodexp-check_jenkins_status.timer]", "parameters": "--- Systemd::Unit[nrpe2nodexp-check_jenkins_status.timer].orig\n+++ Systemd::Unit[nrpe2nodexp-check_jenkins_status.timer]\n\n+    ensure            => absent\n+    require           => ['Class[Systemd]']\n+    override          => False\n+    override_filename => puppet-override.conf\n+    restart           => False\n+    unit              => nrpe2nodexp-check_jenkins_status.timer\n"}, {"resource": "Systemd::Timer[nrpe2nodexp-check_jenkins_status]", "parameters": "--- Systemd::Timer[nrpe2nodexp-check_jenkins_status].orig\n+++ Systemd::Timer[nrpe2nodexp-check_jenkins_status]\n\n+    ensure             => absent\n+    splay              => 300\n+    unit_name          => nrpe2nodexp-check_jenkins_status.service\n+    timer_intervals    => [{'start': 'OnUnitInactiveSec', 'interval': '5min'}, {'interval': '1s', 'start': 'OnActiveSec'}]\n+    fixed_random_delay => True\n+    accuracy           => 15sec\n"}, {"resource": "Rsyslog::Conf[nrpe2nodexp-check_jenkins_status]", "parameters": "--- Rsyslog::Conf[nrpe2nodexp-check_jenkins_status].orig\n+++ Rsyslog::Conf[nrpe2nodexp-check_jenkins_status]\n\n+    ensure   => absent\n+    mode     => 0444\n+    priority => 25\n"}, {"resource": "Nrpe::Monitor_service[check_jenkins_status]", "parameters": "--- Nrpe::Monitor_service[check_jenkins_status].orig\n+++ Nrpe::Monitor_service[check_jenkins_status]\n\n+    ensure                         => present\n+    retry_interval                 => 1\n+    contact_group                  => admins\n+    description                    => Check unit status of jenkins\n+    enable_icinga_check            => True\n+    alertmanager_team              => observability\n+    alertmanager_severity_override => info\n+    check_interval                 => 10\n+    critical                       => False\n+    enable_nrpe2nodexp             => False\n+    timeout                        => 10\n+    nrpe2nodexp_parse_perf_data    => False\n+    migration_task                 => T407130\n+    nrpe_command                   => /usr/local/lib/nagios/plugins/check_systemd_unit_status jenkins\n+    notes_url                      => https://wikitech.wikimedia.org/wiki/Jenkins#How_to\n+    retries                        => 2\n"}, {"resource": "Systemd::Service[nrpe2nodexp-check_jenkins_status]", "parameters": "--- Systemd::Service[nrpe2nodexp-check_jenkins_status].orig\n+++ Systemd::Service[nrpe2nodexp-check_jenkins_status]\n\n+    ensure                   => absent\n+    monitoring_contact_group => admins\n+    override                 => False\n+    monitoring_critical      => False\n+    restart                  => False\n+    service_params           => {}\n+    require                  => Systemd::Unit[nrpe2nodexp-check_jenkins_status.service]\n+    unit_type                => timer\n+    monitoring_enabled       => False\n+    migration_task           => T407130\n"}, {"resource": "File[/etc/sudoers.d/nrpe-check_check_jenkins_status]", "parameters": "--- File[/etc/sudoers.d/nrpe-check_check_jenkins_status].orig\n+++ File[/etc/sudoers.d/nrpe-check_check_jenkins_status]\n\n+    ensure  => absent\n+    owner   => root\n+    require => Package[nagios-nrpe-server]\n+    group   => root\n"}, {"resource": "Class[Jenkins]", "parameters": "--- Class[Jenkins].orig\n+++ Class[Jenkins]\n\n+    monitoring_notes_url => https://wikitech.wikimedia.org/wiki/Jenkins#How_to\n+    monitoring_enabled   => True\n"}, {"resource": "Nrpe::Plugin[check_systemd_unit_status]", "parameters": "--- Nrpe::Plugin[check_systemd_unit_status].orig\n+++ Nrpe::Plugin[check_systemd_unit_status]\n\n+    ensure => present\n+    source => puppet:///modules/systemd/check_systemd_unit_status\n"}, {"resource": "File[/usr/local/lib/nagios/plugins/check_systemd_unit_status]", "parameters": "--- File[/usr/local/lib/nagios/plugins/check_systemd_unit_status].orig\n+++ File[/usr/local/lib/nagios/plugins/check_systemd_unit_status]\n\n+    ensure  => file\n+    tag     => nrpe::plugin\n+    require => File[/usr/local/lib/nagios/plugins/]\n+    source  => puppet:///modules/systemd/check_systemd_unit_status\n+    owner   => root\n+    mode    => 0555\n+    group   => root\n"}, {"resource": "File[/var/lib/prometheus/node.d/check_check_jenkins_status.prom]", "parameters": "--- File[/var/lib/prometheus/node.d/check_check_jenkins_status.prom].orig\n+++ File[/var/lib/prometheus/node.d/check_check_jenkins_status.prom]\n\n+    ensure => absent\n+    owner  => root\n+    group  => root\n"}, {"resource": "Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.timer (nrpe2nodexp-check_jenkins_status.timer)]", "parameters": "--- Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.timer (nrpe2nodexp-check_jenkins_status.timer)].orig\n+++ Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.timer (nrpe2nodexp-check_jenkins_status.timer)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}], "perc_changed": "1.35%"}, "core": {"total": 3697, "only_in_self": [], "only_in_other": ["Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.service (nrpe2nodexp-check_jenkins_status.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.timer (nrpe2nodexp-check_jenkins_status.timer)]", "File[/etc/nagios/nrpe.d/check_check_jenkins_status.cfg]", "File[/etc/rsyslog.d/25-nrpe2nodexp-check-jenkins-status.conf]", "File[/etc/sudoers.d/nrpe-check_check_jenkins_status]", "File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.service]", "File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.timer]", "File[/usr/local/lib/nagios/plugins/check_systemd_unit_status]", "File[/var/lib/prometheus/node.d/check_check_jenkins_status.prom]", "Service[nrpe2nodexp-check_jenkins_status.timer]"], "resource_diffs": [], "perc_changed": "0.27%"}, "main": {"total": 3697, "only_in_self": [], "only_in_other": ["Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.service (nrpe2nodexp-check_jenkins_status.service)]", "Exec[systemd daemon-reload for nrpe2nodexp-check_jenkins_status.timer (nrpe2nodexp-check_jenkins_status.timer)]", "File[/etc/nagios/nrpe.d/check_check_jenkins_status.cfg]", "File[/etc/rsyslog.d/25-nrpe2nodexp-check-jenkins-status.conf]", "File[/etc/sudoers.d/nrpe-check_check_jenkins_status]", "File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.service]", "File[/lib/systemd/system/nrpe2nodexp-check_jenkins_status.timer]", "File[/usr/local/lib/nagios/plugins/check_systemd_unit_status]", "File[/var/lib/prometheus/node.d/check_check_jenkins_status.prom]", "Monitoring::Exported_nagios_service[contint1002 check_jenkins_status]", "Monitoring::Service[check_jenkins_status]", "Nrpe::Check[check_check_jenkins_status]", "Nrpe::Monitor_service[check_jenkins_status]", "Nrpe::Plugin[check_systemd_unit_status]", "Prometheus::Alert::Rule[check_check_jenkins_status_0566fc363c3a4922cf93444b3603f43c]", "Rsyslog::Conf[nrpe2nodexp-check_jenkins_status]", "Service[nrpe2nodexp-check_jenkins_status.timer]", "Sudo::User[nrpe-check_check_jenkins_status]", "Systemd::Monitor[jenkins]", "Systemd::Service[nrpe2nodexp-check_jenkins_status]", "Systemd::Timer::Job[nrpe2nodexp-check_jenkins_status]", "Systemd::Timer[nrpe2nodexp-check_jenkins_status]", "Systemd::Unit[nrpe2nodexp-check_jenkins_status.service]", "Systemd::Unit[nrpe2nodexp-check_jenkins_status.timer]"], "resource_diffs": [{"resource": "Systemd::Service[jenkins]", "parameters": "--- Systemd::Service[jenkins].orig\n+++ Systemd::Service[jenkins]\n\n+    monitoring_notes_url => https://wikitech.wikimedia.org/wiki/Jenkins#How_to\n@@\n-    monitoring_enabled   => False\n+    monitoring_enabled   => True\n"}, {"resource": "Class[Jenkins]", "parameters": "--- Class[Jenkins].orig\n+++ Class[Jenkins]\n\n+    monitoring_notes_url => https://wikitech.wikimedia.org/wiki/Jenkins#How_to\n+    monitoring_enabled   => True\n"}], "perc_changed": "0.70%"}}}