{"host": "titan2002.codfw.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3684, "only_in_self": [], "only_in_other": ["Class[Thanos::Compact::Prometheus]", "Exec[orchestrated restart needed]", "File[/etc/systemd/system/thanos-compact.service.d]", "File[/etc/thanos-compact/relabel.yaml.unreferenced]"], "resource_diffs": [{"resource": "File[/etc/systemd/system/thanos-compact.service.d]", "parameters": "--- File[/etc/systemd/system/thanos-compact.service.d].orig\n+++ File[/etc/systemd/system/thanos-compact.service.d]\n\n+    ensure => directory\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "Systemd::Unit[thanos-compact]", "parameters": "--- Systemd::Unit[thanos-compact].orig\n+++ Systemd::Unit[thanos-compact]\n\n@@\n-    ensure => absent\n+    ensure => present\n"}, {"resource": "Class[Profile::Thanos::Compact]", "parameters": "--- Class[Profile::Thanos::Compact].orig\n+++ Class[Profile::Thanos::Compact]\n\n-    thanos_compact_host => titan2001.codfw.wmnet\n"}, {"resource": "Service[thanos-compact]", "parameters": "--- Service[thanos-compact].orig\n+++ Service[thanos-compact]\n\n-    before => ['Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)]']\n@@\n-    ensure => stopped\n+    ensure => running\n@@\n-    enable => False\n+    enable => True\n"}, {"resource": "Systemd::Service[thanos-compact]", "parameters": "--- Systemd::Service[thanos-compact].orig\n+++ Systemd::Service[thanos-compact]\n\n@@\n-    ensure         => absent\n+    ensure         => present\n@@\n-    service_params => {'enable': False, 'hasrestart': True}\n+    service_params => {'enable': True, 'hasrestart': True}\n"}, {"resource": "File[/etc/systemd/system/thanos-compact.service.d/puppet-override.conf]", "content": "--- /etc/systemd/system/thanos-compact.service.d/puppet-override.conf.orig\n+++ /etc/systemd/system/thanos-compact.service.d/puppet-override.conf\n@@ -1,7 +1,15 @@\n [Service]\n+\n+# Local restart is blocked if a new relabel-config-file is pending deployment via the cookbook.\n+# In this case, the restart must be performed using the sre.o11y.thanos-compact-restart cookbook.\n+\n Slice=thanos.slice\n # force sigkill due to https://github.com/thanos-io/thanos/issues/6398\n SendSIGKILL=yes\n+ExecReload=\n+ExecReload=/bin/bash -c \"/usr/bin/cmp -s /etc/thanos-compact/relabel.yaml.unreferenced /etc/thanos-compact/relabel.yaml && /bin/kill -HUP $MAINPID || (echo 'New relabel config file pending deployment. Restart must be performed using the sre.o11y.thanos-compact-restart cookbook.'; exit 1)\"\n+ExecStartPre=\n+ExecStartPre=/bin/bash -c \"/usr/bin/cmp -s /etc/thanos-compact/relabel.yaml.unreferenced /etc/thanos-compact/relabel.yaml || (echo 'New relabel config file pending deployment. Restart must be performed using the sre.o11y.thanos-compact-restart cookbook.'; exit 1)\"\n ExecStart=\n ExecStart=/usr/bin/thanos compact \\\n   --http-address 0.0.0.0:12902 \\\n@@ -15,4 +23,5 @@\n   --compact.concurrency 2 \\\n   --downsample.concurrency 2 \\\n   --block-meta-fetch-concurrency 96 \\\n+  --selector.relabel-config-file /etc/thanos-compact/relabel.yaml \\\n   --wait", "parameters": "--- File[/etc/systemd/system/thanos-compact.service.d/puppet-override.conf].orig\n+++ File[/etc/systemd/system/thanos-compact.service.d/puppet-override.conf]\n\n@@\n-    ensure => absent\n+    ensure => present\n"}, {"resource": "File[/etc/thanos-compact/relabel.yaml.unreferenced]", "content": "--- /etc/thanos-compact/relabel.yaml.unreferenced.orig\n+++ /etc/thanos-compact/relabel.yaml.unreferenced\n@@ -0,0 +1,30 @@\n+# Note: \"a^\" is a regex that matches nothing,\n+# as \"a\" cannot be followed by the start of the string \"^\".\n+#\n+# regex: \"^(a^|)$\"\n+#   Used by \"ruler blocks designated compactor\" when there are no owned instances,\n+#   to match only Thanos Ruler generated blocks (i.e: without the prometheus label).\n+#   The trailing '|' before the final ')' is used to match an empty string\n+#   (i.e., a missing prometheus label).\n+#\n+# regex: \"^(a^)$\"\n+#   Used by non \"ruler blocks designated compactor\" when there are no owned instances,\n+#   to match nothing, as the compactor should not receive any blocks.\n+#\n+# regex: \"^(instance_1|instance_2|...|)$\"\n+#   Used by \"ruler blocks designated compactor\" when there are owned instances.\n+#   The regex matches both the owned instances and Thanos Ruler generated blocks,\n+#   as the compactor needs to receive all blocks to be able to compact them.\n+#   The trailing '|' before the final ')' is used to match an empty string\n+#   (i.e., a missing prometheus label).\n+#\n+# regex: \"^(instance_1|instance_2|...)$\"\n+#   Used by non \"ruler blocks designated compactor\" when there are owned instances.\n+#   The regex matches only the owned instances, as the compactor should not receive\n+#   Thanos Ruler generated blocks.\n+\n+- action: keep\n+  source_labels:\n+    - prometheus\n+    regex: \"^(k8s-aux|k8s-dse|k8s-mlserve|k8s-mlstaging|k8s-staging)$\"\n+  ", "parameters": "--- File[/etc/thanos-compact/relabel.yaml.unreferenced].orig\n+++ File[/etc/thanos-compact/relabel.yaml.unreferenced]\n\n+    ensure => file\n+    group  => root\n+    mode   => 0550\n+    owner  => thanos\n"}, {"resource": "Class[Thanos::Compact]", "parameters": "--- Class[Thanos::Compact].orig\n+++ Class[Thanos::Compact]\n\n-    run_on_host => titan2001.codfw.wmnet\n"}, {"resource": "Exec[orchestrated restart needed]", "parameters": "--- Exec[orchestrated restart needed].orig\n+++ Exec[orchestrated restart needed]\n\n+    path    => ['/bin', '/usr/bin']\n+    command => /bin/sh -c 'cmp -s \"/etc/thanos-compact/relabel.yaml\" \"/etc/thanos-compact/relabel.yaml.unreferenced\" || (echo \"Please run the sre.o11y.thanos-compact-restart cookbook\" >&2; exit 1)'\n+    require => ['File[/etc/thanos-compact/relabel.yaml.unreferenced]', 'File[/etc/thanos-compact/objstore.yaml]']\n"}, {"resource": "Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)]", "parameters": "--- Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)].orig\n+++ Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)]\n\n+    notify => ['Service[thanos-compact]']\n"}], "perc_changed": "0.38%"}, "core": {"total": 3684, "only_in_self": [], "only_in_other": ["Exec[orchestrated restart needed]", "File[/etc/systemd/system/thanos-compact.service.d]", "File[/etc/thanos-compact/relabel.yaml.unreferenced]"], "resource_diffs": [{"resource": "Service[thanos-compact]", "parameters": "--- Service[thanos-compact].orig\n+++ Service[thanos-compact]\n\n-    before => ['Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)]']\n@@\n-    ensure => stopped\n+    ensure => running\n@@\n-    enable => False\n+    enable => True\n"}, {"resource": "File[/etc/systemd/system/thanos-compact.service.d/puppet-override.conf]", "content": "--- /etc/systemd/system/thanos-compact.service.d/puppet-override.conf.orig\n+++ /etc/systemd/system/thanos-compact.service.d/puppet-override.conf\n@@ -1,7 +1,15 @@\n [Service]\n+\n+# Local restart is blocked if a new relabel-config-file is pending deployment via the cookbook.\n+# In this case, the restart must be performed using the sre.o11y.thanos-compact-restart cookbook.\n+\n Slice=thanos.slice\n # force sigkill due to https://github.com/thanos-io/thanos/issues/6398\n SendSIGKILL=yes\n+ExecReload=\n+ExecReload=/bin/bash -c \"/usr/bin/cmp -s /etc/thanos-compact/relabel.yaml.unreferenced /etc/thanos-compact/relabel.yaml && /bin/kill -HUP $MAINPID || (echo 'New relabel config file pending deployment. Restart must be performed using the sre.o11y.thanos-compact-restart cookbook.'; exit 1)\"\n+ExecStartPre=\n+ExecStartPre=/bin/bash -c \"/usr/bin/cmp -s /etc/thanos-compact/relabel.yaml.unreferenced /etc/thanos-compact/relabel.yaml || (echo 'New relabel config file pending deployment. Restart must be performed using the sre.o11y.thanos-compact-restart cookbook.'; exit 1)\"\n ExecStart=\n ExecStart=/usr/bin/thanos compact \\\n   --http-address 0.0.0.0:12902 \\\n@@ -15,4 +23,5 @@\n   --compact.concurrency 2 \\\n   --downsample.concurrency 2 \\\n   --block-meta-fetch-concurrency 96 \\\n+  --selector.relabel-config-file /etc/thanos-compact/relabel.yaml \\\n   --wait", "parameters": "--- File[/etc/systemd/system/thanos-compact.service.d/puppet-override.conf].orig\n+++ File[/etc/systemd/system/thanos-compact.service.d/puppet-override.conf]\n\n@@\n-    ensure => absent\n+    ensure => present\n"}, {"resource": "Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)]", "parameters": "--- Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)].orig\n+++ Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)]\n\n+    notify => ['Service[thanos-compact]']\n"}], "perc_changed": "0.16%"}, "main": {"total": 3684, "only_in_self": [], "only_in_other": ["Class[Thanos::Compact::Prometheus]", "Exec[orchestrated restart needed]", "File[/etc/systemd/system/thanos-compact.service.d]", "File[/etc/thanos-compact/relabel.yaml.unreferenced]"], "resource_diffs": [{"resource": "Service[thanos-compact]", "parameters": "--- Service[thanos-compact].orig\n+++ Service[thanos-compact]\n\n-    before => ['Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)]']\n@@\n-    ensure => stopped\n+    ensure => running\n@@\n-    enable => False\n+    enable => True\n"}, {"resource": "Systemd::Service[thanos-compact]", "parameters": "--- Systemd::Service[thanos-compact].orig\n+++ Systemd::Service[thanos-compact]\n\n@@\n-    ensure         => absent\n+    ensure         => present\n@@\n-    service_params => {'enable': False, 'hasrestart': True}\n+    service_params => {'enable': True, 'hasrestart': True}\n"}, {"resource": "File[/etc/systemd/system/thanos-compact.service.d/puppet-override.conf]", "content": "--- /etc/systemd/system/thanos-compact.service.d/puppet-override.conf.orig\n+++ /etc/systemd/system/thanos-compact.service.d/puppet-override.conf\n@@ -1,7 +1,15 @@\n [Service]\n+\n+# Local restart is blocked if a new relabel-config-file is pending deployment via the cookbook.\n+# In this case, the restart must be performed using the sre.o11y.thanos-compact-restart cookbook.\n+\n Slice=thanos.slice\n # force sigkill due to https://github.com/thanos-io/thanos/issues/6398\n SendSIGKILL=yes\n+ExecReload=\n+ExecReload=/bin/bash -c \"/usr/bin/cmp -s /etc/thanos-compact/relabel.yaml.unreferenced /etc/thanos-compact/relabel.yaml && /bin/kill -HUP $MAINPID || (echo 'New relabel config file pending deployment. Restart must be performed using the sre.o11y.thanos-compact-restart cookbook.'; exit 1)\"\n+ExecStartPre=\n+ExecStartPre=/bin/bash -c \"/usr/bin/cmp -s /etc/thanos-compact/relabel.yaml.unreferenced /etc/thanos-compact/relabel.yaml || (echo 'New relabel config file pending deployment. Restart must be performed using the sre.o11y.thanos-compact-restart cookbook.'; exit 1)\"\n ExecStart=\n ExecStart=/usr/bin/thanos compact \\\n   --http-address 0.0.0.0:12902 \\\n@@ -15,4 +23,5 @@\n   --compact.concurrency 2 \\\n   --downsample.concurrency 2 \\\n   --block-meta-fetch-concurrency 96 \\\n+  --selector.relabel-config-file /etc/thanos-compact/relabel.yaml \\\n   --wait", "parameters": "--- File[/etc/systemd/system/thanos-compact.service.d/puppet-override.conf].orig\n+++ File[/etc/systemd/system/thanos-compact.service.d/puppet-override.conf]\n\n@@\n-    ensure => absent\n+    ensure => present\n"}, {"resource": "Class[Thanos::Compact]", "parameters": "--- Class[Thanos::Compact].orig\n+++ Class[Thanos::Compact]\n\n-    run_on_host => titan2001.codfw.wmnet\n"}, {"resource": "Systemd::Unit[thanos-compact]", "parameters": "--- Systemd::Unit[thanos-compact].orig\n+++ Systemd::Unit[thanos-compact]\n\n@@\n-    ensure => absent\n+    ensure => present\n"}, {"resource": "Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)]", "parameters": "--- Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)].orig\n+++ Exec[systemd daemon-reload for thanos-compact.service (thanos-compact)]\n\n+    notify => ['Service[thanos-compact]']\n"}, {"resource": "Class[Profile::Thanos::Compact]", "parameters": "--- Class[Profile::Thanos::Compact].orig\n+++ Class[Profile::Thanos::Compact]\n\n-    thanos_compact_host => titan2001.codfw.wmnet\n"}], "perc_changed": "0.30%"}}}