{"host": "cloudelastic1012.eqiad.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3722, "only_in_self": ["Conftool::Scripts::Safe_service_restart[nginx]", "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad]", "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad]", "File[/usr/local/bin/depool-nginx]", "File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/bin/pool-nginx]", "File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/sbin/restart-nginx]", "File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]", "File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]"], "only_in_other": ["Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-chi-eqiad.service]", "Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-omega-eqiad.service]", "Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-psi-eqiad.service]", "File[/usr/local/bin/depool-opensearch_2@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/depool-opensearch_2@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_2@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service]"], "resource_diffs": [{"resource": "Conftool::Scripts::Safe_service_restart[nginx]", "parameters": "--- Conftool::Scripts::Safe_service_restart[nginx].orig\n+++ Conftool::Scripts::Safe_service_restart[nginx]\n\n-    require         => ['Class[Conftool::Scripts]']\n-    max_concurrency => 5\n-    lvs_pools       => ['search-https', 'search-omega-https']\n"}, {"resource": "Systemd::Timer::Job[prometheus_lvs_realserver_mss]", "parameters": "--- Systemd::Timer::Job[prometheus_lvs_realserver_mss].orig\n+++ Systemd::Timer::Job[prometheus_lvs_realserver_mss]\n\n@@\n-    command => /usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+    command => /usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n"}, {"resource": "Class[Profile::Lvs::Realserver]", "parameters": "--- Class[Profile::Lvs::Realserver].orig\n+++ Class[Profile::Lvs::Realserver]\n\n@@\n-    pools => {'search': {'services': ['opensearch_2@production-search-eqiad']}, 'search-https': {'services': ['nginx', 'opensearch_2@production-search-eqiad']}, 'search-omega-https': {'services': ['nginx', 'opensearch_2@production-search-omega-eqiad']}}\n+    pools => {'cloudelastic-chi-https': {'services': ['opensearch_2@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_2@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_2@cloudelastic-omega-eqiad.service']}}\n"}, {"resource": "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service]", "content": "--- /usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service.orig\n+++ /usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --services opensearch_2@cloudelastic-omega-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service].orig\n+++ File[/usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service]\n\n+    ensure => present\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]", "content": "--- /usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad.orig\n+++ /usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools search-omega-https --services opensearch_2@production-search-omega-eqiad --retries 10 --wait 5 --max-concurrency 5 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad].orig\n+++ File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]\n\n-    ensure => present\n-    group  => root\n-    mode   => 0555\n-    owner  => root\n"}, {"resource": "File[/usr/local/bin/depool-opensearch_2@cloudelastic-chi-eqiad.service]", "content": "--- /usr/local/bin/depool-opensearch_2@cloudelastic-chi-eqiad.service.orig\n+++ /usr/local/bin/depool-opensearch_2@cloudelastic-chi-eqiad.service\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools cloudelastic-chi-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_2@cloudelastic-chi-eqiad.service].orig\n+++ File[/usr/local/bin/depool-opensearch_2@cloudelastic-chi-eqiad.service]\n\n+    ensure => present\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv6]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv6.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv6\n@@ -5,7 +5,7 @@\n domain (ip6) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n \t\t}\n \t}\n }"}, {"resource": "File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]", "content": "--- /usr/local/bin/pool-opensearch_2@production-search-eqiad.orig\n+++ /usr/local/bin/pool-opensearch_2@production-search-eqiad\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools search search-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_2@production-search-eqiad].orig\n+++ File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]\n\n-    ensure => present\n-    group  => root\n-    mode   => 0555\n-    owner  => root\n"}, {"resource": "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service]", "content": "--- /usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service.orig\n+++ /usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools cloudelastic-chi-https --services opensearch_2@cloudelastic-chi-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service].orig\n+++ File[/usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service]\n\n+    ensure => present\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "File[/usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service]", "content": "--- /usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service.orig\n+++ /usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service].orig\n+++ File[/usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service]\n\n+    ensure => present\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "Ferm::Rule[clamp-mss-ipv4]", "parameters": "--- Ferm::Rule[clamp-mss-ipv4].orig\n+++ Ferm::Rule[clamp-mss-ipv4]\n\n@@\n-    rule => outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+    rule => outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n"}, {"resource": "Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-omega-eqiad.service]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-omega-eqiad.service].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-omega-eqiad.service]\n\n+    require         => ['Class[Conftool::Scripts]']\n+    max_concurrency => 1\n+    lvs_pools       => ['cloudelastic-omega-https']\n"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -148,11 +148,11 @@\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n # Stop allocating shards to nodes (default 80%)\n-cluster.routing.allocation.disk.watermark.low: 0.75\n+cluster.routing.allocation.disk.watermark.low: 100gb\n # Begin evicting shards off the node (default 90%)\n-cluster.routing.allocation.disk.watermark.high: 0.80\n+cluster.routing.allocation.disk.watermark.high: 50gb\n # Set indexes read-only with allocated shards on affected node (default 95%)\n-cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+cluster.routing.allocation.disk.watermark.flood_stage: 10gb\n \n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s"}, {"resource": "Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-chi-eqiad.service]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-chi-eqiad.service].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-chi-eqiad.service]\n\n+    require         => ['Class[Conftool::Scripts]']\n+    max_concurrency => 1\n+    lvs_pools       => ['cloudelastic-chi-https']\n"}, {"resource": "Class[Lvs::Realserver]", "parameters": "--- Class[Lvs::Realserver].orig\n+++ Class[Lvs::Realserver]\n\n@@\n-    realserver_ips => ['10.2.2.30']\n+    realserver_ips => ['208.80.154.241', '2620:0:861:ed1a::3:241']\n"}, {"resource": "Systemd::Timer::Job[prometheus_ferm_mss]", "parameters": "--- Systemd::Timer::Job[prometheus_ferm_mss].orig\n+++ Systemd::Timer::Job[prometheus_ferm_mss]\n\n@@\n-    command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+    command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv4]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv4.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv4\n@@ -5,7 +5,7 @@\n domain (ip) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n \t\t}\n \t}\n }"}, {"resource": "File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]", "content": "--- /usr/local/bin/pool-opensearch_2@production-search-omega-eqiad.orig\n+++ /usr/local/bin/pool-opensearch_2@production-search-omega-eqiad\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools search-omega-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad].orig\n+++ File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]\n\n-    ensure => present\n-    group  => root\n-    mode   => 0555\n-    owner  => root\n"}, {"resource": "File[/usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service]", "content": "--- /usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service.orig\n+++ /usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service].orig\n+++ File[/usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service]\n\n+    ensure => present\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad]\n\n-    require         => ['Class[Conftool::Scripts]']\n-    max_concurrency => 5\n-    lvs_pools       => ['search', 'search-https']\n"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -148,11 +148,11 @@\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n # Stop allocating shards to nodes (default 80%)\n-cluster.routing.allocation.disk.watermark.low: 0.75\n+cluster.routing.allocation.disk.watermark.low: 100gb\n # Begin evicting shards off the node (default 90%)\n-cluster.routing.allocation.disk.watermark.high: 0.80\n+cluster.routing.allocation.disk.watermark.high: 50gb\n # Set indexes read-only with allocated shards on affected node (default 95%)\n-cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+cluster.routing.allocation.disk.watermark.flood_stage: 10gb\n \n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s"}, {"resource": "File[/usr/local/bin/pool-nginx]", "content": "--- /usr/local/bin/pool-nginx.orig\n+++ /usr/local/bin/pool-nginx\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools search-https search-omega-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-nginx].orig\n+++ File[/usr/local/bin/pool-nginx]\n\n-    owner  => root\n-    mode   => 0555\n-    before => ['Service[nginx]']\n-    ensure => present\n-    group  => root\n"}, {"resource": "File[/lib/systemd/system/prometheus_ferm_mss.service]", "content": "--- /lib/systemd/system/prometheus_ferm_mss.service.orig\n+++ /lib/systemd/system/prometheus_ferm_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643"}, {"resource": "File[/usr/local/sbin/restart-nginx]", "content": "--- /usr/local/sbin/restart-nginx.orig\n+++ /usr/local/sbin/restart-nginx\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools search-https search-omega-https --services nginx --retries 10 --wait 5 --max-concurrency 5 $@", "parameters": "--- File[/usr/local/sbin/restart-nginx].orig\n+++ File[/usr/local/sbin/restart-nginx]\n\n-    owner  => root\n-    mode   => 0555\n-    before => ['Service[nginx]']\n-    ensure => present\n-    group  => root\n"}, {"resource": "Systemd::Unit[prometheus_ferm_mss.service]"}, {"resource": "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad]\n\n-    require         => ['Class[Conftool::Scripts]']\n-    max_concurrency => 5\n-    lvs_pools       => ['search-omega-https']\n"}, {"resource": "File[/usr/local/bin/depool-opensearch_2@cloudelastic-omega-eqiad.service]", "content": "--- /usr/local/bin/depool-opensearch_2@cloudelastic-omega-eqiad.service.orig\n+++ /usr/local/bin/depool-opensearch_2@cloudelastic-omega-eqiad.service\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_2@cloudelastic-omega-eqiad.service].orig\n+++ File[/usr/local/bin/depool-opensearch_2@cloudelastic-omega-eqiad.service]\n\n+    ensure => present\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]", "content": "--- /usr/local/sbin/restart-opensearch_2@production-search-eqiad.orig\n+++ /usr/local/sbin/restart-opensearch_2@production-search-eqiad\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools search search-https --services opensearch_2@production-search-eqiad --retries 10 --wait 5 --max-concurrency 5 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad].orig\n+++ File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]\n\n-    ensure => present\n-    group  => root\n-    mode   => 0555\n-    owner  => root\n"}, {"resource": "File[/lib/systemd/system/prometheus_lvs_realserver_mss.service]", "content": "--- /lib/systemd/system/prometheus_lvs_realserver_mss.service.orig\n+++ /lib/systemd/system/prometheus_lvs_realserver_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643"}, {"resource": "Class[Profile::Lvs::Realserver::Ipip]", "parameters": "--- Class[Profile::Lvs::Realserver::Ipip].orig\n+++ Class[Profile::Lvs::Realserver::Ipip]\n\n@@\n-    pools => {'search': {'services': ['opensearch_2@production-search-eqiad']}, 'search-https': {'services': ['nginx', 'opensearch_2@production-search-eqiad']}, 'search-omega-https': {'services': ['nginx', 'opensearch_2@production-search-omega-eqiad']}}\n+    pools => {'cloudelastic-chi-https': {'services': ['opensearch_2@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_2@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_2@cloudelastic-omega-eqiad.service']}}\n"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n-    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n+    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}}\n"}, {"resource": "Opensearch::Instance[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-chi-eqiad]\n\n@@\n-    watermarks => {}\n+    watermarks => {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}\n"}, {"resource": "Opensearch::Instance[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-omega-eqiad]\n\n@@\n-    watermarks => {}\n+    watermarks => {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}\n"}, {"resource": "Systemd::Unit[tcp-mss-clamper]"}, {"resource": "Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-psi-eqiad.service]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-psi-eqiad.service].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-psi-eqiad.service]\n\n+    require         => ['Class[Conftool::Scripts]']\n+    max_concurrency => 1\n+    lvs_pools       => ['cloudelastic-psi-https']\n"}, {"resource": "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service]", "content": "--- /usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service.orig\n+++ /usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --services opensearch_2@cloudelastic-psi-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service].orig\n+++ File[/usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service]\n\n+    ensure => present\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "Opensearch::Instance[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-psi-eqiad]\n\n@@\n-    watermarks => {}\n+    watermarks => {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}\n"}, {"resource": "Ferm::Rule[clamp-mss-ipv6]", "parameters": "--- Ferm::Rule[clamp-mss-ipv6].orig\n+++ Ferm::Rule[clamp-mss-ipv6]\n\n@@\n-    rule => outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+    rule => outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n"}, {"resource": "File[/etc/default/wikimedia-lvs-realserver]", "content": "--- /etc/default/wikimedia-lvs-realserver.orig\n+++ /etc/default/wikimedia-lvs-realserver\n@@ -7,4 +7,4 @@\n \n # LVS service IPs to be bound to the loopback interface,\n # separate using spaces\n-LVS_SERVICE_IPS=\"10.2.2.30\"\n+LVS_SERVICE_IPS=\"208.80.154.241 2620:0:861:ed1a::3:241\""}, {"resource": "Systemd::Unit[prometheus_lvs_realserver_mss.service]"}, {"resource": "Prometheus::Node_ferm_mss[ferm_clamped_ipport]", "parameters": "--- Prometheus::Node_ferm_mss[ferm_clamped_ipport].orig\n+++ Prometheus::Node_ferm_mss[ferm_clamped_ipport]\n\n@@\n-    clamped_ipport => ['10.2.2.30:9200', '10.2.2.30:9243', '10.2.2.30:9443']\n+    clamped_ipport => ['208.80.154.241:9243', '208.80.154.241:9443', '208.80.154.241:9643', '[2620:0:861:ed1a::3:241]:9243', '[2620:0:861:ed1a::3:241]:9443', '[2620:0:861:ed1a::3:241]:9643']\n"}, {"resource": "File[/usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service]", "content": "--- /usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service.orig\n+++ /usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service].orig\n+++ File[/usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service]\n\n+    ensure => present\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "File[/usr/local/bin/pool-opensearch_2@cloudelastic-chi-eqiad.service]", "content": "--- /usr/local/bin/pool-opensearch_2@cloudelastic-chi-eqiad.service.orig\n+++ /usr/local/bin/pool-opensearch_2@cloudelastic-chi-eqiad.service\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools cloudelastic-chi-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_2@cloudelastic-chi-eqiad.service].orig\n+++ File[/usr/local/bin/pool-opensearch_2@cloudelastic-chi-eqiad.service]\n\n+    ensure => present\n+    group  => root\n+    mode   => 0555\n+    owner  => root\n"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n-    instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+    instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "File[/etc/conftool/local_services.yaml]", "content": "--- /etc/conftool/local_services.yaml.orig\n+++ /etc/conftool/local_services.yaml\n@@ -1,19 +1,19 @@\n ---\n-search:\n-  cluster: elasticsearch\n-  service: elasticsearch\n+cloudelastic-chi-https:\n+  cluster: cloudelastic\n+  service: cloudelastic-chi-ssl\n   servers:\n-  - pybal-low-traffic.svc.eqiad.wmnet\n-  port: 9200\n-search-https:\n-  cluster: elasticsearch\n-  service: elasticsearch-ssl\n+  - pybal-high-traffic2-eqiad.wikimedia.org\n+  port: 9243\n+cloudelastic-omega-https:\n+  cluster: cloudelastic\n+  service: cloudelastic-omega-ssl\n   servers:\n-  - pybal-low-traffic.svc.eqiad.wmnet\n-  port: 9243\n-search-omega-https:\n-  cluster: elasticsearch\n-  service: elasticsearch-omega-ssl\n+  - pybal-high-traffic2-eqiad.wikimedia.org\n+  port: 9443\n+cloudelastic-psi-https:\n+  cluster: cloudelastic\n+  service: cloudelastic-psi-ssl\n   servers:\n-  - pybal-low-traffic.svc.eqiad.wmnet\n-  port: 9443\n+  - pybal-high-traffic2-eqiad.wikimedia.org\n+  port: 9643"}, {"resource": "File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]", "content": "--- /usr/local/bin/depool-opensearch_2@production-search-omega-eqiad.orig\n+++ /usr/local/bin/depool-opensearch_2@production-search-omega-eqiad\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools search-omega-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad].orig\n+++ File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]\n\n-    ensure => present\n-    group  => root\n-    mode   => 0555\n-    owner  => root\n"}, {"resource": "File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]", "content": "--- /usr/local/bin/depool-opensearch_2@production-search-eqiad.orig\n+++ /usr/local/bin/depool-opensearch_2@production-search-eqiad\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools search search-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_2@production-search-eqiad].orig\n+++ File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]\n\n-    ensure => present\n-    group  => root\n-    mode   => 0555\n-    owner  => root\n"}, {"resource": "File[/lib/systemd/system/tcp-mss-clamper.service]", "content": "--- /lib/systemd/system/tcp-mss-clamper.service.orig\n+++ /lib/systemd/system/tcp-mss-clamper.service\n@@ -7,5 +7,5 @@\n \n [Service]\n LimitMEMLOCK=infinity\n-ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"10.2.2.30:9200,10.2.2.30:9243,10.2.2.30:9443\" -i enp152s0f0np0,lo\n+ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"208.80.154.241:9243,208.80.154.241:9443,208.80.154.241:9643,[2620:0:861:ed1a::3:241]:9243,[2620:0:861:ed1a::3:241]:9443,[2620:0:861:ed1a::3:241]:9643\" -i enp152s0f0np0,lo\n Restart=on-failure"}, {"resource": "Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]", "parameters": "--- Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport].orig\n+++ Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]\n\n@@\n-    clamped_ipport => ['10.2.2.30:9200', '10.2.2.30:9243', '10.2.2.30:9443']\n+    clamped_ipport => ['208.80.154.241:9243', '208.80.154.241:9443', '208.80.154.241:9643', '[2620:0:861:ed1a::3:241]:9243', '[2620:0:861:ed1a::3:241]:9443', '[2620:0:861:ed1a::3:241]:9643']\n"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -148,11 +148,11 @@\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n # Stop allocating shards to nodes (default 80%)\n-cluster.routing.allocation.disk.watermark.low: 0.75\n+cluster.routing.allocation.disk.watermark.low: 100gb\n # Begin evicting shards off the node (default 90%)\n-cluster.routing.allocation.disk.watermark.high: 0.80\n+cluster.routing.allocation.disk.watermark.high: 50gb\n # Set indexes read-only with allocated shards on affected node (default 95%)\n-cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+cluster.routing.allocation.disk.watermark.flood_stage: 10gb\n \n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s"}, {"resource": "File[/usr/local/bin/depool-nginx]", "content": "--- /usr/local/bin/depool-nginx.orig\n+++ /usr/local/bin/depool-nginx\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools search-https search-omega-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-nginx].orig\n+++ File[/usr/local/bin/depool-nginx]\n\n-    owner  => root\n-    mode   => 0555\n-    before => ['Service[nginx]']\n-    ensure => present\n-    group  => root\n"}, {"resource": "Systemd::Service[tcp-mss-clamper]"}], "perc_changed": "2.04%"}, "core": {"total": 3722, "only_in_self": ["File[/usr/local/bin/depool-nginx]", "File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/bin/pool-nginx]", "File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/sbin/restart-nginx]", "File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]", "File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]"], "only_in_other": ["File[/usr/local/bin/depool-opensearch_2@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/depool-opensearch_2@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_2@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service]"], "resource_diffs": [{"resource": "File[/lib/systemd/system/prometheus_lvs_realserver_mss.service]", "content": "--- /lib/systemd/system/prometheus_lvs_realserver_mss.service.orig\n+++ /lib/systemd/system/prometheus_lvs_realserver_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv6]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv6.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv6\n@@ -5,7 +5,7 @@\n domain (ip6) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n \t\t}\n \t}\n }"}, {"resource": "File[/etc/default/wikimedia-lvs-realserver]", "content": "--- /etc/default/wikimedia-lvs-realserver.orig\n+++ /etc/default/wikimedia-lvs-realserver\n@@ -7,4 +7,4 @@\n \n # LVS service IPs to be bound to the loopback interface,\n # separate using spaces\n-LVS_SERVICE_IPS=\"10.2.2.30\"\n+LVS_SERVICE_IPS=\"208.80.154.241 2620:0:861:ed1a::3:241\""}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -148,11 +148,11 @@\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n # Stop allocating shards to nodes (default 80%)\n-cluster.routing.allocation.disk.watermark.low: 0.75\n+cluster.routing.allocation.disk.watermark.low: 100gb\n # Begin evicting shards off the node (default 90%)\n-cluster.routing.allocation.disk.watermark.high: 0.80\n+cluster.routing.allocation.disk.watermark.high: 50gb\n # Set indexes read-only with allocated shards on affected node (default 95%)\n-cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+cluster.routing.allocation.disk.watermark.flood_stage: 10gb\n \n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv4]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv4.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv4\n@@ -5,7 +5,7 @@\n domain (ip) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n \t\t}\n \t}\n }"}, {"resource": "File[/etc/conftool/local_services.yaml]", "content": "--- /etc/conftool/local_services.yaml.orig\n+++ /etc/conftool/local_services.yaml\n@@ -1,19 +1,19 @@\n ---\n-search:\n-  cluster: elasticsearch\n-  service: elasticsearch\n+cloudelastic-chi-https:\n+  cluster: cloudelastic\n+  service: cloudelastic-chi-ssl\n   servers:\n-  - pybal-low-traffic.svc.eqiad.wmnet\n-  port: 9200\n-search-https:\n-  cluster: elasticsearch\n-  service: elasticsearch-ssl\n+  - pybal-high-traffic2-eqiad.wikimedia.org\n+  port: 9243\n+cloudelastic-omega-https:\n+  cluster: cloudelastic\n+  service: cloudelastic-omega-ssl\n   servers:\n-  - pybal-low-traffic.svc.eqiad.wmnet\n-  port: 9243\n-search-omega-https:\n-  cluster: elasticsearch\n-  service: elasticsearch-omega-ssl\n+  - pybal-high-traffic2-eqiad.wikimedia.org\n+  port: 9443\n+cloudelastic-psi-https:\n+  cluster: cloudelastic\n+  service: cloudelastic-psi-ssl\n   servers:\n-  - pybal-low-traffic.svc.eqiad.wmnet\n-  port: 9443\n+  - pybal-high-traffic2-eqiad.wikimedia.org\n+  port: 9643"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -148,11 +148,11 @@\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n # Stop allocating shards to nodes (default 80%)\n-cluster.routing.allocation.disk.watermark.low: 0.75\n+cluster.routing.allocation.disk.watermark.low: 100gb\n # Begin evicting shards off the node (default 90%)\n-cluster.routing.allocation.disk.watermark.high: 0.80\n+cluster.routing.allocation.disk.watermark.high: 50gb\n # Set indexes read-only with allocated shards on affected node (default 95%)\n-cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+cluster.routing.allocation.disk.watermark.flood_stage: 10gb\n \n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s"}, {"resource": "File[/lib/systemd/system/tcp-mss-clamper.service]", "content": "--- /lib/systemd/system/tcp-mss-clamper.service.orig\n+++ /lib/systemd/system/tcp-mss-clamper.service\n@@ -7,5 +7,5 @@\n \n [Service]\n LimitMEMLOCK=infinity\n-ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"10.2.2.30:9200,10.2.2.30:9243,10.2.2.30:9443\" -i enp152s0f0np0,lo\n+ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"208.80.154.241:9243,208.80.154.241:9443,208.80.154.241:9643,[2620:0:861:ed1a::3:241]:9243,[2620:0:861:ed1a::3:241]:9443,[2620:0:861:ed1a::3:241]:9643\" -i enp152s0f0np0,lo\n Restart=on-failure"}, {"resource": "File[/lib/systemd/system/prometheus_ferm_mss.service]", "content": "--- /lib/systemd/system/prometheus_ferm_mss.service.orig\n+++ /lib/systemd/system/prometheus_ferm_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -148,11 +148,11 @@\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n # Stop allocating shards to nodes (default 80%)\n-cluster.routing.allocation.disk.watermark.low: 0.75\n+cluster.routing.allocation.disk.watermark.low: 100gb\n # Begin evicting shards off the node (default 90%)\n-cluster.routing.allocation.disk.watermark.high: 0.80\n+cluster.routing.allocation.disk.watermark.high: 50gb\n # Set indexes read-only with allocated shards on affected node (default 95%)\n-cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+cluster.routing.allocation.disk.watermark.flood_stage: 10gb\n \n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s"}], "perc_changed": "0.75%"}, "main": {"total": 3722, "only_in_self": ["Conftool::Scripts::Safe_service_restart[nginx]", "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad]", "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad]", "File[/usr/local/bin/depool-nginx]", "File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/bin/pool-nginx]", "File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/sbin/restart-nginx]", "File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]", "File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]"], "only_in_other": ["Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-chi-eqiad.service]", "Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-omega-eqiad.service]", "Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-psi-eqiad.service]", "File[/usr/local/bin/depool-opensearch_2@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/depool-opensearch_2@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_2@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service]"], "resource_diffs": [{"resource": "File[/lib/systemd/system/prometheus_lvs_realserver_mss.service]", "content": "--- /lib/systemd/system/prometheus_lvs_realserver_mss.service.orig\n+++ /lib/systemd/system/prometheus_lvs_realserver_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643"}, {"resource": "Systemd::Timer::Job[prometheus_lvs_realserver_mss]", "parameters": "--- Systemd::Timer::Job[prometheus_lvs_realserver_mss].orig\n+++ Systemd::Timer::Job[prometheus_lvs_realserver_mss]\n\n@@\n-    command => /usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+    command => /usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n"}, {"resource": "Class[Profile::Lvs::Realserver]", "parameters": "--- Class[Profile::Lvs::Realserver].orig\n+++ Class[Profile::Lvs::Realserver]\n\n@@\n-    pools => {'search': {'services': ['opensearch_2@production-search-eqiad']}, 'search-https': {'services': ['nginx', 'opensearch_2@production-search-eqiad']}, 'search-omega-https': {'services': ['nginx', 'opensearch_2@production-search-omega-eqiad']}}\n+    pools => {'cloudelastic-chi-https': {'services': ['opensearch_2@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_2@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_2@cloudelastic-omega-eqiad.service']}}\n"}, {"resource": "Class[Profile::Lvs::Realserver::Ipip]", "parameters": "--- Class[Profile::Lvs::Realserver::Ipip].orig\n+++ Class[Profile::Lvs::Realserver::Ipip]\n\n@@\n-    pools => {'search': {'services': ['opensearch_2@production-search-eqiad']}, 'search-https': {'services': ['nginx', 'opensearch_2@production-search-eqiad']}, 'search-omega-https': {'services': ['nginx', 'opensearch_2@production-search-omega-eqiad']}}\n+    pools => {'cloudelastic-chi-https': {'services': ['opensearch_2@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_2@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_2@cloudelastic-omega-eqiad.service']}}\n"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n-    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n+    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}}\n"}, {"resource": "Opensearch::Instance[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-chi-eqiad]\n\n@@\n-    watermarks => {}\n+    watermarks => {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}\n"}, {"resource": "Opensearch::Instance[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-omega-eqiad]\n\n@@\n-    watermarks => {}\n+    watermarks => {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}\n"}, {"resource": "Systemd::Unit[tcp-mss-clamper]"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv6]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv6.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv6\n@@ -5,7 +5,7 @@\n domain (ip6) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n \t\t}\n \t}\n }"}, {"resource": "Opensearch::Instance[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-psi-eqiad]\n\n@@\n-    watermarks => {}\n+    watermarks => {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}\n"}, {"resource": "Ferm::Rule[clamp-mss-ipv6]", "parameters": "--- Ferm::Rule[clamp-mss-ipv6].orig\n+++ Ferm::Rule[clamp-mss-ipv6]\n\n@@\n-    rule => outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+    rule => outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n"}, {"resource": "File[/etc/default/wikimedia-lvs-realserver]", "content": "--- /etc/default/wikimedia-lvs-realserver.orig\n+++ /etc/default/wikimedia-lvs-realserver\n@@ -7,4 +7,4 @@\n \n # LVS service IPs to be bound to the loopback interface,\n # separate using spaces\n-LVS_SERVICE_IPS=\"10.2.2.30\"\n+LVS_SERVICE_IPS=\"208.80.154.241 2620:0:861:ed1a::3:241\""}, {"resource": "Systemd::Unit[prometheus_lvs_realserver_mss.service]"}, {"resource": "Prometheus::Node_ferm_mss[ferm_clamped_ipport]", "parameters": "--- Prometheus::Node_ferm_mss[ferm_clamped_ipport].orig\n+++ Prometheus::Node_ferm_mss[ferm_clamped_ipport]\n\n@@\n-    clamped_ipport => ['10.2.2.30:9200', '10.2.2.30:9243', '10.2.2.30:9443']\n+    clamped_ipport => ['208.80.154.241:9243', '208.80.154.241:9443', '208.80.154.241:9643', '[2620:0:861:ed1a::3:241]:9243', '[2620:0:861:ed1a::3:241]:9443', '[2620:0:861:ed1a::3:241]:9643']\n"}, {"resource": "Ferm::Rule[clamp-mss-ipv4]", "parameters": "--- Ferm::Rule[clamp-mss-ipv4].orig\n+++ Ferm::Rule[clamp-mss-ipv4]\n\n@@\n-    rule => outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+    rule => outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -148,11 +148,11 @@\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n # Stop allocating shards to nodes (default 80%)\n-cluster.routing.allocation.disk.watermark.low: 0.75\n+cluster.routing.allocation.disk.watermark.low: 100gb\n # Begin evicting shards off the node (default 90%)\n-cluster.routing.allocation.disk.watermark.high: 0.80\n+cluster.routing.allocation.disk.watermark.high: 50gb\n # Set indexes read-only with allocated shards on affected node (default 95%)\n-cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+cluster.routing.allocation.disk.watermark.flood_stage: 10gb\n \n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n-    instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+    instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "Class[Lvs::Realserver]", "parameters": "--- Class[Lvs::Realserver].orig\n+++ Class[Lvs::Realserver]\n\n@@\n-    realserver_ips => ['10.2.2.30']\n+    realserver_ips => ['208.80.154.241', '2620:0:861:ed1a::3:241']\n"}, {"resource": "Systemd::Timer::Job[prometheus_ferm_mss]", "parameters": "--- Systemd::Timer::Job[prometheus_ferm_mss].orig\n+++ Systemd::Timer::Job[prometheus_ferm_mss]\n\n@@\n-    command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+    command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv4]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv4.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv4\n@@ -5,7 +5,7 @@\n domain (ip) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n \t\t}\n \t}\n }"}, {"resource": "File[/etc/conftool/local_services.yaml]", "content": "--- /etc/conftool/local_services.yaml.orig\n+++ /etc/conftool/local_services.yaml\n@@ -1,19 +1,19 @@\n ---\n-search:\n-  cluster: elasticsearch\n-  service: elasticsearch\n+cloudelastic-chi-https:\n+  cluster: cloudelastic\n+  service: cloudelastic-chi-ssl\n   servers:\n-  - pybal-low-traffic.svc.eqiad.wmnet\n-  port: 9200\n-search-https:\n-  cluster: elasticsearch\n-  service: elasticsearch-ssl\n+  - pybal-high-traffic2-eqiad.wikimedia.org\n+  port: 9243\n+cloudelastic-omega-https:\n+  cluster: cloudelastic\n+  service: cloudelastic-omega-ssl\n   servers:\n-  - pybal-low-traffic.svc.eqiad.wmnet\n-  port: 9243\n-search-omega-https:\n-  cluster: elasticsearch\n-  service: elasticsearch-omega-ssl\n+  - pybal-high-traffic2-eqiad.wikimedia.org\n+  port: 9443\n+cloudelastic-psi-https:\n+  cluster: cloudelastic\n+  service: cloudelastic-psi-ssl\n   servers:\n-  - pybal-low-traffic.svc.eqiad.wmnet\n-  port: 9443\n+  - pybal-high-traffic2-eqiad.wikimedia.org\n+  port: 9643"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -148,11 +148,11 @@\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n # Stop allocating shards to nodes (default 80%)\n-cluster.routing.allocation.disk.watermark.low: 0.75\n+cluster.routing.allocation.disk.watermark.low: 100gb\n # Begin evicting shards off the node (default 90%)\n-cluster.routing.allocation.disk.watermark.high: 0.80\n+cluster.routing.allocation.disk.watermark.high: 50gb\n # Set indexes read-only with allocated shards on affected node (default 95%)\n-cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+cluster.routing.allocation.disk.watermark.flood_stage: 10gb\n \n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s"}, {"resource": "File[/lib/systemd/system/tcp-mss-clamper.service]", "content": "--- /lib/systemd/system/tcp-mss-clamper.service.orig\n+++ /lib/systemd/system/tcp-mss-clamper.service\n@@ -7,5 +7,5 @@\n \n [Service]\n LimitMEMLOCK=infinity\n-ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"10.2.2.30:9200,10.2.2.30:9243,10.2.2.30:9443\" -i enp152s0f0np0,lo\n+ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"208.80.154.241:9243,208.80.154.241:9443,208.80.154.241:9643,[2620:0:861:ed1a::3:241]:9243,[2620:0:861:ed1a::3:241]:9443,[2620:0:861:ed1a::3:241]:9643\" -i enp152s0f0np0,lo\n Restart=on-failure"}, {"resource": "Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]", "parameters": "--- Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport].orig\n+++ Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]\n\n@@\n-    clamped_ipport => ['10.2.2.30:9200', '10.2.2.30:9243', '10.2.2.30:9443']\n+    clamped_ipport => ['208.80.154.241:9243', '208.80.154.241:9443', '208.80.154.241:9643', '[2620:0:861:ed1a::3:241]:9243', '[2620:0:861:ed1a::3:241]:9443', '[2620:0:861:ed1a::3:241]:9643']\n"}, {"resource": "File[/lib/systemd/system/prometheus_ferm_mss.service]", "content": "--- /lib/systemd/system/prometheus_ferm_mss.service.orig\n+++ /lib/systemd/system/prometheus_ferm_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n+ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -148,11 +148,11 @@\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n # Stop allocating shards to nodes (default 80%)\n-cluster.routing.allocation.disk.watermark.low: 0.75\n+cluster.routing.allocation.disk.watermark.low: 100gb\n # Begin evicting shards off the node (default 90%)\n-cluster.routing.allocation.disk.watermark.high: 0.80\n+cluster.routing.allocation.disk.watermark.high: 50gb\n # Set indexes read-only with allocated shards on affected node (default 95%)\n-cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+cluster.routing.allocation.disk.watermark.flood_stage: 10gb\n \n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s"}, {"resource": "Systemd::Unit[prometheus_ferm_mss.service]"}, {"resource": "Systemd::Service[tcp-mss-clamper]"}], "perc_changed": "1.40%"}}}