{"host": "cloudelastic1011.eqiad.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3790, "only_in_self": ["Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-chi-eqiad.service]", "Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-omega-eqiad.service]", "Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-psi-eqiad.service]", "Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "File[/lib/systemd/system/opensearch_1@.service]", "File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_1@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-psi-eqiad.service]", "Service[opensearch_1@cloudelastic-chi-eqiad]", "Service[opensearch_1@cloudelastic-omega-eqiad]", "Service[opensearch_1@cloudelastic-psi-eqiad]", "Systemd::Unit[opensearch_1@.service]"], "only_in_other": ["Conftool::Scripts::Safe_service_restart[nginx]", "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad]", "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad]", "Exec[install-opensearch]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "File[/etc/systemd/system/opensearch_2@.service.d]", "File[/lib/systemd/system/opensearch_2@.service]", "File[/usr/local/bin/depool-nginx]", "File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/bin/pool-nginx]", "File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/sbin/restart-nginx]", "File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]", "File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]", "Service[opensearch_2@cloudelastic-chi-eqiad]", "Service[opensearch_2@cloudelastic-omega-eqiad]", "Service[opensearch_2@cloudelastic-psi-eqiad]", "Systemd::Override[opensearch_2@]", "Systemd::Unit[opensearch_2@-opensearch_2@]", "Systemd::Unit[opensearch_2@.service]"], "resource_diffs": [{"resource": "Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-omega-eqiad.service]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-omega-eqiad.service].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-omega-eqiad.service]\n\n- max_concurrency => 1\n- lvs_pools => ['cloudelastic-omega-https']\n- require => ['Class[Conftool::Scripts]']\n"}, {"resource": "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]", "parameters": "--- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)].orig\n+++ Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]\n\n+ refreshonly => True\n+ command => /bin/systemctl daemon-reload\n"}, {"resource": "Opensearch::Cross_cluster_settings[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Cross_cluster_settings[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Cross_cluster_settings[cloudelastic-psi-eqiad]\n\n@@\n- settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+ settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "File[/lib/systemd/system/opensearch_1@.service]", "content": "--- /lib/systemd/system/opensearch_1@.service.orig\n+++ /lib/systemd/system/opensearch_1@.service\n@@ -1,73 +0,0 @@\n-[Unit]\n-Description=OpenSearch (cluster %i)\n-PartOf=opensearch.service\n-Documentation=https://opensearch.org/docs\n-Wants=network-online.target\n-After=network-online.target\n-\n-[Service]\n-Type=notify\n-RuntimeDirectory=opensearch\n-PrivateTmp=true\n-Environment=OPENSEARCH_HOME=/usr/share/opensearch\n-Environment=OPENSEARCH_PATH_CONF=/etc/opensearch/%i\n-Environment=OPENSEARCH_JVM_OPTIONS=/etc/opensearch/%i/jvm.options\n-Environment=PID_DIR=/run/opensearch-%i\n-Environment=OPENSEARCH_SD_NOTIFY=true\n-\n-Environment=LD_LIBRARY_PATH=/usr/share/opensearch/plugins/opensearch-knn/lib\n-Environment=JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64\n-\n-WorkingDirectory=/usr/share/opensearch\n-\n-User=opensearch\n-Group=opensearch\n-\n-# Force creation of /run/opensearch (see T276198)\n-ExecStartPre=/bin/systemd-tmpfiles --create /etc/tmpfiles.d/opensearch-%i.conf\n-ExecStart=/usr/share/opensearch/bin/opensearch \\\n- -p ${PID_DIR}/%i.pid \\\n- --quiet \\\n- -Epath.logs=/var/log/opensearch \\\n- -Epath.data=/srv/opensearch/%i\n-\n-# StandardOutput is configured to redirect to journalctl since\n-# some error messages may be logged in standard output before\n-# opensearch logging system is initialized. OpenSearch\n-# stores its logs in /var/log/opensearch and does not use\n-# journalctl by default. If you also want to enable journalctl\n-# logging, you can simply remove the \"quiet\" option from ExecStart.\n-StandardOutput=journal\n-StandardError=inherit\n-\n-# Specifies the maximum file descriptor number that can be opened by this process\n-LimitNOFILE=65535\n-\n-# Specifies the maximum number of processes\n-LimitNPROC=4096\n-\n-# Specifies the maximum size of virtual memory\n-LimitAS=infinity\n-\n-# Specifies the maximum file size\n-LimitFSIZE=infinity\n-\n-# Disable timeout logic and wait until process is stopped\n-TimeoutStopSec=0\n-\n-# SIGTERM signal is used to stop the Java process\n-KillSignal=SIGTERM\n-\n-# Send the signal only to the JVM rather than its control group\n-KillMode=process\n-\n-# Java process is never killed\n-SendSIGKILL=no\n-\n-# When a JVM receives a SIGTERM signal it exits with code 143\n-SuccessExitStatus=143\n-\n-[Install]\n-WantedBy=multi-user.target\n-\n-# Built for packages-7.4.1 (packages)", "parameters": "--- File[/lib/systemd/system/opensearch_1@.service].orig\n+++ File[/lib/systemd/system/opensearch_1@.service]\n\n- group => root\n- owner => root\n- notify => Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]\n- ensure => present\n- mode => 0444\n"}, {"resource": "Systemd::Timer::Job[prometheus_ferm_mss]", "parameters": "--- Systemd::Timer::Job[prometheus_ferm_mss].orig\n+++ Systemd::Timer::Job[prometheus_ferm_mss]\n\n@@\n- command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n"}, {"resource": "Conftool::Scripts::Safe_service_restart[nginx]", "parameters": "--- Conftool::Scripts::Safe_service_restart[nginx].orig\n+++ Conftool::Scripts::Safe_service_restart[nginx]\n\n+ max_concurrency => 5\n+ lvs_pools => ['search-https', 'search-omega-https']\n+ require => ['Class[Conftool::Scripts]']\n"}, {"resource": "Systemd::Override[opensearch_2@]", "parameters": "--- Systemd::Override[opensearch_2@].orig\n+++ Systemd::Override[opensearch_2@]\n\n+ restart => False\n+ ensure => present\n+ unit => opensearch_2@\n"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv6]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv6.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv6\n@@ -5,7 +5,7 @@\n domain (ip6) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n \t\t}\n \t}\n }"}, {"resource": "File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service]", "content": "--- /usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service.orig\n+++ /usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service].orig\n+++ File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service]\n\n- group => root\n- owner => root\n- ensure => present\n- mode => 0555\n"}, {"resource": "Opensearch::Cross_cluster_settings[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Cross_cluster_settings[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Cross_cluster_settings[cloudelastic-omega-eqiad]\n\n@@\n- settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+ settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service]", "content": "--- /usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service.orig\n+++ /usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --services opensearch_1@cloudelastic-omega-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service].orig\n+++ File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service]\n\n- group => root\n- owner => root\n- ensure => present\n- mode => 0555\n"}, {"resource": "File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service]", "content": "--- /usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service.orig\n+++ /usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service].orig\n+++ File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service]\n\n- group => root\n- owner => root\n- ensure => present\n- mode => 0555\n"}, {"resource": "File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service]", "content": "--- /usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service.orig\n+++ /usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service].orig\n+++ File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service]\n\n- group => root\n- owner => root\n- ensure => present\n- mode => 0555\n"}, {"resource": "Class[Opensearch::Packages]", "parameters": "--- Class[Opensearch::Packages].orig\n+++ Class[Opensearch::Packages]\n\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -127,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,12 +144,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -191,3 +199,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "Exec[install-opensearch]", "parameters": "--- Exec[install-opensearch].orig\n+++ Exec[install-opensearch]\n\n+ unless => /usr/bin/dpkg-query -W -f='${Status} ${Version}\\n' opensearch 2>/dev/null | /bin/grep -Fxq 'install ok installed 2.19.5'\n+ command => /usr/bin/apt-get -q -y -o DPkg::Options::=--force-confold install opensearch=2.19.5\n+ logoutput => False\n+ before => Package[opensearch]\n+ environment => ['OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpensearchTemp1!', 'DEBIAN_FRONTEND=noninteractive']\n+ timeout => 300\n"}, {"resource": "File[/etc/systemd/system/opensearch_2@.service.d]", "parameters": "--- File[/etc/systemd/system/opensearch_2@.service.d].orig\n+++ File[/etc/systemd/system/opensearch_2@.service.d]\n\n+ group => root\n+ owner => root\n+ ensure => directory\n+ mode => 0555\n"}, {"resource": "Ferm::Rule[clamp-mss-ipv4]", "parameters": "--- Ferm::Rule[clamp-mss-ipv4].orig\n+++ Ferm::Rule[clamp-mss-ipv4]\n\n@@\n- rule => outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+ rule => outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n"}, {"resource": "File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service]", "content": "--- /usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service.orig\n+++ /usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service].orig\n+++ File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service]\n\n- group => root\n- owner => root\n- ensure => present\n- mode => 0555\n"}, {"resource": "Opensearch::Instance[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-chi-eqiad]\n\n@@\n- plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+ plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3']\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "Class[Lvs::Realserver]", "parameters": "--- Class[Lvs::Realserver].orig\n+++ Class[Lvs::Realserver]\n\n@@\n- realserver_ips => ['208.80.154.241', '2620:0:861:ed1a::3:241']\n+ realserver_ips => ['10.2.2.30']\n"}, {"resource": "File[/usr/local/sbin/restart-nginx]", "content": "--- /usr/local/sbin/restart-nginx.orig\n+++ /usr/local/sbin/restart-nginx\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools search-https search-omega-https --services nginx --retries 10 --wait 5 --max-concurrency 5 $@", "parameters": "--- File[/usr/local/sbin/restart-nginx].orig\n+++ File[/usr/local/sbin/restart-nginx]\n\n+ group => root\n+ owner => root\n+ before => ['Service[nginx]']\n+ ensure => present\n+ mode => 0555\n"}, {"resource": "Package[wmf-opensearch-search-plugins]", "parameters": "--- Package[wmf-opensearch-search-plugins].orig\n+++ Package[wmf-opensearch-search-plugins]\n\n@@\n- before => ['Service[opensearch_1@cloudelastic-chi-eqiad]', 'Service[opensearch_1@cloudelastic-omega-eqiad]', 'Service[opensearch_1@cloudelastic-psi-eqiad]']\n+ before => ['Service[opensearch_2@cloudelastic-chi-eqiad]', 'Service[opensearch_2@cloudelastic-omega-eqiad]', 'Service[opensearch_2@cloudelastic-psi-eqiad]']\n"}, {"resource": "Prometheus::Node_ferm_mss[ferm_clamped_ipport]", "parameters": "--- Prometheus::Node_ferm_mss[ferm_clamped_ipport].orig\n+++ Prometheus::Node_ferm_mss[ferm_clamped_ipport]\n\n@@\n- clamped_ipport => ['208.80.154.241:9243', '208.80.154.241:9443', '208.80.154.241:9643', '[2620:0:861:ed1a::3:241]:9243', '[2620:0:861:ed1a::3:241]:9443', '[2620:0:861:ed1a::3:241]:9643']\n+ clamped_ipport => ['10.2.2.30:9200', '10.2.2.30:9243', '10.2.2.30:9443']\n"}, {"resource": "File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]", "content": "--- /usr/local/bin/pool-opensearch_2@production-search-eqiad.orig\n+++ /usr/local/bin/pool-opensearch_2@production-search-eqiad\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools search search-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_2@production-search-eqiad].orig\n+++ File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]\n\n+ group => root\n+ owner => root\n+ ensure => present\n+ mode => 0555\n"}, {"resource": "File[/etc/conftool/local_services.yaml]", "content": "--- /etc/conftool/local_services.yaml.orig\n+++ /etc/conftool/local_services.yaml\n@@ -1,19 +1,19 @@\n ---\n-cloudelastic-chi-https:\n- cluster: cloudelastic\n- service: cloudelastic-chi-ssl\n+search:\n+ cluster: elasticsearch\n+ service: elasticsearch\n servers:\n- - pybal-high-traffic2-eqiad.wikimedia.org\n+ - pybal-low-traffic.svc.eqiad.wmnet\n+ port: 9200\n+search-https:\n+ cluster: elasticsearch\n+ service: elasticsearch-ssl\n+ servers:\n+ - pybal-low-traffic.svc.eqiad.wmnet\n port: 9243\n-cloudelastic-omega-https:\n- cluster: cloudelastic\n- service: cloudelastic-omega-ssl\n+search-omega-https:\n+ cluster: elasticsearch\n+ service: elasticsearch-omega-ssl\n servers:\n- - pybal-high-traffic2-eqiad.wikimedia.org\n+ - pybal-low-traffic.svc.eqiad.wmnet\n port: 9443\n-cloudelastic-psi-https:\n- cluster: cloudelastic\n- service: cloudelastic-psi-ssl\n- servers:\n- - pybal-high-traffic2-eqiad.wikimedia.org\n- port: 9643"}, {"resource": "File[/lib/systemd/system/opensearch_2@.service]", "content": "--- /lib/systemd/system/opensearch_2@.service.orig\n+++ /lib/systemd/system/opensearch_2@.service\n@@ -0,0 +1,73 @@\n+[Unit]\n+Description=OpenSearch (cluster %i)\n+PartOf=opensearch.service\n+Documentation=https://opensearch.org/docs\n+Wants=network-online.target\n+After=network-online.target\n+\n+[Service]\n+Type=notify\n+RuntimeDirectory=opensearch\n+PrivateTmp=true\n+Environment=OPENSEARCH_HOME=/usr/share/opensearch\n+Environment=OPENSEARCH_PATH_CONF=/etc/opensearch/%i\n+Environment=OPENSEARCH_JVM_OPTIONS=/etc/opensearch/%i/jvm.options\n+Environment=PID_DIR=/run/opensearch-%i\n+Environment=OPENSEARCH_SD_NOTIFY=true\n+\n+Environment=LD_LIBRARY_PATH=/usr/share/opensearch/plugins/opensearch-knn/lib\n+Environment=JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64\n+\n+WorkingDirectory=/usr/share/opensearch\n+\n+User=opensearch\n+Group=opensearch\n+\n+# Force creation of /run/opensearch (see T276198)\n+ExecStartPre=/bin/systemd-tmpfiles --create /etc/tmpfiles.d/opensearch-%i.conf\n+ExecStart=/usr/share/opensearch/bin/opensearch \\\n+ -p ${PID_DIR}/%i.pid \\\n+ --quiet \\\n+ -Epath.logs=/var/log/opensearch \\\n+ -Epath.data=/srv/opensearch/%i\n+\n+# StandardOutput is configured to redirect to journalctl since\n+# some error messages may be logged in standard output before\n+# opensearch logging system is initialized. OpenSearch\n+# stores its logs in /var/log/opensearch and does not use\n+# journalctl by default. If you also want to enable journalctl\n+# logging, you can simply remove the \"quiet\" option from ExecStart.\n+StandardOutput=journal\n+StandardError=inherit\n+\n+# Specifies the maximum file descriptor number that can be opened by this process\n+LimitNOFILE=65535\n+\n+# Specifies the maximum number of processes\n+LimitNPROC=4096\n+\n+# Specifies the maximum size of virtual memory\n+LimitAS=infinity\n+\n+# Specifies the maximum file size\n+LimitFSIZE=infinity\n+\n+# Disable timeout logic and wait until process is stopped\n+TimeoutStopSec=0\n+\n+# SIGTERM signal is used to stop the Java process\n+KillSignal=SIGTERM\n+\n+# Send the signal only to the JVM rather than its control group\n+KillMode=process\n+\n+# Java process is never killed\n+SendSIGKILL=no\n+\n+# When a JVM receives a SIGTERM signal it exits with code 143\n+SuccessExitStatus=143\n+\n+[Install]\n+WantedBy=multi-user.target\n+\n+# Built for packages-7.4.1 (packages)", "parameters": "--- File[/lib/systemd/system/opensearch_2@.service].orig\n+++ File[/lib/systemd/system/opensearch_2@.service]\n\n+ group => root\n+ owner => root\n+ notify => Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]\n+ ensure => present\n+ mode => 0444\n"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n- common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n+ common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n@@\n- java_home => /usr/lib/jvm/java-11-openjdk-amd64\n+ java_home => /usr/lib/jvm/java-21-openjdk-amd64\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "Apt::Repository[wikimedia-opensearch]", "parameters": "--- Apt::Repository[wikimedia-opensearch].orig\n+++ Apt::Repository[wikimedia-opensearch]\n\n@@\n- components => thirdparty/opensearch1\n+ components => thirdparty/opensearch2\n"}, {"resource": "File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]", "content": "--- /usr/local/bin/depool-opensearch_2@production-search-omega-eqiad.orig\n+++ /usr/local/bin/depool-opensearch_2@production-search-omega-eqiad\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools search-omega-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad].orig\n+++ File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]\n\n+ group => root\n+ owner => root\n+ ensure => present\n+ mode => 0555\n"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n- java_home => /usr/lib/jvm/java-11-openjdk-amd64\n+ java_home => /usr/lib/jvm/java-21-openjdk-amd64\n@@\n- instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+ instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n@@\n- version => 1.3.20\n+ version => 2.19.5\n@@\n- plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+ plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3']\n"}, {"resource": "Package[opensearch]", "parameters": "--- Package[opensearch].orig\n+++ Package[opensearch]\n\n@@\n- ensure => 1.3.20\n+ ensure => 2.19.5\n"}, {"resource": "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-psi-eqiad.service]", "content": "--- /usr/local/sbin/restart-opensearch_1@cloudelastic-psi-eqiad.service.orig\n+++ /usr/local/sbin/restart-opensearch_1@cloudelastic-psi-eqiad.service\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --services opensearch_1@cloudelastic-psi-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_1@cloudelastic-psi-eqiad.service].orig\n+++ File[/usr/local/sbin/restart-opensearch_1@cloudelastic-psi-eqiad.service]\n\n- group => root\n- owner => root\n- ensure => present\n- mode => 0555\n"}, {"resource": "Opensearch::Cross_cluster_settings[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Cross_cluster_settings[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Cross_cluster_settings[cloudelastic-chi-eqiad]\n\n@@\n- settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+ settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2"}, {"resource": "File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]", "content": "--- /usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad.orig\n+++ /usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools search-omega-https --services opensearch_2@production-search-omega-eqiad --retries 10 --wait 5 --max-concurrency 5 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad].orig\n+++ File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]\n\n+ group => root\n+ owner => root\n+ ensure => present\n+ mode => 0555\n"}, {"resource": "File[/lib/systemd/system/tcp-mss-clamper.service]", "content": "--- /lib/systemd/system/tcp-mss-clamper.service.orig\n+++ /lib/systemd/system/tcp-mss-clamper.service\n@@ -7,5 +7,5 @@\n \n [Service]\n LimitMEMLOCK=infinity\n-ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"208.80.154.241:9243,208.80.154.241:9443,208.80.154.241:9643,[2620:0:861:ed1a::3:241]:9243,[2620:0:861:ed1a::3:241]:9443,[2620:0:861:ed1a::3:241]:9643\" -i enp152s0f0np0,lo\n+ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"10.2.2.30:9200,10.2.2.30:9243,10.2.2.30:9443\" -i enp152s0f0np0,lo\n Restart=on-failure"}, {"resource": "Service[opensearch_2@cloudelastic-psi-eqiad]", "parameters": "--- Service[opensearch_2@cloudelastic-psi-eqiad].orig\n+++ Service[opensearch_2@cloudelastic-psi-eqiad]\n\n+ tag => opensearch_services\n+ require => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-psi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-psi-eqiad]']\n+ provider => systemd\n+ enable => True\n+ ensure => running\n"}, {"resource": "Systemd::Unit[opensearch_2@-opensearch_2@]", "parameters": "--- Systemd::Unit[opensearch_2@-opensearch_2@].orig\n+++ Systemd::Unit[opensearch_2@-opensearch_2@]\n\n+ override_filename => opensearch_2@\n+ require => ['Class[Systemd]']\n+ override => True\n+ restart => False\n+ ensure => present\n+ unit => opensearch_2@\n"}, {"resource": "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad]\n\n+ max_concurrency => 5\n+ lvs_pools => ['search-omega-https']\n+ require => ['Class[Conftool::Scripts]']\n"}, {"resource": "File[/usr/local/bin/pool-opensearch_1@cloudelastic-chi-eqiad.service]", "content": "--- /usr/local/bin/pool-opensearch_1@cloudelastic-chi-eqiad.service.orig\n+++ /usr/local/bin/pool-opensearch_1@cloudelastic-chi-eqiad.service\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools cloudelastic-chi-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_1@cloudelastic-chi-eqiad.service].orig\n+++ File[/usr/local/bin/pool-opensearch_1@cloudelastic-chi-eqiad.service]\n\n- group => root\n- owner => root\n- ensure => present\n- mode => 0555\n"}, {"resource": "File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service]", "content": "--- /usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service.orig\n+++ /usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools cloudelastic-chi-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service].orig\n+++ File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service]\n\n- group => root\n- owner => root\n- ensure => present\n- mode => 0555\n"}, {"resource": "File[/usr/local/bin/pool-nginx]", "content": "--- /usr/local/bin/pool-nginx.orig\n+++ /usr/local/bin/pool-nginx\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools search-https search-omega-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-nginx].orig\n+++ File[/usr/local/bin/pool-nginx]\n\n+ group => root\n+ owner => root\n+ before => ['Service[nginx]']\n+ ensure => present\n+ mode => 0555\n"}, {"resource": "File[/lib/systemd/system/prometheus_lvs_realserver_mss.service]", "content": "--- /lib/systemd/system/prometheus_lvs_realserver_mss.service.orig\n+++ /lib/systemd/system/prometheus_lvs_realserver_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443"}, {"resource": "Class[Profile::Lvs::Realserver]", "parameters": "--- Class[Profile::Lvs::Realserver].orig\n+++ Class[Profile::Lvs::Realserver]\n\n@@\n- pools => {'cloudelastic-chi-https': {'services': ['opensearch_1@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_1@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_1@cloudelastic-omega-eqiad.service']}}\n+ pools => {'search': {'services': ['opensearch_2@production-search-eqiad']}, 'search-https': {'services': ['nginx', 'opensearch_2@production-search-eqiad']}, 'search-omega-https': {'services': ['nginx', 'opensearch_2@production-search-omega-eqiad']}}\n"}, {"resource": "File[/usr/local/bin/depool-nginx]", "content": "--- /usr/local/bin/depool-nginx.orig\n+++ /usr/local/bin/depool-nginx\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools search-https search-omega-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-nginx].orig\n+++ File[/usr/local/bin/depool-nginx]\n\n+ group => root\n+ owner => root\n+ before => ['Service[nginx]']\n+ ensure => present\n+ mode => 0555\n"}, {"resource": "Class[Profile::Lvs::Realserver::Ipip]", "parameters": "--- Class[Profile::Lvs::Realserver::Ipip].orig\n+++ Class[Profile::Lvs::Realserver::Ipip]\n\n@@\n- pools => {'cloudelastic-chi-https': {'services': ['opensearch_1@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_1@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_1@cloudelastic-omega-eqiad.service']}}\n+ pools => {'search': {'services': ['opensearch_2@production-search-eqiad']}, 'search-https': {'services': ['nginx', 'opensearch_2@production-search-eqiad']}, 'search-omega-https': {'services': ['nginx', 'opensearch_2@production-search-omega-eqiad']}}\n"}, {"resource": "Systemd::Unit[tcp-mss-clamper]"}, {"resource": "File[/etc/default/wikimedia-lvs-realserver]", "content": "--- /etc/default/wikimedia-lvs-realserver.orig\n+++ /etc/default/wikimedia-lvs-realserver\n@@ -7,4 +7,4 @@\n \n # LVS service IPs to be bound to the loopback interface,\n # separate using spaces\n-LVS_SERVICE_IPS=\"208.80.154.241 2620:0:861:ed1a::3:241\"\n+LVS_SERVICE_IPS=\"10.2.2.30\""}, {"resource": "Systemd::Unit[prometheus_lvs_realserver_mss.service]"}, {"resource": "Apt::Repository[wikimedia-opensearch-plugins]", "parameters": "--- Apt::Repository[wikimedia-opensearch-plugins].orig\n+++ Apt::Repository[wikimedia-opensearch-plugins]\n\n@@\n- components => component/opensearch13\n+ components => component/opensearch2\n"}, {"resource": "Service[opensearch_2@cloudelastic-omega-eqiad]", "parameters": "--- Service[opensearch_2@cloudelastic-omega-eqiad].orig\n+++ Service[opensearch_2@cloudelastic-omega-eqiad]\n\n+ tag => opensearch_services\n+ require => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-omega-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-omega-eqiad]']\n+ provider => systemd\n+ enable => True\n+ ensure => running\n"}, {"resource": "Opensearch::Instance[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-omega-eqiad]\n\n@@\n- plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+ plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3']\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "Class[Profile::Opensearch::Cirrus::Server]", "parameters": "--- Class[Profile::Opensearch::Cirrus::Server].orig\n+++ Class[Profile::Opensearch::Cirrus::Server]\n\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -127,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,12 +144,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -191,3 +199,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "Ferm::Rule[clamp-mss-ipv6]", "parameters": "--- Ferm::Rule[clamp-mss-ipv6].orig\n+++ Ferm::Rule[clamp-mss-ipv6]\n\n@@\n- rule => outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+ rule => outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n"}, {"resource": "File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]", "content": "--- /usr/local/bin/pool-opensearch_2@production-search-omega-eqiad.orig\n+++ /usr/local/bin/pool-opensearch_2@production-search-omega-eqiad\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools search-omega-https --pool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad].orig\n+++ File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]\n\n+ group => root\n+ owner => root\n+ ensure => present\n+ mode => 0555\n"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv4]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv4.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv4\n@@ -5,7 +5,7 @@\n domain (ip) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n \t\t}\n \t}\n }"}, {"resource": "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service]", "content": "--- /usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service.orig\n+++ /usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service\n@@ -1,2 +0,0 @@\n-#!/bin/bash\n-/usr/local/bin/safe-service-restart --pools cloudelastic-chi-https --services opensearch_1@cloudelastic-chi-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service].orig\n+++ File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service]\n\n- group => root\n- owner => root\n- ensure => present\n- mode => 0555\n"}, {"resource": "File[/lib/systemd/system/prometheus_ferm_mss.service]", "content": "--- /lib/systemd/system/prometheus_ferm_mss.service.orig\n+++ /lib/systemd/system/prometheus_ferm_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2"}, {"resource": "Opensearch::Instance[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-psi-eqiad]\n\n@@\n- plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+ plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3']\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]", "parameters": "--- Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport].orig\n+++ Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]\n\n@@\n- clamped_ipport => ['208.80.154.241:9243', '208.80.154.241:9443', '208.80.154.241:9643', '[2620:0:861:ed1a::3:241]:9243', '[2620:0:861:ed1a::3:241]:9443', '[2620:0:861:ed1a::3:241]:9643']\n+ clamped_ipport => ['10.2.2.30:9200', '10.2.2.30:9243', '10.2.2.30:9443']\n"}, {"resource": "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "parameters": "--- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)].orig\n+++ Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]\n\n+ refreshonly => True\n+ command => /bin/systemctl daemon-reload\n"}, {"resource": "Service[opensearch_1@cloudelastic-psi-eqiad]", "parameters": "--- Service[opensearch_1@cloudelastic-psi-eqiad].orig\n+++ Service[opensearch_1@cloudelastic-psi-eqiad]\n\n- tag => opensearch_services\n- require => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-psi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-psi-eqiad]']\n- provider => systemd\n- enable => True\n- ensure => running\n"}, {"resource": "File[/etc/opensearch/instances]", "content": "--- /etc/opensearch/instances.orig\n+++ /etc/opensearch/instances\n@@ -1,3 +1,3 @@\n-opensearch_1@cloudelastic-chi-eqiad\n-opensearch_1@cloudelastic-omega-eqiad\n-opensearch_1@cloudelastic-psi-eqiad\n+opensearch_2@cloudelastic-chi-eqiad\n+opensearch_2@cloudelastic-omega-eqiad\n+opensearch_2@cloudelastic-psi-eqiad"}, {"resource": "File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]", "content": "--- /usr/local/bin/depool-opensearch_2@production-search-eqiad.orig\n+++ /usr/local/bin/depool-opensearch_2@production-search-eqiad\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools search search-https --depool --retries 10 --wait 5", "parameters": "--- File[/usr/local/bin/depool-opensearch_2@production-search-eqiad].orig\n+++ File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]\n\n+ group => root\n+ owner => root\n+ ensure => present\n+ mode => 0555\n"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "content": "--- /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf.orig\n+++ /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf\n@@ -0,0 +1,25 @@\n+# The OpenSearch packages come with a lot of plugins\n+# we don't want or need. This override helps us\n+# expose only the plugins we explicitly set via\n+# $plugins_mandatory var\n+[Service]\n+# Give OpenSearch its own mount namespace\n+PrivateMounts=yes\n+\n+# Replace plugins dir with an empty tmpfs\n+TemporaryFileSystem=/usr/share/opensearch/plugins:rw\n+\n+BindPaths=\n+BindPaths=/usr/share/opensearch/plugins/analysis-hebrew\n+BindPaths=/usr/share/opensearch/plugins/analysis-icu\n+BindPaths=/usr/share/opensearch/plugins/analysis-smartcn\n+BindPaths=/usr/share/opensearch/plugins/analysis-stempel\n+BindPaths=/usr/share/opensearch/plugins/analysis-ukrainian\n+BindPaths=/usr/share/opensearch/plugins/cirrus-highlighter\n+BindPaths=/usr/share/opensearch/plugins/opensearch-analysis-stconvert\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-esperanto\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-serbian\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-slovak\n+BindPaths=/usr/share/opensearch/plugins/opensearch-ltr\n+BindPaths=/usr/share/opensearch/plugins/repository-s3", "parameters": "--- File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf].orig\n+++ File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]\n\n+ group => root\n+ owner => root\n+ notify => Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]\n+ ensure => present\n+ mode => 0444\n"}, {"resource": "Systemd::Unit[opensearch_2@.service]", "parameters": "--- Systemd::Unit[opensearch_2@.service].orig\n+++ Systemd::Unit[opensearch_2@.service]\n\n+ override_filename => puppet-override.conf\n+ require => ['Class[Systemd]']\n+ override => False\n+ restart => False\n+ ensure => present\n+ unit => opensearch_2@.service\n"}, {"resource": "Service[opensearch_1@cloudelastic-omega-eqiad]", "parameters": "--- Service[opensearch_1@cloudelastic-omega-eqiad].orig\n+++ Service[opensearch_1@cloudelastic-omega-eqiad]\n\n- tag => opensearch_services\n- require => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-omega-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-omega-eqiad]']\n- provider => systemd\n- enable => True\n- ensure => running\n"}, {"resource": "Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-psi-eqiad.service]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-psi-eqiad.service].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-psi-eqiad.service]\n\n- max_concurrency => 1\n- lvs_pools => ['cloudelastic-psi-https']\n- require => ['Class[Conftool::Scripts]']\n"}, {"resource": "Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-chi-eqiad.service]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-chi-eqiad.service].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-chi-eqiad.service]\n\n- max_concurrency => 1\n- lvs_pools => ['cloudelastic-chi-https']\n- require => ['Class[Conftool::Scripts]']\n"}, {"resource": "Systemd::Unit[prometheus_ferm_mss.service]"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -127,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,12 +144,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -191,3 +199,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]", "content": "--- /usr/local/sbin/restart-opensearch_2@production-search-eqiad.orig\n+++ /usr/local/sbin/restart-opensearch_2@production-search-eqiad\n@@ -0,0 +1,2 @@\n+#!/bin/bash\n+/usr/local/bin/safe-service-restart --pools search search-https --services opensearch_2@production-search-eqiad --retries 10 --wait 5 --max-concurrency 5 $@", "parameters": "--- File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad].orig\n+++ File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]\n\n+ group => root\n+ owner => root\n+ ensure => present\n+ mode => 0555\n"}, {"resource": "Service[opensearch_2@cloudelastic-chi-eqiad]", "parameters": "--- Service[opensearch_2@cloudelastic-chi-eqiad].orig\n+++ Service[opensearch_2@cloudelastic-chi-eqiad]\n\n+ tag => opensearch_services\n+ require => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-chi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-chi-eqiad]']\n+ provider => systemd\n+ enable => True\n+ ensure => running\n"}, {"resource": "Systemd::Unit[opensearch_1@.service]", "parameters": "--- Systemd::Unit[opensearch_1@.service].orig\n+++ Systemd::Unit[opensearch_1@.service]\n\n- override_filename => puppet-override.conf\n- require => ['Class[Systemd]']\n- override => False\n- restart => False\n- ensure => present\n- unit => opensearch_1@.service\n"}, {"resource": "Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "parameters": "--- Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)].orig\n+++ Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]\n\n- refreshonly => True\n- command => /bin/systemctl daemon-reload\n"}, {"resource": "Systemd::Service[tcp-mss-clamper]"}, {"resource": "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad]", "parameters": "--- Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad].orig\n+++ Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad]\n\n+ max_concurrency => 5\n+ lvs_pools => ['search', 'search-https']\n+ require => ['Class[Conftool::Scripts]']\n"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Systemd::Timer::Job[prometheus_lvs_realserver_mss]", "parameters": "--- Systemd::Timer::Job[prometheus_lvs_realserver_mss].orig\n+++ Systemd::Timer::Job[prometheus_lvs_realserver_mss]\n\n@@\n- command => /usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ command => /usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n"}, {"resource": "Service[opensearch_1@cloudelastic-chi-eqiad]", "parameters": "--- Service[opensearch_1@cloudelastic-chi-eqiad].orig\n+++ Service[opensearch_1@cloudelastic-chi-eqiad]\n\n- tag => opensearch_services\n- require => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-chi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-chi-eqiad]']\n- provider => systemd\n- enable => True\n- ensure => running\n"}], "perc_changed": "3.35%"}, "core": {"total": 3790, "only_in_self": ["Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "File[/lib/systemd/system/opensearch_1@.service]", "File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_1@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-psi-eqiad.service]", "Service[opensearch_1@cloudelastic-chi-eqiad]", "Service[opensearch_1@cloudelastic-omega-eqiad]", "Service[opensearch_1@cloudelastic-psi-eqiad]"], "only_in_other": ["Exec[install-opensearch]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "File[/etc/systemd/system/opensearch_2@.service.d]", "File[/lib/systemd/system/opensearch_2@.service]", "File[/usr/local/bin/depool-nginx]", "File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/bin/pool-nginx]", "File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/sbin/restart-nginx]", "File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]", "File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]", "Service[opensearch_2@cloudelastic-chi-eqiad]", "Service[opensearch_2@cloudelastic-omega-eqiad]", "Service[opensearch_2@cloudelastic-psi-eqiad]"], "resource_diffs": [{"resource": "File[/lib/systemd/system/tcp-mss-clamper.service]", "content": "--- /lib/systemd/system/tcp-mss-clamper.service.orig\n+++ /lib/systemd/system/tcp-mss-clamper.service\n@@ -7,5 +7,5 @@\n \n [Service]\n LimitMEMLOCK=infinity\n-ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"208.80.154.241:9243,208.80.154.241:9443,208.80.154.241:9643,[2620:0:861:ed1a::3:241]:9243,[2620:0:861:ed1a::3:241]:9443,[2620:0:861:ed1a::3:241]:9643\" -i enp152s0f0np0,lo\n+ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"10.2.2.30:9200,10.2.2.30:9243,10.2.2.30:9443\" -i enp152s0f0np0,lo\n Restart=on-failure"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv6]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv6.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv6\n@@ -5,7 +5,7 @@\n domain (ip6) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n \t\t}\n \t}\n }"}, {"resource": "File[/lib/systemd/system/prometheus_lvs_realserver_mss.service]", "content": "--- /lib/systemd/system/prometheus_lvs_realserver_mss.service.orig\n+++ /lib/systemd/system/prometheus_lvs_realserver_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -127,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,12 +144,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -191,3 +199,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/default/wikimedia-lvs-realserver]", "content": "--- /etc/default/wikimedia-lvs-realserver.orig\n+++ /etc/default/wikimedia-lvs-realserver\n@@ -7,4 +7,4 @@\n \n # LVS service IPs to be bound to the loopback interface,\n # separate using spaces\n-LVS_SERVICE_IPS=\"208.80.154.241 2620:0:861:ed1a::3:241\"\n+LVS_SERVICE_IPS=\"10.2.2.30\""}, {"resource": "Package[wmf-opensearch-search-plugins]", "parameters": "--- Package[wmf-opensearch-search-plugins].orig\n+++ Package[wmf-opensearch-search-plugins]\n\n@@\n- before => ['Service[opensearch_1@cloudelastic-chi-eqiad]', 'Service[opensearch_1@cloudelastic-omega-eqiad]', 'Service[opensearch_1@cloudelastic-psi-eqiad]']\n+ before => ['Service[opensearch_2@cloudelastic-chi-eqiad]', 'Service[opensearch_2@cloudelastic-omega-eqiad]', 'Service[opensearch_2@cloudelastic-psi-eqiad]']\n"}, {"resource": "File[/etc/conftool/local_services.yaml]", "content": "--- /etc/conftool/local_services.yaml.orig\n+++ /etc/conftool/local_services.yaml\n@@ -1,19 +1,19 @@\n ---\n-cloudelastic-chi-https:\n- cluster: cloudelastic\n- service: cloudelastic-chi-ssl\n+search:\n+ cluster: elasticsearch\n+ service: elasticsearch\n servers:\n- - pybal-high-traffic2-eqiad.wikimedia.org\n+ - pybal-low-traffic.svc.eqiad.wmnet\n+ port: 9200\n+search-https:\n+ cluster: elasticsearch\n+ service: elasticsearch-ssl\n+ servers:\n+ - pybal-low-traffic.svc.eqiad.wmnet\n port: 9243\n-cloudelastic-omega-https:\n- cluster: cloudelastic\n- service: cloudelastic-omega-ssl\n+search-omega-https:\n+ cluster: elasticsearch\n+ service: elasticsearch-omega-ssl\n servers:\n- - pybal-high-traffic2-eqiad.wikimedia.org\n+ - pybal-low-traffic.svc.eqiad.wmnet\n port: 9443\n-cloudelastic-psi-https:\n- cluster: cloudelastic\n- service: cloudelastic-psi-ssl\n- servers:\n- - pybal-high-traffic2-eqiad.wikimedia.org\n- port: 9643"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -127,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,12 +144,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -191,3 +199,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv4]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv4.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv4\n@@ -5,7 +5,7 @@\n domain (ip) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n \t\t}\n \t}\n }"}, {"resource": "File[/lib/systemd/system/prometheus_ferm_mss.service]", "content": "--- /lib/systemd/system/prometheus_ferm_mss.service.orig\n+++ /lib/systemd/system/prometheus_ferm_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2"}, {"resource": "File[/etc/opensearch/instances]", "content": "--- /etc/opensearch/instances.orig\n+++ /etc/opensearch/instances\n@@ -1,3 +1,3 @@\n-opensearch_1@cloudelastic-chi-eqiad\n-opensearch_1@cloudelastic-omega-eqiad\n-opensearch_1@cloudelastic-psi-eqiad\n+opensearch_2@cloudelastic-chi-eqiad\n+opensearch_2@cloudelastic-omega-eqiad\n+opensearch_2@cloudelastic-psi-eqiad"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Package[opensearch]", "parameters": "--- Package[opensearch].orig\n+++ Package[opensearch]\n\n@@\n- ensure => 1.3.20\n+ ensure => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -127,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,12 +144,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -191,3 +199,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}], "perc_changed": "1.32%"}, "main": {"total": 3790, "only_in_self": ["Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-chi-eqiad.service]", "Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-omega-eqiad.service]", "Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-psi-eqiad.service]", "Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "File[/lib/systemd/system/opensearch_1@.service]", "File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_1@cloudelastic-chi-eqiad.service]", "File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service]", "File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service]", "File[/usr/local/sbin/restart-opensearch_1@cloudelastic-psi-eqiad.service]", "Service[opensearch_1@cloudelastic-chi-eqiad]", "Service[opensearch_1@cloudelastic-omega-eqiad]", "Service[opensearch_1@cloudelastic-psi-eqiad]", "Systemd::Unit[opensearch_1@.service]"], "only_in_other": ["Conftool::Scripts::Safe_service_restart[nginx]", "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-eqiad]", "Conftool::Scripts::Safe_service_restart[opensearch_2@production-search-omega-eqiad]", "Exec[install-opensearch]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "File[/etc/systemd/system/opensearch_2@.service.d]", "File[/lib/systemd/system/opensearch_2@.service]", "File[/usr/local/bin/depool-nginx]", "File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/bin/pool-nginx]", "File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]", "File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]", "File[/usr/local/sbin/restart-nginx]", "File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]", "File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]", "Service[opensearch_2@cloudelastic-chi-eqiad]", "Service[opensearch_2@cloudelastic-omega-eqiad]", "Service[opensearch_2@cloudelastic-psi-eqiad]", "Systemd::Override[opensearch_2@]", "Systemd::Unit[opensearch_2@-opensearch_2@]", "Systemd::Unit[opensearch_2@.service]"], "resource_diffs": [{"resource": "File[/lib/systemd/system/tcp-mss-clamper.service]", "content": "--- /lib/systemd/system/tcp-mss-clamper.service.orig\n+++ /lib/systemd/system/tcp-mss-clamper.service\n@@ -7,5 +7,5 @@\n \n [Service]\n LimitMEMLOCK=infinity\n-ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"208.80.154.241:9243,208.80.154.241:9443,208.80.154.241:9643,[2620:0:861:ed1a::3:241]:9243,[2620:0:861:ed1a::3:241]:9443,[2620:0:861:ed1a::3:241]:9643\" -i enp152s0f0np0,lo\n+ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s \"10.2.2.30:9200,10.2.2.30:9243,10.2.2.30:9443\" -i enp152s0f0np0,lo\n Restart=on-failure"}, {"resource": "Opensearch::Cross_cluster_settings[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Cross_cluster_settings[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Cross_cluster_settings[cloudelastic-psi-eqiad]\n\n@@\n- settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+ settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "Systemd::Timer::Job[prometheus_ferm_mss]", "parameters": "--- Systemd::Timer::Job[prometheus_ferm_mss].orig\n+++ Systemd::Timer::Job[prometheus_ferm_mss]\n\n@@\n- command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv6]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv6.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv6\n@@ -5,7 +5,7 @@\n domain (ip6) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n \t\t}\n \t}\n }"}, {"resource": "File[/lib/systemd/system/prometheus_lvs_realserver_mss.service]", "content": "--- /lib/systemd/system/prometheus_lvs_realserver_mss.service.orig\n+++ /lib/systemd/system/prometheus_lvs_realserver_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443"}, {"resource": "Opensearch::Cross_cluster_settings[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Cross_cluster_settings[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Cross_cluster_settings[cloudelastic-omega-eqiad]\n\n@@\n- settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+ settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "Class[Profile::Lvs::Realserver]", "parameters": "--- Class[Profile::Lvs::Realserver].orig\n+++ Class[Profile::Lvs::Realserver]\n\n@@\n- pools => {'cloudelastic-chi-https': {'services': ['opensearch_1@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_1@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_1@cloudelastic-omega-eqiad.service']}}\n+ pools => {'search': {'services': ['opensearch_2@production-search-eqiad']}, 'search-https': {'services': ['nginx', 'opensearch_2@production-search-eqiad']}, 'search-omega-https': {'services': ['nginx', 'opensearch_2@production-search-omega-eqiad']}}\n"}, {"resource": "Class[Profile::Lvs::Realserver::Ipip]", "parameters": "--- Class[Profile::Lvs::Realserver::Ipip].orig\n+++ Class[Profile::Lvs::Realserver::Ipip]\n\n@@\n- pools => {'cloudelastic-chi-https': {'services': ['opensearch_1@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_1@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_1@cloudelastic-omega-eqiad.service']}}\n+ pools => {'search': {'services': ['opensearch_2@production-search-eqiad']}, 'search-https': {'services': ['nginx', 'opensearch_2@production-search-eqiad']}, 'search-omega-https': {'services': ['nginx', 'opensearch_2@production-search-omega-eqiad']}}\n"}, {"resource": "Systemd::Unit[tcp-mss-clamper]"}, {"resource": "Class[Opensearch::Packages]", "parameters": "--- Class[Opensearch::Packages].orig\n+++ Class[Opensearch::Packages]\n\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -127,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,12 +144,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -191,3 +199,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/default/wikimedia-lvs-realserver]", "content": "--- /etc/default/wikimedia-lvs-realserver.orig\n+++ /etc/default/wikimedia-lvs-realserver\n@@ -7,4 +7,4 @@\n \n # LVS service IPs to be bound to the loopback interface,\n # separate using spaces\n-LVS_SERVICE_IPS=\"208.80.154.241 2620:0:861:ed1a::3:241\"\n+LVS_SERVICE_IPS=\"10.2.2.30\""}, {"resource": "Ferm::Rule[clamp-mss-ipv4]", "parameters": "--- Ferm::Rule[clamp-mss-ipv4].orig\n+++ Ferm::Rule[clamp-mss-ipv4]\n\n@@\n- rule => outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+ rule => outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n"}, {"resource": "Opensearch::Instance[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-chi-eqiad]\n\n@@\n- plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+ plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3']\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "Class[Lvs::Realserver]", "parameters": "--- Class[Lvs::Realserver].orig\n+++ Class[Lvs::Realserver]\n\n@@\n- realserver_ips => ['208.80.154.241', '2620:0:861:ed1a::3:241']\n+ realserver_ips => ['10.2.2.30']\n"}, {"resource": "Systemd::Unit[prometheus_lvs_realserver_mss.service]"}, {"resource": "Apt::Repository[wikimedia-opensearch-plugins]", "parameters": "--- Apt::Repository[wikimedia-opensearch-plugins].orig\n+++ Apt::Repository[wikimedia-opensearch-plugins]\n\n@@\n- components => component/opensearch13\n+ components => component/opensearch2\n"}, {"resource": "Opensearch::Instance[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-omega-eqiad]\n\n@@\n- plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+ plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3']\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "Package[wmf-opensearch-search-plugins]", "parameters": "--- Package[wmf-opensearch-search-plugins].orig\n+++ Package[wmf-opensearch-search-plugins]\n\n@@\n- before => ['Service[opensearch_1@cloudelastic-chi-eqiad]', 'Service[opensearch_1@cloudelastic-omega-eqiad]', 'Service[opensearch_1@cloudelastic-psi-eqiad]']\n+ before => ['Service[opensearch_2@cloudelastic-chi-eqiad]', 'Service[opensearch_2@cloudelastic-omega-eqiad]', 'Service[opensearch_2@cloudelastic-psi-eqiad]']\n"}, {"resource": "Prometheus::Node_ferm_mss[ferm_clamped_ipport]", "parameters": "--- Prometheus::Node_ferm_mss[ferm_clamped_ipport].orig\n+++ Prometheus::Node_ferm_mss[ferm_clamped_ipport]\n\n@@\n- clamped_ipport => ['208.80.154.241:9243', '208.80.154.241:9443', '208.80.154.241:9643', '[2620:0:861:ed1a::3:241]:9243', '[2620:0:861:ed1a::3:241]:9443', '[2620:0:861:ed1a::3:241]:9643']\n+ clamped_ipport => ['10.2.2.30:9200', '10.2.2.30:9243', '10.2.2.30:9443']\n"}, {"resource": "Class[Profile::Opensearch::Cirrus::Server]", "parameters": "--- Class[Profile::Opensearch::Cirrus::Server].orig\n+++ Class[Profile::Opensearch::Cirrus::Server]\n\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "File[/etc/conftool/local_services.yaml]", "content": "--- /etc/conftool/local_services.yaml.orig\n+++ /etc/conftool/local_services.yaml\n@@ -1,19 +1,19 @@\n ---\n-cloudelastic-chi-https:\n- cluster: cloudelastic\n- service: cloudelastic-chi-ssl\n+search:\n+ cluster: elasticsearch\n+ service: elasticsearch\n servers:\n- - pybal-high-traffic2-eqiad.wikimedia.org\n+ - pybal-low-traffic.svc.eqiad.wmnet\n+ port: 9200\n+search-https:\n+ cluster: elasticsearch\n+ service: elasticsearch-ssl\n+ servers:\n+ - pybal-low-traffic.svc.eqiad.wmnet\n port: 9243\n-cloudelastic-omega-https:\n- cluster: cloudelastic\n- service: cloudelastic-omega-ssl\n+search-omega-https:\n+ cluster: elasticsearch\n+ service: elasticsearch-omega-ssl\n servers:\n- - pybal-high-traffic2-eqiad.wikimedia.org\n+ - pybal-low-traffic.svc.eqiad.wmnet\n port: 9443\n-cloudelastic-psi-https:\n- cluster: cloudelastic\n- service: cloudelastic-psi-ssl\n- servers:\n- - pybal-high-traffic2-eqiad.wikimedia.org\n- port: 9643"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -127,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,12 +144,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -191,3 +199,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "Ferm::Rule[clamp-mss-ipv6]", "parameters": "--- Ferm::Rule[clamp-mss-ipv6].orig\n+++ Ferm::Rule[clamp-mss-ipv6]\n\n@@\n- rule => outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n+ rule => outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;\n"}, {"resource": "File[/etc/ferm/conf.d/10_clamp-mss-ipv4]", "content": "--- /etc/ferm/conf.d/10_clamp-mss-ipv4.orig\n+++ /etc/ferm/conf.d/10_clamp-mss-ipv4\n@@ -5,7 +5,7 @@\n domain (ip) {\n \ttable filter {\n \t\tchain OUTPUT {\n-\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n+\t\t\touterface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;\n \t\t}\n \t}\n }"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n- common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n+ common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n@@\n- java_home => /usr/lib/jvm/java-11-openjdk-amd64\n+ java_home => /usr/lib/jvm/java-21-openjdk-amd64\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "File[/lib/systemd/system/prometheus_ferm_mss.service]", "content": "--- /lib/systemd/system/prometheus_ferm_mss.service.orig\n+++ /lib/systemd/system/prometheus_ferm_mss.service\n@@ -5,4 +5,4 @@\n [Service]\n Type=oneshot\n User=root\n-ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2"}, {"resource": "Opensearch::Instance[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-psi-eqiad]\n\n@@\n- plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+ plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3']\n@@\n- version => 1.3.20\n+ version => 2.19.5\n"}, {"resource": "Apt::Repository[wikimedia-opensearch]", "parameters": "--- Apt::Repository[wikimedia-opensearch].orig\n+++ Apt::Repository[wikimedia-opensearch]\n\n@@\n- components => thirdparty/opensearch1\n+ components => thirdparty/opensearch2\n"}, {"resource": "Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]", "parameters": "--- Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport].orig\n+++ Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]\n\n@@\n- clamped_ipport => ['208.80.154.241:9243', '208.80.154.241:9443', '208.80.154.241:9643', '[2620:0:861:ed1a::3:241]:9243', '[2620:0:861:ed1a::3:241]:9443', '[2620:0:861:ed1a::3:241]:9643']\n+ clamped_ipport => ['10.2.2.30:9200', '10.2.2.30:9243', '10.2.2.30:9443']\n"}, {"resource": "File[/etc/opensearch/instances]", "content": "--- /etc/opensearch/instances.orig\n+++ /etc/opensearch/instances\n@@ -1,3 +1,3 @@\n-opensearch_1@cloudelastic-chi-eqiad\n-opensearch_1@cloudelastic-omega-eqiad\n-opensearch_1@cloudelastic-psi-eqiad\n+opensearch_2@cloudelastic-chi-eqiad\n+opensearch_2@cloudelastic-omega-eqiad\n+opensearch_2@cloudelastic-psi-eqiad"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n- java_home => /usr/lib/jvm/java-11-openjdk-amd64\n+ java_home => /usr/lib/jvm/java-21-openjdk-amd64\n@@\n- instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+ instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n@@\n- version => 1.3.20\n+ version => 2.19.5\n@@\n- plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+ plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3']\n"}, {"resource": "Package[opensearch]", "parameters": "--- Package[opensearch].orig\n+++ Package[opensearch]\n\n@@\n- ensure => 1.3.20\n+ ensure => 2.19.5\n"}, {"resource": "Systemd::Unit[prometheus_ferm_mss.service]"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -127,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,12 +144,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -191,3 +199,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "Opensearch::Cross_cluster_settings[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Cross_cluster_settings[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Cross_cluster_settings[cloudelastic-chi-eqiad]\n\n@@\n- settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n+ settings => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "Systemd::Service[tcp-mss-clamper]"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Systemd::Timer::Job[prometheus_lvs_realserver_mss]", "parameters": "--- Systemd::Timer::Job[prometheus_lvs_realserver_mss].orig\n+++ Systemd::Timer::Job[prometheus_lvs_realserver_mss]\n\n@@\n- command => /usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643\n+ command => /usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443\n"}], "perc_changed": "2.24%"}}}