{"host": "cloudelastic1012.eqiad.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3700, "only_in_self": ["Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "File[/lib/systemd/system/opensearch_1@.service]", "Service[opensearch_1@cloudelastic-chi-eqiad]", "Service[opensearch_1@cloudelastic-omega-eqiad]", "Service[opensearch_1@cloudelastic-psi-eqiad]", "Systemd::Unit[opensearch_1@.service]"], "only_in_other": ["Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "File[/lib/systemd/system/opensearch_2@.service]", "Service[opensearch_2@cloudelastic-chi-eqiad]", "Service[opensearch_2@cloudelastic-omega-eqiad]", "Service[opensearch_2@cloudelastic-psi-eqiad]", "Systemd::Unit[opensearch_2@.service]"], "resource_diffs": [{"resource": "Opensearch::Instance[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-omega-eqiad]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "Package[wmf-opensearch-search-plugins]", "parameters": "--- Package[wmf-opensearch-search-plugins].orig\n+++ Package[wmf-opensearch-search-plugins]\n\n@@\n-    before => ['Service[opensearch_1@cloudelastic-chi-eqiad]', 'Service[opensearch_1@cloudelastic-omega-eqiad]', 'Service[opensearch_1@cloudelastic-psi-eqiad]']\n+    before => ['Service[opensearch_2@cloudelastic-chi-eqiad]', 'Service[opensearch_2@cloudelastic-omega-eqiad]', 'Service[opensearch_2@cloudelastic-psi-eqiad]']\n"}, {"resource": "Service[opensearch_1@cloudelastic-omega-eqiad]", "parameters": "--- Service[opensearch_1@cloudelastic-omega-eqiad].orig\n+++ Service[opensearch_1@cloudelastic-omega-eqiad]\n\n-    require  => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-omega-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-omega-eqiad]']\n-    enable   => True\n-    tag      => opensearch_services\n-    ensure   => running\n-    provider => systemd\n"}, {"resource": "File[/lib/systemd/system/opensearch_2@.service]", "content": "--- /lib/systemd/system/opensearch_2@.service.orig\n+++ /lib/systemd/system/opensearch_2@.service\n@@ -0,0 +1,73 @@\n+[Unit]\n+Description=OpenSearch (cluster %i)\n+PartOf=opensearch.service\n+Documentation=https://opensearch.org/docs\n+Wants=network-online.target\n+After=network-online.target\n+\n+[Service]\n+Type=notify\n+RuntimeDirectory=opensearch\n+PrivateTmp=true\n+Environment=OPENSEARCH_HOME=/usr/share/opensearch\n+Environment=OPENSEARCH_PATH_CONF=/etc/opensearch/%i\n+Environment=OPENSEARCH_JVM_OPTIONS=/etc/opensearch/%i/jvm.options\n+Environment=PID_DIR=/run/opensearch-%i\n+Environment=OPENSEARCH_SD_NOTIFY=true\n+\n+Environment=LD_LIBRARY_PATH=/usr/share/opensearch/plugins/opensearch-knn/lib\n+Environment=JAVA_HOME=/usr/lib/jvm/java21-openjdk-amd64\n+\n+WorkingDirectory=/usr/share/opensearch\n+\n+User=opensearch\n+Group=opensearch\n+\n+# Force creation of /run/opensearch (see T276198)\n+ExecStartPre=/bin/systemd-tmpfiles --create /etc/tmpfiles.d/opensearch-%i.conf\n+ExecStart=/usr/share/opensearch/bin/opensearch \\\n+        -p ${PID_DIR}/%i.pid \\\n+        --quiet \\\n+        -Epath.logs=/var/log/opensearch \\\n+        -Epath.data=/srv/opensearch/%i\n+\n+# StandardOutput is configured to redirect to journalctl since\n+# some error messages may be logged in standard output before\n+# opensearch logging system is initialized. OpenSearch\n+# stores its logs in /var/log/opensearch and does not use\n+# journalctl by default. If you also want to enable journalctl\n+# logging, you can simply remove the \"quiet\" option from ExecStart.\n+StandardOutput=journal\n+StandardError=inherit\n+\n+# Specifies the maximum file descriptor number that can be opened by this process\n+LimitNOFILE=65535\n+\n+# Specifies the maximum number of processes\n+LimitNPROC=4096\n+\n+# Specifies the maximum size of virtual memory\n+LimitAS=infinity\n+\n+# Specifies the maximum file size\n+LimitFSIZE=infinity\n+\n+# Disable timeout logic and wait until process is stopped\n+TimeoutStopSec=0\n+\n+# SIGTERM signal is used to stop the Java process\n+KillSignal=SIGTERM\n+\n+# Send the signal only to the JVM rather than its control group\n+KillMode=process\n+\n+# Java process is never killed\n+SendSIGKILL=no\n+\n+# When a JVM receives a SIGTERM signal it exits with code 143\n+SuccessExitStatus=143\n+\n+[Install]\n+WantedBy=multi-user.target\n+\n+# Built for packages-7.4.1 (packages)", "parameters": "--- File[/lib/systemd/system/opensearch_2@.service].orig\n+++ File[/lib/systemd/system/opensearch_2@.service]\n\n+    mode   => 0444\n+    ensure => present\n+    group  => root\n+    owner  => root\n+    notify => Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]\n"}, {"resource": "Package[opensearch]", "parameters": "--- Package[opensearch].orig\n+++ Package[opensearch]\n\n@@\n-    ensure => 1.3.20\n+    ensure => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -142,12 +146,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +201,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "parameters": "--- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)].orig\n+++ Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]\n\n+    command     => /bin/systemctl daemon-reload\n+    refreshonly => True\n"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Service[opensearch_1@cloudelastic-psi-eqiad]", "parameters": "--- Service[opensearch_1@cloudelastic-psi-eqiad].orig\n+++ Service[opensearch_1@cloudelastic-psi-eqiad]\n\n-    require  => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-psi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-psi-eqiad]']\n-    enable   => True\n-    tag      => opensearch_services\n-    ensure   => running\n-    provider => systemd\n"}, {"resource": "Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "parameters": "--- Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)].orig\n+++ Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]\n\n-    command     => /bin/systemctl daemon-reload\n-    refreshonly => True\n"}, {"resource": "Apt::Repository[wikimedia-opensearch]", "parameters": "--- Apt::Repository[wikimedia-opensearch].orig\n+++ Apt::Repository[wikimedia-opensearch]\n\n@@\n-    components => thirdparty/opensearch1\n+    components => thirdparty/opensearch2\n"}, {"resource": "Class[Opensearch::Packages]", "parameters": "--- Class[Opensearch::Packages].orig\n+++ Class[Opensearch::Packages]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "Opensearch::Instance[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-psi-eqiad]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "Apt::Repository[wikimedia-opensearch-plugins]", "parameters": "--- Apt::Repository[wikimedia-opensearch-plugins].orig\n+++ Apt::Repository[wikimedia-opensearch-plugins]\n\n@@\n-    components => component/opensearch13\n+    components => 13\n"}, {"resource": "File[/lib/systemd/system/opensearch_1@.service]", "content": "--- /lib/systemd/system/opensearch_1@.service.orig\n+++ /lib/systemd/system/opensearch_1@.service\n@@ -1,73 +0,0 @@\n-[Unit]\n-Description=OpenSearch (cluster %i)\n-PartOf=opensearch.service\n-Documentation=https://opensearch.org/docs\n-Wants=network-online.target\n-After=network-online.target\n-\n-[Service]\n-Type=notify\n-RuntimeDirectory=opensearch\n-PrivateTmp=true\n-Environment=OPENSEARCH_HOME=/usr/share/opensearch\n-Environment=OPENSEARCH_PATH_CONF=/etc/opensearch/%i\n-Environment=OPENSEARCH_JVM_OPTIONS=/etc/opensearch/%i/jvm.options\n-Environment=PID_DIR=/run/opensearch-%i\n-Environment=OPENSEARCH_SD_NOTIFY=true\n-\n-Environment=LD_LIBRARY_PATH=/usr/share/opensearch/plugins/opensearch-knn/lib\n-Environment=JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64\n-\n-WorkingDirectory=/usr/share/opensearch\n-\n-User=opensearch\n-Group=opensearch\n-\n-# Force creation of /run/opensearch (see T276198)\n-ExecStartPre=/bin/systemd-tmpfiles --create /etc/tmpfiles.d/opensearch-%i.conf\n-ExecStart=/usr/share/opensearch/bin/opensearch \\\n-        -p ${PID_DIR}/%i.pid \\\n-        --quiet \\\n-        -Epath.logs=/var/log/opensearch \\\n-        -Epath.data=/srv/opensearch/%i\n-\n-# StandardOutput is configured to redirect to journalctl since\n-# some error messages may be logged in standard output before\n-# opensearch logging system is initialized. OpenSearch\n-# stores its logs in /var/log/opensearch and does not use\n-# journalctl by default. If you also want to enable journalctl\n-# logging, you can simply remove the \"quiet\" option from ExecStart.\n-StandardOutput=journal\n-StandardError=inherit\n-\n-# Specifies the maximum file descriptor number that can be opened by this process\n-LimitNOFILE=65535\n-\n-# Specifies the maximum number of processes\n-LimitNPROC=4096\n-\n-# Specifies the maximum size of virtual memory\n-LimitAS=infinity\n-\n-# Specifies the maximum file size\n-LimitFSIZE=infinity\n-\n-# Disable timeout logic and wait until process is stopped\n-TimeoutStopSec=0\n-\n-# SIGTERM signal is used to stop the Java process\n-KillSignal=SIGTERM\n-\n-# Send the signal only to the JVM rather than its control group\n-KillMode=process\n-\n-# Java process is never killed\n-SendSIGKILL=no\n-\n-# When a JVM receives a SIGTERM signal it exits with code 143\n-SuccessExitStatus=143\n-\n-[Install]\n-WantedBy=multi-user.target\n-\n-# Built for packages-7.4.1 (packages)", "parameters": "--- File[/lib/systemd/system/opensearch_1@.service].orig\n+++ File[/lib/systemd/system/opensearch_1@.service]\n\n-    mode   => 0444\n-    ensure => present\n-    group  => root\n-    owner  => root\n-    notify => Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]\n"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia 13\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia 13"}, {"resource": "Systemd::Unit[opensearch_1@.service]", "parameters": "--- Systemd::Unit[opensearch_1@.service].orig\n+++ Systemd::Unit[opensearch_1@.service]\n\n-    override_filename => puppet-override.conf\n-    unit              => opensearch_1@.service\n-    require           => ['Class[Systemd]']\n-    restart           => False\n-    override          => False\n-    ensure            => present\n"}, {"resource": "Service[opensearch_2@cloudelastic-chi-eqiad]", "parameters": "--- Service[opensearch_2@cloudelastic-chi-eqiad].orig\n+++ Service[opensearch_2@cloudelastic-chi-eqiad]\n\n+    require  => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-chi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-chi-eqiad]']\n+    enable   => True\n+    tag      => opensearch_services\n+    ensure   => running\n+    provider => systemd\n"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n-    java_home => /usr/lib/jvm/java-11-openjdk-amd64\n+    java_home => /usr/lib/jvm/java21-openjdk-amd64\n@@\n-    version   => 1.3.20\n+    version   => 2.19.5\n"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2"}, {"resource": "Service[opensearch_1@cloudelastic-chi-eqiad]", "parameters": "--- Service[opensearch_1@cloudelastic-chi-eqiad].orig\n+++ Service[opensearch_1@cloudelastic-chi-eqiad]\n\n-    require  => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-chi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-chi-eqiad]']\n-    enable   => True\n-    tag      => opensearch_services\n-    ensure   => running\n-    provider => systemd\n"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n-    java_home => /usr/lib/jvm/java-11-openjdk-amd64\n+    java_home => /usr/lib/jvm/java21-openjdk-amd64\n@@\n-    version   => 1.3.20\n+    version   => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -142,12 +146,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +201,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Class[Profile::Opensearch::Cirrus::Server]", "parameters": "--- Class[Profile::Opensearch::Cirrus::Server].orig\n+++ Class[Profile::Opensearch::Cirrus::Server]\n\n+    apt_component => 13\n+    version       => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -142,12 +146,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +201,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/instances]", "content": "--- /etc/opensearch/instances.orig\n+++ /etc/opensearch/instances\n@@ -1,3 +1,3 @@\n-opensearch_1@cloudelastic-chi-eqiad\n-opensearch_1@cloudelastic-omega-eqiad\n-opensearch_1@cloudelastic-psi-eqiad\n+opensearch_2@cloudelastic-chi-eqiad\n+opensearch_2@cloudelastic-omega-eqiad\n+opensearch_2@cloudelastic-psi-eqiad"}, {"resource": "Service[opensearch_2@cloudelastic-omega-eqiad]", "parameters": "--- Service[opensearch_2@cloudelastic-omega-eqiad].orig\n+++ Service[opensearch_2@cloudelastic-omega-eqiad]\n\n+    require  => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-omega-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-omega-eqiad]']\n+    enable   => True\n+    tag      => opensearch_services\n+    ensure   => running\n+    provider => systemd\n"}, {"resource": "Service[opensearch_2@cloudelastic-psi-eqiad]", "parameters": "--- Service[opensearch_2@cloudelastic-psi-eqiad].orig\n+++ Service[opensearch_2@cloudelastic-psi-eqiad]\n\n+    require  => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-psi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-psi-eqiad]']\n+    enable   => True\n+    tag      => opensearch_services\n+    ensure   => running\n+    provider => systemd\n"}, {"resource": "Systemd::Unit[opensearch_2@.service]", "parameters": "--- Systemd::Unit[opensearch_2@.service].orig\n+++ Systemd::Unit[opensearch_2@.service]\n\n+    override_filename => puppet-override.conf\n+    unit              => opensearch_2@.service\n+    require           => ['Class[Systemd]']\n+    restart           => False\n+    override          => False\n+    ensure            => present\n"}, {"resource": "Opensearch::Instance[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-chi-eqiad]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}], "perc_changed": "1.19%"}, "core": {"total": 3700, "only_in_self": ["Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "File[/lib/systemd/system/opensearch_1@.service]", "Service[opensearch_1@cloudelastic-chi-eqiad]", "Service[opensearch_1@cloudelastic-omega-eqiad]", "Service[opensearch_1@cloudelastic-psi-eqiad]"], "only_in_other": ["Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "File[/lib/systemd/system/opensearch_2@.service]", "Service[opensearch_2@cloudelastic-chi-eqiad]", "Service[opensearch_2@cloudelastic-omega-eqiad]", "Service[opensearch_2@cloudelastic-psi-eqiad]"], "resource_diffs": [{"resource": "Package[wmf-opensearch-search-plugins]", "parameters": "--- Package[wmf-opensearch-search-plugins].orig\n+++ Package[wmf-opensearch-search-plugins]\n\n@@\n-    before => ['Service[opensearch_1@cloudelastic-chi-eqiad]', 'Service[opensearch_1@cloudelastic-omega-eqiad]', 'Service[opensearch_1@cloudelastic-psi-eqiad]']\n+    before => ['Service[opensearch_2@cloudelastic-chi-eqiad]', 'Service[opensearch_2@cloudelastic-omega-eqiad]', 'Service[opensearch_2@cloudelastic-psi-eqiad]']\n"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia 13\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia 13"}, {"resource": "Package[opensearch]", "parameters": "--- Package[opensearch].orig\n+++ Package[opensearch]\n\n@@\n-    ensure => 1.3.20\n+    ensure => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -142,12 +146,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +201,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -142,12 +146,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +201,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -142,12 +146,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +201,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/instances]", "content": "--- /etc/opensearch/instances.orig\n+++ /etc/opensearch/instances\n@@ -1,3 +1,3 @@\n-opensearch_1@cloudelastic-chi-eqiad\n-opensearch_1@cloudelastic-omega-eqiad\n-opensearch_1@cloudelastic-psi-eqiad\n+opensearch_2@cloudelastic-chi-eqiad\n+opensearch_2@cloudelastic-omega-eqiad\n+opensearch_2@cloudelastic-psi-eqiad"}], "perc_changed": "0.57%"}, "main": {"total": 3700, "only_in_self": ["Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "File[/lib/systemd/system/opensearch_1@.service]", "Service[opensearch_1@cloudelastic-chi-eqiad]", "Service[opensearch_1@cloudelastic-omega-eqiad]", "Service[opensearch_1@cloudelastic-psi-eqiad]", "Systemd::Unit[opensearch_1@.service]"], "only_in_other": ["Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "File[/lib/systemd/system/opensearch_2@.service]", "Service[opensearch_2@cloudelastic-chi-eqiad]", "Service[opensearch_2@cloudelastic-omega-eqiad]", "Service[opensearch_2@cloudelastic-psi-eqiad]", "Systemd::Unit[opensearch_2@.service]"], "resource_diffs": [{"resource": "Apt::Repository[wikimedia-opensearch-plugins]", "parameters": "--- Apt::Repository[wikimedia-opensearch-plugins].orig\n+++ Apt::Repository[wikimedia-opensearch-plugins]\n\n@@\n-    components => component/opensearch13\n+    components => 13\n"}, {"resource": "Opensearch::Instance[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-omega-eqiad]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "Package[wmf-opensearch-search-plugins]", "parameters": "--- Package[wmf-opensearch-search-plugins].orig\n+++ Package[wmf-opensearch-search-plugins]\n\n@@\n-    before => ['Service[opensearch_1@cloudelastic-chi-eqiad]', 'Service[opensearch_1@cloudelastic-omega-eqiad]', 'Service[opensearch_1@cloudelastic-psi-eqiad]']\n+    before => ['Service[opensearch_2@cloudelastic-chi-eqiad]', 'Service[opensearch_2@cloudelastic-omega-eqiad]', 'Service[opensearch_2@cloudelastic-psi-eqiad]']\n"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia 13\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia 13"}, {"resource": "Package[opensearch]", "parameters": "--- Package[opensearch].orig\n+++ Package[opensearch]\n\n@@\n-    ensure => 1.3.20\n+    ensure => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -142,12 +146,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +201,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n-    java_home => /usr/lib/jvm/java-11-openjdk-amd64\n+    java_home => /usr/lib/jvm/java21-openjdk-amd64\n@@\n-    version   => 1.3.20\n+    version   => 2.19.5\n"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n-    java_home => /usr/lib/jvm/java-11-openjdk-amd64\n+    java_home => /usr/lib/jvm/java21-openjdk-amd64\n@@\n-    version   => 1.3.20\n+    version   => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -142,12 +146,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +201,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Class[Profile::Opensearch::Cirrus::Server]", "parameters": "--- Class[Profile::Opensearch::Cirrus::Server].orig\n+++ Class[Profile::Opensearch::Cirrus::Server]\n\n+    apt_component => 13\n+    version       => 2.19.5\n"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -100,6 +100,10 @@\n #\n #cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n #\n+# Set to ensure a node sees N other master eligible nodes to be considered\n+# operational within the cluster. Its recommended to set it to a higher value\n+# than 1 when running more than 2 nodes in the cluster.\n+#\n discovery.zen.ping.unicast.hosts: [\"cloudelastic1009.eqiad.wmnet\", \"cloudelastic1010.eqiad.wmnet\", \"cloudelastic1011.eqiad.wmnet\"]\n #\n # For more information, consult the discovery and cluster formation module documentation.\n@@ -142,12 +146,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes with 75% disk utilization\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 0.75\n-# Move shards off of nodes with 80% disk utilization\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 0.80\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n+cluster.routing.allocation.disk.watermark.flood_stage: 0.95\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +201,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/instances]", "content": "--- /etc/opensearch/instances.orig\n+++ /etc/opensearch/instances\n@@ -1,3 +1,3 @@\n-opensearch_1@cloudelastic-chi-eqiad\n-opensearch_1@cloudelastic-omega-eqiad\n-opensearch_1@cloudelastic-psi-eqiad\n+opensearch_2@cloudelastic-chi-eqiad\n+opensearch_2@cloudelastic-omega-eqiad\n+opensearch_2@cloudelastic-psi-eqiad"}, {"resource": "Apt::Repository[wikimedia-opensearch]", "parameters": "--- Apt::Repository[wikimedia-opensearch].orig\n+++ Apt::Repository[wikimedia-opensearch]\n\n@@\n-    components => thirdparty/opensearch1\n+    components => thirdparty/opensearch2\n"}, {"resource": "Class[Opensearch::Packages]", "parameters": "--- Class[Opensearch::Packages].orig\n+++ Class[Opensearch::Packages]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "Opensearch::Instance[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-chi-eqiad]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "Opensearch::Instance[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-psi-eqiad]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}], "perc_changed": "0.86%"}}}