--- Service[opensearch_1@cloudelastic-omega-eqiad].orig
+++ Service[opensearch_1@cloudelastic-omega-eqiad]
- tag => opensearch_services
- ensure => running
- enable => True
- require => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-omega-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-omega-eqiad]']
- provider => systemd
Service[opensearch_2@cloudelastic-omega-eqiad]
- Parameters differences:
--- Service[opensearch_2@cloudelastic-omega-eqiad].orig
+++ Service[opensearch_2@cloudelastic-omega-eqiad]
+ tag => opensearch_services
+ ensure => running
+ enable => True
+ require => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-omega-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-omega-eqiad]']
+ provider => systemd
- Class[Profile::Opensearch::Cirrus::Server]
- Parameters differences:
--- Class[Profile::Opensearch::Cirrus::Server].orig
+++ Class[Profile::Opensearch::Cirrus::Server]
@@
- version => 1.3.20
+ version => 2.19.5
- File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]
- Content differences:
--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig
+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml
@@ -100,6 +100,10 @@
#
#cluster.initial_master_nodes: ["node-1", "node-2"]
#
+# Set to ensure a node sees N other master eligible nodes to be considered
+# operational within the cluster. Its recommended to set it to a higher value
+# than 1 when running more than 2 nodes in the cluster.
+#
discovery.zen.ping.unicast.hosts: ["cloudelastic1009.eqiad.wmnet", "cloudelastic1010.eqiad.wmnet", "cloudelastic1011.eqiad.wmnet"]
#
# For more information, consult the discovery and cluster formation module documentation.
@@ -140,12 +144,16 @@
# 1k is too low to allow mwgrep to operate.
action.search.shard_count.limit: 5000
+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation
# Enable the disk space aware shard allocator
cluster.routing.allocation.disk.threshold_enabled: true
-# Stop allocating new shards to nodes with 75% disk utilization
+# Stop allocating shards to nodes (default 80%)
cluster.routing.allocation.disk.watermark.low: 0.75
-# Move shards off of nodes with 80% disk utilization
+# Begin evicting shards off the node (default 90%)
cluster.routing.allocation.disk.watermark.high: 0.80
+# Set indexes read-only with allocated shards on affected node (default 95%)
+cluster.routing.allocation.disk.watermark.flood_stage: 0.95
+
# Check disk utilization every 60 seconds
cluster.info.update.interval: 60s
@@ -191,3 +199,4 @@
ltr.caches.max_mem: 100mb
+
- Service[opensearch_2@cloudelastic-psi-eqiad]
- Parameters differences:
--- Service[opensearch_2@cloudelastic-psi-eqiad].orig
+++ Service[opensearch_2@cloudelastic-psi-eqiad]
+ tag => opensearch_services
+ ensure => running
+ enable => True
+ require => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-psi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-psi-eqiad]']
+ provider => systemd
- Exec[install-opensearch]
- Parameters differences:
--- Exec[install-opensearch].orig
+++ Exec[install-opensearch]
+ logoutput => False
+ environment => ['OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpensearchTemp1!', 'DEBIAN_FRONTEND=noninteractive']
+ unless => /usr/bin/dpkg-query -W -f='${Status} ${Version}\n' opensearch 2>/dev/null | /bin/grep -Fxq 'install ok installed 2.19.5'
+ before => Package[opensearch]
+ timeout => 300
+ command => /usr/bin/apt-get -q -y -o DPkg::Options::=--force-confold install opensearch=2.19.5
- Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]
- Parameters differences:
--- Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)].orig
+++ Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]
- command => /bin/systemctl daemon-reload
- refreshonly => True
- File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service]
- Parameters differences:
--- File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service].orig
+++ File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service]
- ensure => present
- mode => 0555
- owner => root
- group => root
- Content differences:
--- /usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service.orig
+++ /usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service
@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --pool --retries 10 --wait 5
- File[/usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service]
- Parameters differences:
--- File[/usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service].orig
+++ File[/usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service]
+ ensure => present
+ mode => 0555
+ owner => root
+ group => root
- Content differences:
--- /usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service.orig
+++ /usr/local/sbin/restart-opensearch_2@cloudelastic-psi-eqiad.service
@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --services opensearch_2@cloudelastic-psi-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@
- File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service]
- Parameters differences:
--- File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service].orig
+++ File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service]
- ensure => present
- mode => 0555
- owner => root
- group => root
- Content differences:
--- /usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service.orig
+++ /usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service
@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/local/bin/safe-service-restart --pools cloudelastic-chi-https --depool --retries 10 --wait 5
- Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-psi-eqiad.service]
- Parameters differences:
--- Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-psi-eqiad.service].orig
+++ Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-psi-eqiad.service]
+ lvs_pools => ['cloudelastic-psi-https']
+ max_concurrency => 1
+ require => ['Class[Conftool::Scripts]']
- File[/etc/apt/sources.list.d/wikimedia-opensearch.list]
- Content differences:
--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig
+++ /etc/apt/sources.list.d/wikimedia-opensearch.list
@@ -1,2 +1,2 @@
-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1
-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1
+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2
+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2
- File[/usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service]
- Parameters differences:
--- File[/usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service].orig
+++ File[/usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service]
+ ensure => present
+ mode => 0555
+ owner => root
+ group => root
- Content differences:
--- /usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service.orig
+++ /usr/local/sbin/restart-opensearch_2@cloudelastic-chi-eqiad.service
@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/local/bin/safe-service-restart --pools cloudelastic-chi-https --services opensearch_2@cloudelastic-chi-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@
- Systemd::Unit[opensearch_1@.service]
- Parameters differences:
--- Systemd::Unit[opensearch_1@.service].orig
+++ Systemd::Unit[opensearch_1@.service]
- restart => False
- override_filename => puppet-override.conf
- ensure => present
- unit => opensearch_1@.service
- override => False
- require => ['Class[Systemd]']
- Service[opensearch_1@cloudelastic-chi-eqiad]
- Parameters differences:
--- Service[opensearch_1@cloudelastic-chi-eqiad].orig
+++ Service[opensearch_1@cloudelastic-chi-eqiad]
- tag => opensearch_services
- ensure => running
- enable => True
- require => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-chi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-chi-eqiad]']
- provider => systemd
- File[/usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service]
- Parameters differences:
--- File[/usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service].orig
+++ File[/usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service]
+ ensure => present
+ mode => 0555
+ owner => root
+ group => root
- Content differences:
--- /usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service.orig
+++ /usr/local/bin/pool-opensearch_2@cloudelastic-omega-eqiad.service
@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --pool --retries 10 --wait 5
- Apt::Repository[wikimedia-opensearch]
- Parameters differences:
--- Apt::Repository[wikimedia-opensearch].orig
+++ Apt::Repository[wikimedia-opensearch]
@@
- components => thirdparty/opensearch1
+ components => thirdparty/opensearch2
- File[/lib/systemd/system/opensearch_2@.service]
- Parameters differences:
--- File[/lib/systemd/system/opensearch_2@.service].orig
+++ File[/lib/systemd/system/opensearch_2@.service]
+ mode => 0444
+ owner => root
+ ensure => present
+ notify => Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]
+ group => root
- Content differences:
--- /lib/systemd/system/opensearch_2@.service.orig
+++ /lib/systemd/system/opensearch_2@.service
@@ -0,0 +1,73 @@
+[Unit]
+Description=OpenSearch (cluster %i)
+PartOf=opensearch.service
+Documentation=https://opensearch.org/docs
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=notify
+RuntimeDirectory=opensearch
+PrivateTmp=true
+Environment=OPENSEARCH_HOME=/usr/share/opensearch
+Environment=OPENSEARCH_PATH_CONF=/etc/opensearch/%i
+Environment=OPENSEARCH_JVM_OPTIONS=/etc/opensearch/%i/jvm.options
+Environment=PID_DIR=/run/opensearch-%i
+Environment=OPENSEARCH_SD_NOTIFY=true
+
+Environment=LD_LIBRARY_PATH=/usr/share/opensearch/plugins/opensearch-knn/lib
+Environment=JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64
+
+WorkingDirectory=/usr/share/opensearch
+
+User=opensearch
+Group=opensearch
+
+# Force creation of /run/opensearch (see T276198)
+ExecStartPre=/bin/systemd-tmpfiles --create /etc/tmpfiles.d/opensearch-%i.conf
+ExecStart=/usr/share/opensearch/bin/opensearch \
+ -p ${PID_DIR}/%i.pid \
+ --quiet \
+ -Epath.logs=/var/log/opensearch \
+ -Epath.data=/srv/opensearch/%i
+
+# StandardOutput is configured to redirect to journalctl since
+# some error messages may be logged in standard output before
+# opensearch logging system is initialized. OpenSearch
+# stores its logs in /var/log/opensearch and does not use
+# journalctl by default. If you also want to enable journalctl
+# logging, you can simply remove the "quiet" option from ExecStart.
+StandardOutput=journal
+StandardError=inherit
+
+# Specifies the maximum file descriptor number that can be opened by this process
+LimitNOFILE=65535
+
+# Specifies the maximum number of processes
+LimitNPROC=4096
+
+# Specifies the maximum size of virtual memory
+LimitAS=infinity
+
+# Specifies the maximum file size
+LimitFSIZE=infinity
+
+# Disable timeout logic and wait until process is stopped
+TimeoutStopSec=0
+
+# SIGTERM signal is used to stop the Java process
+KillSignal=SIGTERM
+
+# Send the signal only to the JVM rather than its control group
+KillMode=process
+
+# Java process is never killed
+SendSIGKILL=no
+
+# When a JVM receives a SIGTERM signal it exits with code 143
+SuccessExitStatus=143
+
+[Install]
+WantedBy=multi-user.target
+
+# Built for packages-7.4.1 (packages)
- Opensearch::Instance[cloudelastic-chi-eqiad]
- Parameters differences:
--- Opensearch::Instance[cloudelastic-chi-eqiad].orig
+++ Opensearch::Instance[cloudelastic-chi-eqiad]
@@
- version => 1.3.20
+ version => 2.19.5
- Package[wmf-opensearch-search-plugins]
- Parameters differences:
--- Package[wmf-opensearch-search-plugins].orig
+++ Package[wmf-opensearch-search-plugins]
@@
- before => ['Service[opensearch_1@cloudelastic-chi-eqiad]', 'Service[opensearch_1@cloudelastic-omega-eqiad]', 'Service[opensearch_1@cloudelastic-psi-eqiad]']
+ before => ['Service[opensearch_2@cloudelastic-chi-eqiad]', 'Service[opensearch_2@cloudelastic-omega-eqiad]', 'Service[opensearch_2@cloudelastic-psi-eqiad]']
- File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]
- Content differences:
--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig
+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list
@@ -1,2 +1,2 @@
-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13
-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13
+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2
+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2
- File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service]
- Parameters differences:
--- File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service].orig
+++ File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service]
- ensure => present
- mode => 0555
- owner => root
- group => root
- Content differences:
--- /usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service.orig
+++ /usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service
@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/local/bin/safe-service-restart --pools cloudelastic-chi-https --services opensearch_1@cloudelastic-chi-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@
- Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-chi-eqiad.service]
- Parameters differences:
--- Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-chi-eqiad.service].orig
+++ Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-chi-eqiad.service]
+ lvs_pools => ['cloudelastic-chi-https']
+ max_concurrency => 1
+ require => ['Class[Conftool::Scripts]']
- File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]
- Content differences:
--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig
+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml
@@ -100,6 +100,10 @@
#
#cluster.initial_master_nodes: ["node-1", "node-2"]
#
+# Set to ensure a node sees N other master eligible nodes to be considered
+# operational within the cluster. Its recommended to set it to a higher value
+# than 1 when running more than 2 nodes in the cluster.
+#
discovery.zen.ping.unicast.hosts: ["cloudelastic1009.eqiad.wmnet", "cloudelastic1010.eqiad.wmnet", "cloudelastic1011.eqiad.wmnet"]
#
# For more information, consult the discovery and cluster formation module documentation.
@@ -140,12 +144,16 @@
# 1k is too low to allow mwgrep to operate.
action.search.shard_count.limit: 5000
+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation
# Enable the disk space aware shard allocator
cluster.routing.allocation.disk.threshold_enabled: true
-# Stop allocating new shards to nodes with 75% disk utilization
+# Stop allocating shards to nodes (default 80%)
cluster.routing.allocation.disk.watermark.low: 0.75
-# Move shards off of nodes with 80% disk utilization
+# Begin evicting shards off the node (default 90%)
cluster.routing.allocation.disk.watermark.high: 0.80
+# Set indexes read-only with allocated shards on affected node (default 95%)
+cluster.routing.allocation.disk.watermark.flood_stage: 0.95
+
# Check disk utilization every 60 seconds
cluster.info.update.interval: 60s
@@ -191,3 +199,4 @@
ltr.caches.max_mem: 100mb
+
- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]
- Parameters differences:
--- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)].orig
+++ Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]
+ command => /bin/systemctl daemon-reload
+ refreshonly => True
- File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]
- Content differences:
--- /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties.orig
+++ /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties
@@ -185,7 +185,6 @@
rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
# Define Logstash appender
appender.ship_to_logstash.name=ship_to_logstash
- Package[opensearch]
- Parameters differences:
--- Package[opensearch].orig
+++ Package[opensearch]
@@
- ensure => 1.3.20
+ ensure => 2.19.5
- Systemd::Unit[opensearch_2@.service]
- Parameters differences:
--- Systemd::Unit[opensearch_2@.service].orig
+++ Systemd::Unit[opensearch_2@.service]
+ restart => False
+ override_filename => puppet-override.conf
+ ensure => present
+ unit => opensearch_2@.service
+ override => False
+ require => ['Class[Systemd]']
- Class[Profile::Opensearch::Server]
- Parameters differences:
--- Class[Profile::Opensearch::Server].orig
+++ Class[Profile::Opensearch::Server]
@@
- version => 1.3.20
+ version => 2.19.5
@@
- java_home => /usr/lib/jvm/java-11-openjdk-amd64
+ java_home => /usr/lib/jvm/java-21-openjdk-amd64
- File[/etc/opensearch/instances]
- Content differences:
--- /etc/opensearch/instances.orig
+++ /etc/opensearch/instances
@@ -1,3 +1,3 @@
-opensearch_1@cloudelastic-chi-eqiad
-opensearch_1@cloudelastic-omega-eqiad
-opensearch_1@cloudelastic-psi-eqiad
+opensearch_2@cloudelastic-chi-eqiad
+opensearch_2@cloudelastic-omega-eqiad
+opensearch_2@cloudelastic-psi-eqiad
- File[/usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service]
- Parameters differences:
--- File[/usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service].orig
+++ File[/usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service]
+ ensure => present
+ mode => 0555
+ owner => root
+ group => root
- Content differences:
--- /usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service.orig
+++ /usr/local/bin/depool-opensearch_2@cloudelastic-psi-eqiad.service
@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --depool --retries 10 --wait 5
- File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service]
- Parameters differences:
--- File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service].orig
+++ File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service]
- ensure => present
- mode => 0555
- owner => root
- group => root
- Content differences:
--- /usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service.orig
+++ /usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service
@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --services opensearch_1@cloudelastic-omega-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@
- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]
- Parameters differences:
--- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)].orig
+++ Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]
+ command => /bin/systemctl daemon-reload
+ refreshonly => True
- Service[opensearch_2@cloudelastic-chi-eqiad]
- Parameters differences:
--- Service[opensearch_2@cloudelastic-chi-eqiad].orig
+++ Service[opensearch_2@cloudelastic-chi-eqiad]
+ tag => opensearch_services
+ ensure => running
+ enable => True
+ require => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-chi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-chi-eqiad]']
+ provider => systemd
- File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service]
- Parameters differences:
--- File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service].orig
+++ File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service]
- ensure => present
- mode => 0555
- owner => root
- group => root
- Content differences:
--- /usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service.orig
+++ /usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service
@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --depool --retries 10 --wait 5
- Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-omega-eqiad.service]
- Parameters differences:
--- Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-omega-eqiad.service].orig
+++ Conftool::Scripts::Safe_service_restart[opensearch_2@cloudelastic-omega-eqiad.service]
+ lvs_pools => ['cloudelastic-omega-https']
+ max_concurrency => 1
+ require => ['Class[Conftool::Scripts]']
- File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]
- Parameters differences:
--- File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf].orig
+++ File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]
+ mode => 0444
+ owner => root
+ ensure => present
+ notify => Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]
+ group => root
- Content differences:
--- /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf.orig
+++ /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf
@@ -0,0 +1,24 @@
+# The OpenSearch packages come with a lot of plugins
+# we don't want or need. This override helps us
+# expose only the plugins we explicitly set via
+# $plugins_mandatory var
+[Service]
+# Give OpenSearch its own mount namespace
+PrivateMounts=yes
+
+# Replace plugins dir with an empty tmpfs
+TemporaryFileSystem=/usr/share/opensearch/plugins:rw
+
+BindPaths=
+BindPaths=/usr/share/opensearch/plugins/analysis-hebrew
+BindPaths=/usr/share/opensearch/plugins/analysis-icu
+BindPaths=/usr/share/opensearch/plugins/analysis-smartcn
+BindPaths=/usr/share/opensearch/plugins/analysis-stempel
+BindPaths=/usr/share/opensearch/plugins/analysis-ukrainian
+BindPaths=/usr/share/opensearch/plugins/cirrus-highlighter
+BindPaths=/usr/share/opensearch/plugins/opensearch-analysis-stconvert
+BindPaths=/usr/share/opensearch/plugins/opensearch-extra
+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-esperanto
+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-serbian
+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-slovak
+BindPaths=/usr/share/opensearch/plugins/ltr
- File[/etc/systemd/system/opensearch_2@.service.d]
- Parameters differences:
--- File[/etc/systemd/system/opensearch_2@.service.d].orig
+++ File[/etc/systemd/system/opensearch_2@.service.d]
+ ensure => directory
+ mode => 0555
+ owner => root
+ group => root
- Systemd::Override[opensearch_2@]
- Parameters differences:
--- Systemd::Override[opensearch_2@].orig
+++ Systemd::Override[opensearch_2@]
+ ensure => present
+ unit => opensearch_2@
+ restart => False
- File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]
- Content differences:
--- /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties.orig
+++ /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties
@@ -185,7 +185,6 @@
rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
# Define Logstash appender
appender.ship_to_logstash.name=ship_to_logstash
- Class[Profile::Lvs::Realserver::Ipip]
- Parameters differences:
--- Class[Profile::Lvs::Realserver::Ipip].orig
+++ Class[Profile::Lvs::Realserver::Ipip]
@@
- pools => {'cloudelastic-chi-https': {'services': ['opensearch_1@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_1@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_1@cloudelastic-omega-eqiad.service']}}
+ pools => {'cloudelastic-chi-https': {'services': ['opensearch_2@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_2@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_2@cloudelastic-omega-eqiad.service']}}
- File[/usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service]
- Parameters differences:
--- File[/usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service].orig
+++ File[/usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service]
+ ensure => present
+ mode => 0555
+ owner => root
+ group => root
- Content differences:
--- /usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service.orig
+++ /usr/local/sbin/restart-opensearch_2@cloudelastic-omega-eqiad.service
@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --services opensearch_2@cloudelastic-omega-eqiad.service --retries 10 --wait 5 --max-concurrency 1 $@
- File[/usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service]
- Parameters differences:
--- File[/usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service].orig
+++ File[/usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service]
+ ensure => present
+ mode => 0555
+ owner => root
+ group => root
- Content differences:
--- /usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service.orig
+++ /usr/local/bin/pool-opensearch_2@cloudelastic-psi-eqiad.service
@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/local/bin/safe-service-restart --pools cloudelastic-psi-https --pool --retries 10 --wait 5
- Apt::Repository[wikimedia-opensearch-plugins]
- Parameters differences:
--- Apt::Repository[wikimedia-opensearch-plugins].orig
+++ Apt::Repository[wikimedia-opensearch-plugins]
@@
- components => component/opensearch13
+ components => component/opensearch2
- File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]
- Content differences:
--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig
+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml
@@ -100,6 +100,10 @@
#
#cluster.initial_master_nodes: ["node-1", "node-2"]
#
+# Set to ensure a node sees N other master eligible nodes to be considered
+# operational within the cluster. Its recommended to set it to a higher value
+# than 1 when running more than 2 nodes in the cluster.
+#
discovery.zen.ping.unicast.hosts: ["cloudelastic1009.eqiad.wmnet", "cloudelastic1010.eqiad.wmnet", "cloudelastic1011.eqiad.wmnet"]
#
# For more information, consult the discovery and cluster formation module documentation.
@@ -140,12 +144,16 @@
# 1k is too low to allow mwgrep to operate.
action.search.shard_count.limit: 5000
+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation
# Enable the disk space aware shard allocator
cluster.routing.allocation.disk.threshold_enabled: true
-# Stop allocating new shards to nodes with 75% disk utilization
+# Stop allocating shards to nodes (default 80%)
cluster.routing.allocation.disk.watermark.low: 0.75
-# Move shards off of nodes with 80% disk utilization
+# Begin evicting shards off the node (default 90%)
cluster.routing.allocation.disk.watermark.high: 0.80
+# Set indexes read-only with allocated shards on affected node (default 95%)
+cluster.routing.allocation.disk.watermark.flood_stage: 0.95
+
# Check disk utilization every 60 seconds
cluster.info.update.interval: 60s
@@ -191,3 +199,4 @@
ltr.caches.max_mem: 100mb
+
- Class[Profile::Lvs::Realserver]
- Parameters differences:
--- Class[Profile::Lvs::Realserver].orig
+++ Class[Profile::Lvs::Realserver]
@@
- pools => {'cloudelastic-chi-https': {'services': ['opensearch_1@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_1@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_1@cloudelastic-omega-eqiad.service']}}
+ pools => {'cloudelastic-chi-https': {'services': ['opensearch_2@cloudelastic-chi-eqiad.service']}, 'cloudelastic-psi-https': {'services': ['opensearch_2@cloudelastic-psi-eqiad.service']}, 'cloudelastic-omega-https': {'services': ['opensearch_2@cloudelastic-omega-eqiad.service']}}
- Opensearch::Instance[cloudelastic-omega-eqiad]
- Parameters differences:
--- Opensearch::Instance[cloudelastic-omega-eqiad].orig
+++ Opensearch::Instance[cloudelastic-omega-eqiad]
@@
- version => 1.3.20
+ version => 2.19.5
- File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service]
- Parameters differences:
--- File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service].orig
+++ File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service]
- ensure => present
- mode => 0555
- owner => root
- group => root
- Content differences:
--- /usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service.orig
+++ /usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service
@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --depool --retries 10 --wait 5
- Class[Opensearch]
- Parameters differences:
--- Class[Opensearch].orig
+++ Class[Opensearch]
@@
- version => 1.3.20
+ version => 2.19.5
@@
- java_home => /usr/lib/jvm/java-11-openjdk-amd64
+ java_home => /usr/lib/jvm/java-21-openjdk-amd64
- File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service]
- Parameters differences:
--- File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service].orig
+++ File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service]
- ensure => present
- mode => 0555
- owner => root
- group => root
- Content differences:
--- /usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service.orig
+++ /usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service
@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/local/bin/safe-service-restart --pools cloudelastic-omega-https --pool --retries 10 --wait 5
- Conftool::Scripts::Safe_service_restart[opensearch_1@cloudelastic-chi-eqiad.service]