--- Service[opensearch_1@cloudelastic-omega-eqiad].orig
+++ Service[opensearch_1@cloudelastic-omega-eqiad]
- enable => True
- tag => opensearch_services
- ensure => running
- provider => systemd
- require => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-omega-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-omega-eqiad]']
Package[wmf-opensearch-search-plugins]
- Parameters differences:
--- Package[wmf-opensearch-search-plugins].orig
+++ Package[wmf-opensearch-search-plugins]
@@
- before => ['Service[opensearch_1@cloudelastic-chi-eqiad]', 'Service[opensearch_1@cloudelastic-omega-eqiad]', 'Service[opensearch_1@cloudelastic-psi-eqiad]']
+ before => ['Service[opensearch_2@cloudelastic-chi-eqiad]', 'Service[opensearch_2@cloudelastic-omega-eqiad]', 'Service[opensearch_2@cloudelastic-psi-eqiad]']
- File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]
- Content differences:
--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig
+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml
@@ -100,6 +100,10 @@
#
#cluster.initial_master_nodes: ["node-1", "node-2"]
#
+# Set to ensure a node sees N other master eligible nodes to be considered
+# operational within the cluster. Its recommended to set it to a higher value
+# than 1 when running more than 2 nodes in the cluster.
+#
discovery.zen.ping.unicast.hosts: ["cloudelastic1009.eqiad.wmnet", "cloudelastic1010.eqiad.wmnet", "cloudelastic1011.eqiad.wmnet"]
#
# For more information, consult the discovery and cluster formation module documentation.
@@ -142,12 +146,16 @@
# 1k is too low to allow mwgrep to operate.
action.search.shard_count.limit: 5000
+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation
# Enable the disk space aware shard allocator
cluster.routing.allocation.disk.threshold_enabled: true
-# Stop allocating new shards to nodes with 75% disk utilization
+# Stop allocating shards to nodes (default 80%)
cluster.routing.allocation.disk.watermark.low: 0.75
-# Move shards off of nodes with 80% disk utilization
+# Begin evicting shards off the node (default 90%)
cluster.routing.allocation.disk.watermark.high: 0.80
+# Set indexes read-only with allocated shards on affected node (default 95%)
+cluster.routing.allocation.disk.watermark.flood_stage: 0.95
+
# Check disk utilization every 60 seconds
cluster.info.update.interval: 60s
@@ -193,3 +201,4 @@
ltr.caches.max_mem: 100mb
+
- File[/etc/opensearch/instances]
- Content differences:
--- /etc/opensearch/instances.orig
+++ /etc/opensearch/instances
@@ -1,3 +1,3 @@
-opensearch_1@cloudelastic-chi-eqiad
-opensearch_1@cloudelastic-omega-eqiad
-opensearch_1@cloudelastic-psi-eqiad
+opensearch_2@cloudelastic-chi-eqiad
+opensearch_2@cloudelastic-omega-eqiad
+opensearch_2@cloudelastic-psi-eqiad
- File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]
- Content differences:
--- /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties.orig
+++ /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties
@@ -185,7 +185,6 @@
rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
# Define Logstash appender
appender.ship_to_logstash.name=ship_to_logstash
- Service[opensearch_2@cloudelastic-omega-eqiad]
- Parameters differences:
--- Service[opensearch_2@cloudelastic-omega-eqiad].orig
+++ Service[opensearch_2@cloudelastic-omega-eqiad]
+ enable => True
+ tag => opensearch_services
+ ensure => running
+ provider => systemd
+ require => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-omega-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-omega-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-omega-eqiad]']
- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]
- Parameters differences:
--- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)].orig
+++ Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]
+ refreshonly => True
+ command => /bin/systemctl daemon-reload
- Systemd::Unit[opensearch_2@.service]
- Parameters differences:
--- Systemd::Unit[opensearch_2@.service].orig
+++ Systemd::Unit[opensearch_2@.service]
+ override => False
+ ensure => present
+ override_filename => puppet-override.conf
+ restart => False
+ require => ['Class[Systemd]']
+ unit => opensearch_2@.service
- Opensearch::Instance[cloudelastic-omega-eqiad]
- Parameters differences:
--- Opensearch::Instance[cloudelastic-omega-eqiad].orig
+++ Opensearch::Instance[cloudelastic-omega-eqiad]
@@
- version => 1.3.20
+ version => 2.19.5
- Systemd::Unit[opensearch_1@.service]
- Parameters differences:
--- Systemd::Unit[opensearch_1@.service].orig
+++ Systemd::Unit[opensearch_1@.service]
- override => False
- ensure => present
- override_filename => puppet-override.conf
- restart => False
- require => ['Class[Systemd]']
- unit => opensearch_1@.service
- Class[Profile::Opensearch::Server]
- Parameters differences:
--- Class[Profile::Opensearch::Server].orig
+++ Class[Profile::Opensearch::Server]
@@
- version => 1.3.20
+ version => 2.19.5
@@
- java_home => /usr/lib/jvm/java-11-openjdk-amd64
+ java_home => /usr/lib/jvm/java21-openjdk-amd64
- Class[Profile::Opensearch::Cirrus::Server]
- Parameters differences:
--- Class[Profile::Opensearch::Cirrus::Server].orig
+++ Class[Profile::Opensearch::Cirrus::Server]
+ version => 2.19.5
+ apt_component => 13
- File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]
- Content differences:
--- /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties.orig
+++ /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties
@@ -185,7 +185,6 @@
rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
# Define Logstash appender
appender.ship_to_logstash.name=ship_to_logstash
- File[/etc/apt/sources.list.d/wikimedia-opensearch.list]
- Content differences:
--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig
+++ /etc/apt/sources.list.d/wikimedia-opensearch.list
@@ -1,2 +1,2 @@
-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1
-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1
+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2
+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2
- File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]
- Content differences:
--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig
+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml
@@ -100,6 +100,10 @@
#
#cluster.initial_master_nodes: ["node-1", "node-2"]
#
+# Set to ensure a node sees N other master eligible nodes to be considered
+# operational within the cluster. Its recommended to set it to a higher value
+# than 1 when running more than 2 nodes in the cluster.
+#
discovery.zen.ping.unicast.hosts: ["cloudelastic1009.eqiad.wmnet", "cloudelastic1010.eqiad.wmnet", "cloudelastic1011.eqiad.wmnet"]
#
# For more information, consult the discovery and cluster formation module documentation.
@@ -142,12 +146,16 @@
# 1k is too low to allow mwgrep to operate.
action.search.shard_count.limit: 5000
+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation
# Enable the disk space aware shard allocator
cluster.routing.allocation.disk.threshold_enabled: true
-# Stop allocating new shards to nodes with 75% disk utilization
+# Stop allocating shards to nodes (default 80%)
cluster.routing.allocation.disk.watermark.low: 0.75
-# Move shards off of nodes with 80% disk utilization
+# Begin evicting shards off the node (default 90%)
cluster.routing.allocation.disk.watermark.high: 0.80
+# Set indexes read-only with allocated shards on affected node (default 95%)
+cluster.routing.allocation.disk.watermark.flood_stage: 0.95
+
# Check disk utilization every 60 seconds
cluster.info.update.interval: 60s
@@ -193,3 +201,4 @@
ltr.caches.max_mem: 100mb
+
- File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]
- Content differences:
--- /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties.orig
+++ /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties
@@ -185,7 +185,6 @@
rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash
logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
# Define Logstash appender
appender.ship_to_logstash.name=ship_to_logstash
- File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]
- Content differences:
--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig
+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list
@@ -1,2 +1,2 @@
-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13
-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13
+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia opensearch{13}
+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia opensearch{13}
- Opensearch::Instance[cloudelastic-psi-eqiad]
- Parameters differences:
--- Opensearch::Instance[cloudelastic-psi-eqiad].orig
+++ Opensearch::Instance[cloudelastic-psi-eqiad]
@@
- version => 1.3.20
+ version => 2.19.5
- Service[opensearch_1@cloudelastic-psi-eqiad]
- Parameters differences:
--- Service[opensearch_1@cloudelastic-psi-eqiad].orig
+++ Service[opensearch_1@cloudelastic-psi-eqiad]
- enable => True
- tag => opensearch_services
- ensure => running
- provider => systemd
- require => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-psi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-psi-eqiad]']
- Package[opensearch]
- Parameters differences:
--- Package[opensearch].orig
+++ Package[opensearch]
@@
- ensure => 1.3.20
+ ensure => 2.19.5
- Class[Opensearch::Packages]
- Parameters differences:
--- Class[Opensearch::Packages].orig
+++ Class[Opensearch::Packages]
@@
- version => 1.3.20
+ version => 2.19.5
- Service[opensearch_2@cloudelastic-chi-eqiad]
- Parameters differences:
--- Service[opensearch_2@cloudelastic-chi-eqiad].orig
+++ Service[opensearch_2@cloudelastic-chi-eqiad]
+ enable => True
+ tag => opensearch_services
+ ensure => running
+ provider => systemd
+ require => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-chi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-chi-eqiad]']
- Apt::Repository[wikimedia-opensearch-plugins]
- Parameters differences:
--- Apt::Repository[wikimedia-opensearch-plugins].orig
+++ Apt::Repository[wikimedia-opensearch-plugins]
@@
- components => component/opensearch13
+ components => opensearch{13}
- Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]
- Parameters differences:
--- Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)].orig
+++ Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]
- refreshonly => True
- command => /bin/systemctl daemon-reload
- Service[opensearch_2@cloudelastic-psi-eqiad]
- Parameters differences:
--- Service[opensearch_2@cloudelastic-psi-eqiad].orig
+++ Service[opensearch_2@cloudelastic-psi-eqiad]
+ enable => True
+ tag => opensearch_services
+ ensure => running
+ provider => systemd
+ require => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-psi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-psi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-psi-eqiad]']
- Service[opensearch_1@cloudelastic-chi-eqiad]
- Parameters differences:
--- Service[opensearch_1@cloudelastic-chi-eqiad].orig
+++ Service[opensearch_1@cloudelastic-chi-eqiad]
- enable => True
- tag => opensearch_services
- ensure => running
- provider => systemd
- require => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/logging.yml]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]', 'File[/etc/opensearch/cloudelastic-chi-eqiad/jvm.options]', 'File[/srv/opensearch/cloudelastic-chi-eqiad]', 'Systemd::Tmpfile[opensearch-cloudelastic-chi-eqiad]']