--- Class[Opensearch].orig
+++ Class[Opensearch]
@@
- instances => {'production-search-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch1122.eqiad.wmnet', 'cirrussearch1074.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1094.eqiad.wmnet', 'cirrussearch1100.eqiad.wmnet'], 'cluster_hosts': ['cirrussearch1068.eqiad.wmnet', 'cirrussearch1069.eqiad.wmnet', 'cirrussearch1070.eqiad.wmnet', 'cirrussearch1071.eqiad.wmnet', 'cirrussearch1072.eqiad.wmnet', 'cirrussearch1073.eqiad.wmnet', 'cirrussearch1074.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1076.eqiad.wmnet', 'cirrussearch1077.eqiad.wmnet', 'cirrussearch1078.eqiad.wmnet', 'cirrussearch1079.eqiad.wmnet', 'cirrussearch1080.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1082.eqiad.wmnet', 'cirrussearch1083.eqiad.wmnet', 'cirrussearch1084.eqiad.wmnet', 'cirrussearch1085.eqiad.wmnet', 'cirrussearch1086.eqiad.wmnet', 'cirrussearch1087.eqiad.wmnet', 'cirrussearch1088.eqiad.wmnet', 'cirrussearch1089.eqiad.wmnet', 'cirrussearch1090.eqiad.wmnet', 'cirrussearch1091.eqiad.wmnet', 'cirrussearch1092.eqiad.wmnet', 'cirrussearch1093.eqiad.wmnet', 'cirrussearch1094.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1096.eqiad.wmnet', 'cirrussearch1097.eqiad.wmnet', 'cirrussearch1098.eqiad.wmnet', 'cirrussearch1099.eqiad.wmnet', 'cirrussearch1100.eqiad.wmnet', 'cirrussearch1101.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet', 'cirrussearch1103.eqiad.wmnet', 'cirrussearch1107.eqiad.wmnet', 'cirrussearch1108.eqiad.wmnet', 'cirrussearch1109.eqiad.wmnet', 'cirrussearch1110.eqiad.wmnet', 'cirrussearch1111.eqiad.wmnet', 'cirrussearch1112.eqiad.wmnet', 'cirrussearch1113.eqiad.wmnet', 'cirrussearch1114.eqiad.wmnet', 'cirrussearch1115.eqiad.wmnet', 'cirrussearch1116.eqiad.wmnet', 'cirrussearch1117.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1119.eqiad.wmnet', 'cirrussearch1120.eqiad.wmnet', 'cirrussearch1121.eqiad.wmnet', 'cirrussearch1122.eqiad.wmnet', 'cirrussearch1123.eqiad.wmnet', 'cirrussearch1124.eqiad.wmnet', 'cirrussearch1125.eqiad.wmnet'], 'expected_nodes': 35, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9700, 'unicast_hosts': ['cirrussearch1073.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet'], 'cluster_hosts': ['cirrussearch1069.eqiad.wmnet', 'cirrussearch1072.eqiad.wmnet', 'cirrussearch1073.eqiad.wmnet', 'cirrussearch1084.eqiad.wmnet', 'cirrussearch1111.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1078.eqiad.wmnet', 'cirrussearch1079.eqiad.wmnet', 'cirrussearch1085.eqiad.wmnet', 'cirrussearch1086.eqiad.wmnet', 'cirrussearch1115.eqiad.wmnet', 'cirrussearch1116.eqiad.wmnet', 'cirrussearch1117.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1083.eqiad.wmnet', 'cirrussearch1087.eqiad.wmnet', 'cirrussearch1088.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1121.eqiad.wmnet', 'cirrussearch1122.eqiad.wmnet', 'cirrussearch1090.eqiad.wmnet', 'cirrussearch1092.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1108.eqiad.wmnet', 'cirrussearch1123.eqiad.wmnet', 'cirrussearch1097.eqiad.wmnet', 'cirrussearch1099.eqiad.wmnet', 'cirrussearch1101.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet', 'cirrussearch1110.eqiad.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}
+ instances => {'production-search-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch1122.eqiad.wmnet', 'cirrussearch1074.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1094.eqiad.wmnet', 'cirrussearch1100.eqiad.wmnet'], 'cluster_hosts': ['cirrussearch1068.eqiad.wmnet', 'cirrussearch1069.eqiad.wmnet', 'cirrussearch1070.eqiad.wmnet', 'cirrussearch1071.eqiad.wmnet', 'cirrussearch1072.eqiad.wmnet', 'cirrussearch1073.eqiad.wmnet', 'cirrussearch1074.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1076.eqiad.wmnet', 'cirrussearch1077.eqiad.wmnet', 'cirrussearch1078.eqiad.wmnet', 'cirrussearch1079.eqiad.wmnet', 'cirrussearch1080.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1082.eqiad.wmnet', 'cirrussearch1083.eqiad.wmnet', 'cirrussearch1084.eqiad.wmnet', 'cirrussearch1085.eqiad.wmnet', 'cirrussearch1086.eqiad.wmnet', 'cirrussearch1087.eqiad.wmnet', 'cirrussearch1088.eqiad.wmnet', 'cirrussearch1089.eqiad.wmnet', 'cirrussearch1090.eqiad.wmnet', 'cirrussearch1091.eqiad.wmnet', 'cirrussearch1092.eqiad.wmnet', 'cirrussearch1093.eqiad.wmnet', 'cirrussearch1094.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1096.eqiad.wmnet', 'cirrussearch1097.eqiad.wmnet', 'cirrussearch1098.eqiad.wmnet', 'cirrussearch1099.eqiad.wmnet', 'cirrussearch1100.eqiad.wmnet', 'cirrussearch1101.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet', 'cirrussearch1103.eqiad.wmnet', 'cirrussearch1107.eqiad.wmnet', 'cirrussearch1108.eqiad.wmnet', 'cirrussearch1109.eqiad.wmnet', 'cirrussearch1110.eqiad.wmnet', 'cirrussearch1111.eqiad.wmnet', 'cirrussearch1112.eqiad.wmnet', 'cirrussearch1113.eqiad.wmnet', 'cirrussearch1114.eqiad.wmnet', 'cirrussearch1115.eqiad.wmnet', 'cirrussearch1116.eqiad.wmnet', 'cirrussearch1117.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1119.eqiad.wmnet', 'cirrussearch1120.eqiad.wmnet', 'cirrussearch1121.eqiad.wmnet', 'cirrussearch1122.eqiad.wmnet', 'cirrussearch1123.eqiad.wmnet', 'cirrussearch1124.eqiad.wmnet', 'cirrussearch1125.eqiad.wmnet'], 'expected_nodes': 35, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9700, 'unicast_hosts': ['cirrussearch1073.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet'], 'cluster_hosts': ['cirrussearch1069.eqiad.wmnet', 'cirrussearch1072.eqiad.wmnet', 'cirrussearch1073.eqiad.wmnet', 'cirrussearch1084.eqiad.wmnet', 'cirrussearch1111.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1078.eqiad.wmnet', 'cirrussearch1079.eqiad.wmnet', 'cirrussearch1085.eqiad.wmnet', 'cirrussearch1086.eqiad.wmnet', 'cirrussearch1115.eqiad.wmnet', 'cirrussearch1116.eqiad.wmnet', 'cirrussearch1117.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1083.eqiad.wmnet', 'cirrussearch1087.eqiad.wmnet', 'cirrussearch1088.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1121.eqiad.wmnet', 'cirrussearch1122.eqiad.wmnet', 'cirrussearch1090.eqiad.wmnet', 'cirrussearch1092.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1108.eqiad.wmnet', 'cirrussearch1123.eqiad.wmnet', 'cirrussearch1097.eqiad.wmnet', 'cirrussearch1099.eqiad.wmnet', 'cirrussearch1101.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet', 'cirrussearch1110.eqiad.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}
Opensearch::Instance[production-search-psi-eqiad]
- Parameters differences:
--- Opensearch::Instance[production-search-psi-eqiad].orig
+++ Opensearch::Instance[production-search-psi-eqiad]
@@
- watermarks => {}
+ watermarks => {'low': '75%', 'high': '80%', 'flood_stage': '95%'}
- File[/etc/opensearch/production-search-eqiad/opensearch.yml]
- Content differences:
--- /etc/opensearch/production-search-eqiad/opensearch.yml.orig
+++ /etc/opensearch/production-search-eqiad/opensearch.yml
@@ -142,10 +142,12 @@
# Enable the disk space aware shard allocator
cluster.routing.allocation.disk.threshold_enabled: true
-# Stop allocating new shards to nodes with 75% disk utilization
-cluster.routing.allocation.disk.watermark.low: 0.75
-# Move shards off of nodes with 80% disk utilization
-cluster.routing.allocation.disk.watermark.high: 0.80
+# Stop allocating new shards to nodes when this threshold is reached
+cluster.routing.allocation.disk.watermark.low: 75%
+# Move shards off of nodes when this threshold is reached
+cluster.routing.allocation.disk.watermark.high: 80%
+# Set indexes read-only with allocated shards on affected node when this threshold is reached
+cluster.routing.allocation.disk.watermark.flood_stage: 95%
# Check disk utilization every 60 seconds
cluster.info.update.interval: 60s
- File[/etc/opensearch/production-search-psi-eqiad/opensearch.yml]
- Content differences:
--- /etc/opensearch/production-search-psi-eqiad/opensearch.yml.orig
+++ /etc/opensearch/production-search-psi-eqiad/opensearch.yml
@@ -142,10 +142,12 @@
# Enable the disk space aware shard allocator
cluster.routing.allocation.disk.threshold_enabled: true
-# Stop allocating new shards to nodes with 75% disk utilization
-cluster.routing.allocation.disk.watermark.low: 0.75
-# Move shards off of nodes with 80% disk utilization
-cluster.routing.allocation.disk.watermark.high: 0.80
+# Stop allocating new shards to nodes when this threshold is reached
+cluster.routing.allocation.disk.watermark.low: 75%
+# Move shards off of nodes when this threshold is reached
+cluster.routing.allocation.disk.watermark.high: 80%
+# Set indexes read-only with allocated shards on affected node when this threshold is reached
+cluster.routing.allocation.disk.watermark.flood_stage: 95%
# Check disk utilization every 60 seconds
cluster.info.update.interval: 60s
Relevant files