Compilation results for cirrussearch1090.eqiad.wmnet: System changes detected

Catalog differences

Summary

Resources modified

• Class[Profile::Opensearch::Server]

  Parameters differences:

  --- Class[Profile::Opensearch::Server].orig
  +++ Class[Profile::Opensearch::Server]
  
  @@
  -    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}
  +    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}
  

• Opensearch::Instance[production-search-eqiad]

  Parameters differences:

  --- Opensearch::Instance[production-search-eqiad].orig
  +++ Opensearch::Instance[production-search-eqiad]
  
  @@
  -    watermarks => {}
  +    watermarks => {'low': '75%', 'high': '80%', 'flood_stage': '95%'}
  

• Class[Opensearch]

  Parameters differences:

  --- Class[Opensearch].orig
  +++ Class[Opensearch]
  
  @@
  -    instances => {'production-search-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch1122.eqiad.wmnet', 'cirrussearch1074.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1094.eqiad.wmnet', 'cirrussearch1100.eqiad.wmnet'], 'cluster_hosts': ['cirrussearch1068.eqiad.wmnet', 'cirrussearch1069.eqiad.wmnet', 'cirrussearch1070.eqiad.wmnet', 'cirrussearch1071.eqiad.wmnet', 'cirrussearch1072.eqiad.wmnet', 'cirrussearch1073.eqiad.wmnet', 'cirrussearch1074.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1076.eqiad.wmnet', 'cirrussearch1077.eqiad.wmnet', 'cirrussearch1078.eqiad.wmnet', 'cirrussearch1079.eqiad.wmnet', 'cirrussearch1080.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1082.eqiad.wmnet', 'cirrussearch1083.eqiad.wmnet', 'cirrussearch1084.eqiad.wmnet', 'cirrussearch1085.eqiad.wmnet', 'cirrussearch1086.eqiad.wmnet', 'cirrussearch1087.eqiad.wmnet', 'cirrussearch1088.eqiad.wmnet', 'cirrussearch1089.eqiad.wmnet', 'cirrussearch1090.eqiad.wmnet', 'cirrussearch1091.eqiad.wmnet', 'cirrussearch1092.eqiad.wmnet', 'cirrussearch1093.eqiad.wmnet', 'cirrussearch1094.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1096.eqiad.wmnet', 'cirrussearch1097.eqiad.wmnet', 'cirrussearch1098.eqiad.wmnet', 'cirrussearch1099.eqiad.wmnet', 'cirrussearch1100.eqiad.wmnet', 'cirrussearch1101.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet', 'cirrussearch1103.eqiad.wmnet', 'cirrussearch1107.eqiad.wmnet', 'cirrussearch1108.eqiad.wmnet', 'cirrussearch1109.eqiad.wmnet', 'cirrussearch1110.eqiad.wmnet', 'cirrussearch1111.eqiad.wmnet', 'cirrussearch1112.eqiad.wmnet', 'cirrussearch1113.eqiad.wmnet', 'cirrussearch1114.eqiad.wmnet', 'cirrussearch1115.eqiad.wmnet', 'cirrussearch1116.eqiad.wmnet', 'cirrussearch1117.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1119.eqiad.wmnet', 'cirrussearch1120.eqiad.wmnet', 'cirrussearch1121.eqiad.wmnet', 'cirrussearch1122.eqiad.wmnet', 'cirrussearch1123.eqiad.wmnet', 'cirrussearch1124.eqiad.wmnet', 'cirrussearch1125.eqiad.wmnet'], 'expected_nodes': 35, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9700, 'unicast_hosts': ['cirrussearch1073.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet'], 'cluster_hosts': ['cirrussearch1069.eqiad.wmnet', 'cirrussearch1072.eqiad.wmnet', 'cirrussearch1073.eqiad.wmnet', 'cirrussearch1084.eqiad.wmnet', 'cirrussearch1111.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1078.eqiad.wmnet', 'cirrussearch1079.eqiad.wmnet', 'cirrussearch1085.eqiad.wmnet', 'cirrussearch1086.eqiad.wmnet', 'cirrussearch1115.eqiad.wmnet', 'cirrussearch1116.eqiad.wmnet', 'cirrussearch1117.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1083.eqiad.wmnet', 'cirrussearch1087.eqiad.wmnet', 'cirrussearch1088.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1121.eqiad.wmnet', 'cirrussearch1122.eqiad.wmnet', 'cirrussearch1090.eqiad.wmnet', 'cirrussearch1092.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1108.eqiad.wmnet', 'cirrussearch1123.eqiad.wmnet', 'cirrussearch1097.eqiad.wmnet', 'cirrussearch1099.eqiad.wmnet', 'cirrussearch1101.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet', 'cirrussearch1110.eqiad.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}
  +    instances => {'production-search-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch1122.eqiad.wmnet', 'cirrussearch1074.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1094.eqiad.wmnet', 'cirrussearch1100.eqiad.wmnet'], 'cluster_hosts': ['cirrussearch1068.eqiad.wmnet', 'cirrussearch1069.eqiad.wmnet', 'cirrussearch1070.eqiad.wmnet', 'cirrussearch1071.eqiad.wmnet', 'cirrussearch1072.eqiad.wmnet', 'cirrussearch1073.eqiad.wmnet', 'cirrussearch1074.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1076.eqiad.wmnet', 'cirrussearch1077.eqiad.wmnet', 'cirrussearch1078.eqiad.wmnet', 'cirrussearch1079.eqiad.wmnet', 'cirrussearch1080.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1082.eqiad.wmnet', 'cirrussearch1083.eqiad.wmnet', 'cirrussearch1084.eqiad.wmnet', 'cirrussearch1085.eqiad.wmnet', 'cirrussearch1086.eqiad.wmnet', 'cirrussearch1087.eqiad.wmnet', 'cirrussearch1088.eqiad.wmnet', 'cirrussearch1089.eqiad.wmnet', 'cirrussearch1090.eqiad.wmnet', 'cirrussearch1091.eqiad.wmnet', 'cirrussearch1092.eqiad.wmnet', 'cirrussearch1093.eqiad.wmnet', 'cirrussearch1094.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1096.eqiad.wmnet', 'cirrussearch1097.eqiad.wmnet', 'cirrussearch1098.eqiad.wmnet', 'cirrussearch1099.eqiad.wmnet', 'cirrussearch1100.eqiad.wmnet', 'cirrussearch1101.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet', 'cirrussearch1103.eqiad.wmnet', 'cirrussearch1107.eqiad.wmnet', 'cirrussearch1108.eqiad.wmnet', 'cirrussearch1109.eqiad.wmnet', 'cirrussearch1110.eqiad.wmnet', 'cirrussearch1111.eqiad.wmnet', 'cirrussearch1112.eqiad.wmnet', 'cirrussearch1113.eqiad.wmnet', 'cirrussearch1114.eqiad.wmnet', 'cirrussearch1115.eqiad.wmnet', 'cirrussearch1116.eqiad.wmnet', 'cirrussearch1117.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1119.eqiad.wmnet', 'cirrussearch1120.eqiad.wmnet', 'cirrussearch1121.eqiad.wmnet', 'cirrussearch1122.eqiad.wmnet', 'cirrussearch1123.eqiad.wmnet', 'cirrussearch1124.eqiad.wmnet', 'cirrussearch1125.eqiad.wmnet'], 'expected_nodes': 35, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9700, 'unicast_hosts': ['cirrussearch1073.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet'], 'cluster_hosts': ['cirrussearch1069.eqiad.wmnet', 'cirrussearch1072.eqiad.wmnet', 'cirrussearch1073.eqiad.wmnet', 'cirrussearch1084.eqiad.wmnet', 'cirrussearch1111.eqiad.wmnet', 'cirrussearch1075.eqiad.wmnet', 'cirrussearch1078.eqiad.wmnet', 'cirrussearch1079.eqiad.wmnet', 'cirrussearch1085.eqiad.wmnet', 'cirrussearch1086.eqiad.wmnet', 'cirrussearch1115.eqiad.wmnet', 'cirrussearch1116.eqiad.wmnet', 'cirrussearch1117.eqiad.wmnet', 'cirrussearch1081.eqiad.wmnet', 'cirrussearch1083.eqiad.wmnet', 'cirrussearch1087.eqiad.wmnet', 'cirrussearch1088.eqiad.wmnet', 'cirrussearch1118.eqiad.wmnet', 'cirrussearch1121.eqiad.wmnet', 'cirrussearch1122.eqiad.wmnet', 'cirrussearch1090.eqiad.wmnet', 'cirrussearch1092.eqiad.wmnet', 'cirrussearch1095.eqiad.wmnet', 'cirrussearch1108.eqiad.wmnet', 'cirrussearch1123.eqiad.wmnet', 'cirrussearch1097.eqiad.wmnet', 'cirrussearch1099.eqiad.wmnet', 'cirrussearch1101.eqiad.wmnet', 'cirrussearch1102.eqiad.wmnet', 'cirrussearch1110.eqiad.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}
  

• Opensearch::Instance[production-search-psi-eqiad]

  Parameters differences:

  --- Opensearch::Instance[production-search-psi-eqiad].orig
  +++ Opensearch::Instance[production-search-psi-eqiad]
  
  @@
  -    watermarks => {}
  +    watermarks => {'low': '75%', 'high': '80%', 'flood_stage': '95%'}
  

• File[/etc/opensearch/production-search-eqiad/opensearch.yml]

  Content differences:

  --- /etc/opensearch/production-search-eqiad/opensearch.yml.orig
  +++ /etc/opensearch/production-search-eqiad/opensearch.yml
  @@ -142,10 +142,12 @@
   
   # Enable the disk space aware shard allocator
   cluster.routing.allocation.disk.threshold_enabled: true
  -# Stop allocating new shards to nodes with 75% disk utilization
  -cluster.routing.allocation.disk.watermark.low: 0.75
  -# Move shards off of nodes with 80% disk utilization
  -cluster.routing.allocation.disk.watermark.high: 0.80
  +# Stop allocating new shards to nodes when this threshold is reached
  +cluster.routing.allocation.disk.watermark.low: 75%
  +# Move shards off of nodes when this threshold is reached
  +cluster.routing.allocation.disk.watermark.high: 80%
  +# Set indexes read-only with allocated shards on affected node when this threshold is reached
  +cluster.routing.allocation.disk.watermark.flood_stage: 95%
   # Check disk utilization every 60 seconds
   cluster.info.update.interval: 60s
   

• File[/etc/opensearch/production-search-psi-eqiad/opensearch.yml]

  Content differences:

  --- /etc/opensearch/production-search-psi-eqiad/opensearch.yml.orig
  +++ /etc/opensearch/production-search-psi-eqiad/opensearch.yml
  @@ -142,10 +142,12 @@
   
   # Enable the disk space aware shard allocator
   cluster.routing.allocation.disk.threshold_enabled: true
  -# Stop allocating new shards to nodes with 75% disk utilization
  -cluster.routing.allocation.disk.watermark.low: 0.75
  -# Move shards off of nodes with 80% disk utilization
  -cluster.routing.allocation.disk.watermark.high: 0.80
  +# Stop allocating new shards to nodes when this threshold is reached
  +cluster.routing.allocation.disk.watermark.low: 75%
  +# Move shards off of nodes when this threshold is reached
  +cluster.routing.allocation.disk.watermark.high: 80%
  +# Set indexes read-only with allocated shards on affected node when this threshold is reached
  +cluster.routing.allocation.disk.watermark.flood_stage: 95%
   # Check disk utilization every 60 seconds
   cluster.info.update.interval: 60s
   

Relevant files

• Production catalog
• Change catalog
• Production errors/warnings
• Change errors/warnings
• Core Diff
• Full Diff