{"host": "cirrussearch2100.codfw.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3588, "only_in_self": ["Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "File[/lib/systemd/system/opensearch_1@.service]", "Service[opensearch_1@production-search-codfw]", "Service[opensearch_1@production-search-omega-codfw]", "Systemd::Unit[opensearch_1@.service]"], "only_in_other": ["Exec[install-opensearch]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "File[/etc/systemd/system/opensearch_2@.service.d]", "File[/lib/systemd/system/opensearch_2@.service]", "Service[opensearch_2@production-search-codfw]", "Service[opensearch_2@production-search-omega-codfw]", "Systemd::Override[opensearch_2@]", "Systemd::Unit[opensearch_2@-opensearch_2@]", "Systemd::Unit[opensearch_2@.service]"], "resource_diffs": [{"resource": "Apt::Repository[wikimedia-opensearch]", "parameters": "--- Apt::Repository[wikimedia-opensearch].orig\n+++ Apt::Repository[wikimedia-opensearch]\n\n@@\n-    components => thirdparty/opensearch1\n+    components => thirdparty/opensearch2\n"}, {"resource": "File[/etc/opensearch/production-search-codfw/opensearch.yml]", "content": "--- /etc/opensearch/production-search-codfw/opensearch.yml.orig\n+++ /etc/opensearch/production-search-codfw/opensearch.yml\n@@ -90,18 +90,17 @@\n transport.tcp.port: 9300\n #\n # --------------------------------- Discovery ----------------------------------\n+# See also: https://docs.opensearch.org/2.19/tuning-your-cluster#step-4-configure-discovery-hosts-and-initial-cluster-manager-nodes-for-a-cluster\n #\n # Pass an initial list of hosts to perform discovery when this node is started:\n # The default list of hosts is [\"127.0.0.1\", \"[::1]\"]\n #\n #discovery.seed_hosts: [\"host1\", \"host2\"]\n-#\n-# Bootstrap the cluster using an initial set of master-eligible nodes:\n-#\n-#cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n-#\n-discovery.zen.ping.unicast.hosts: [\"cirrussearch2093.codfw.wmnet\", \"cirrussearch2061.codfw.wmnet\", \"cirrussearch2081.codfw.wmnet\", \"cirrussearch2084.codfw.wmnet\", \"cirrussearch2111.codfw.wmnet\"]\n-#\n+discovery.seed_hosts: [\"cirrussearch2093.codfw.wmnet\", \"cirrussearch2061.codfw.wmnet\", \"cirrussearch2081.codfw.wmnet\", \"cirrussearch2084.codfw.wmnet\", \"cirrussearch2111.codfw.wmnet\"]\n+\n+# Bootstrap the cluster using an initial set of cluster-manager-eligible node names\n+# Only set this during first cluster formation; remove it after the cluster forms\n+# cluster.initial_cluster_manager_nodes: [\"node-1\", \"node-2\"]\n # For more information, consult the discovery and cluster formation module documentation.\n #\n # ---------------------------------- Gateway -----------------------------------\n@@ -127,7 +126,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr,analysis-nori,analysis-sudachi,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: +apifeatureusage-*,+glent_*,-*\n@@ -140,14 +139,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes when this threshold is reached\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 75%\n-# Move shards off of nodes when this threshold is reached\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 80%\n-# Set indexes read-only with allocated shards on affected node when this threshold is reached\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n cluster.routing.allocation.disk.watermark.flood_stage: 95%\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +194,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]", "parameters": "--- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)].orig\n+++ Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}, {"resource": "File[/etc/opensearch/production-search-omega-codfw/log4j2.properties]", "content": "--- /etc/opensearch/production-search-omega-codfw/log4j2.properties.orig\n+++ /etc/opensearch/production-search-omega-codfw/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Opensearch::Cross_cluster_settings[production-search-omega-codfw]", "parameters": "--- Opensearch::Cross_cluster_settings[production-search-omega-codfw].orig\n+++ Opensearch::Cross_cluster_settings[production-search-omega-codfw]\n\n@@\n-    settings => {'production-search-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-codfw', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch2093.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2111.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2061.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2089.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2114.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 36, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-omega-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-omega-codfw', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9500, 'unicast_hosts': ['cirrussearch2092.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2086.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2114.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}, 'production-search-psi-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-psi-codfw', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9700, 'unicast_hosts': ['cirrussearch2108.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2089.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}\n+    settings => {'production-search-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-codfw', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch2093.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2111.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2061.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2089.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2114.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 36, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-omega-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-omega-codfw', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9500, 'unicast_hosts': ['cirrussearch2092.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2086.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2114.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}, 'production-search-psi-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-psi-codfw', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9700, 'unicast_hosts': ['cirrussearch2108.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2089.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}\n"}, {"resource": "Systemd::Unit[opensearch_2@.service]", "parameters": "--- Systemd::Unit[opensearch_2@.service].orig\n+++ Systemd::Unit[opensearch_2@.service]\n\n+    ensure            => present\n+    override          => False\n+    override_filename => puppet-override.conf\n+    require           => ['Class[Systemd]']\n+    unit              => opensearch_2@.service\n+    restart           => False\n"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2"}, {"resource": "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "parameters": "--- Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)].orig\n+++ Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]\n\n+    refreshonly => True\n+    command     => /bin/systemctl daemon-reload\n"}, {"resource": "Service[opensearch_1@production-search-codfw]", "parameters": "--- Service[opensearch_1@production-search-codfw].orig\n+++ Service[opensearch_1@production-search-codfw]\n\n-    provider => systemd\n-    ensure   => running\n-    tag      => opensearch_services\n-    require  => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/production-search-codfw/opensearch.yml]', 'File[/etc/opensearch/production-search-codfw/logging.yml]', 'File[/etc/opensearch/production-search-codfw/log4j2.properties]', 'File[/etc/opensearch/production-search-codfw/jvm.options]', 'File[/srv/opensearch/production-search-codfw]', 'Systemd::Tmpfile[opensearch-production-search-codfw]']\n-    enable   => True\n"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2"}, {"resource": "Class[Profile::Opensearch::Cirrus::Server]", "parameters": "--- Class[Profile::Opensearch::Cirrus::Server].orig\n+++ Class[Profile::Opensearch::Cirrus::Server]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "Service[opensearch_2@production-search-omega-codfw]", "parameters": "--- Service[opensearch_2@production-search-omega-codfw].orig\n+++ Service[opensearch_2@production-search-omega-codfw]\n\n+    provider => systemd\n+    ensure   => running\n+    tag      => opensearch_services\n+    require  => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/production-search-omega-codfw/opensearch.yml]', 'File[/etc/opensearch/production-search-omega-codfw/logging.yml]', 'File[/etc/opensearch/production-search-omega-codfw/log4j2.properties]', 'File[/etc/opensearch/production-search-omega-codfw/jvm.options]', 'File[/srv/opensearch/production-search-omega-codfw]', 'Systemd::Tmpfile[opensearch-production-search-omega-codfw]']\n+    enable   => True\n"}, {"resource": "Class[Opensearch::Packages]", "parameters": "--- Class[Opensearch::Packages].orig\n+++ Class[Opensearch::Packages]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "File[/lib/systemd/system/opensearch_1@.service]", "content": "--- /lib/systemd/system/opensearch_1@.service.orig\n+++ /lib/systemd/system/opensearch_1@.service\n@@ -1,73 +0,0 @@\n-[Unit]\n-Description=OpenSearch (cluster %i)\n-PartOf=opensearch.service\n-Documentation=https://opensearch.org/docs\n-Wants=network-online.target\n-After=network-online.target\n-\n-[Service]\n-Type=notify\n-RuntimeDirectory=opensearch\n-PrivateTmp=true\n-Environment=OPENSEARCH_HOME=/usr/share/opensearch\n-Environment=OPENSEARCH_PATH_CONF=/etc/opensearch/%i\n-Environment=OPENSEARCH_JVM_OPTIONS=/etc/opensearch/%i/jvm.options\n-Environment=PID_DIR=/run/opensearch-%i\n-Environment=OPENSEARCH_SD_NOTIFY=true\n-\n-Environment=LD_LIBRARY_PATH=/usr/share/opensearch/plugins/opensearch-knn/lib\n-Environment=JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64\n-\n-WorkingDirectory=/usr/share/opensearch\n-\n-User=opensearch\n-Group=opensearch\n-\n-# Force creation of /run/opensearch (see T276198)\n-ExecStartPre=/bin/systemd-tmpfiles --create /etc/tmpfiles.d/opensearch-%i.conf\n-ExecStart=/usr/share/opensearch/bin/opensearch \\\n-        -p ${PID_DIR}/%i.pid \\\n-        --quiet \\\n-        -Epath.logs=/var/log/opensearch \\\n-        -Epath.data=/srv/opensearch/%i\n-\n-# StandardOutput is configured to redirect to journalctl since\n-# some error messages may be logged in standard output before\n-# opensearch logging system is initialized. OpenSearch\n-# stores its logs in /var/log/opensearch and does not use\n-# journalctl by default. If you also want to enable journalctl\n-# logging, you can simply remove the \"quiet\" option from ExecStart.\n-StandardOutput=journal\n-StandardError=inherit\n-\n-# Specifies the maximum file descriptor number that can be opened by this process\n-LimitNOFILE=65535\n-\n-# Specifies the maximum number of processes\n-LimitNPROC=4096\n-\n-# Specifies the maximum size of virtual memory\n-LimitAS=infinity\n-\n-# Specifies the maximum file size\n-LimitFSIZE=infinity\n-\n-# Disable timeout logic and wait until process is stopped\n-TimeoutStopSec=0\n-\n-# SIGTERM signal is used to stop the Java process\n-KillSignal=SIGTERM\n-\n-# Send the signal only to the JVM rather than its control group\n-KillMode=process\n-\n-# Java process is never killed\n-SendSIGKILL=no\n-\n-# When a JVM receives a SIGTERM signal it exits with code 143\n-SuccessExitStatus=143\n-\n-[Install]\n-WantedBy=multi-user.target\n-\n-# Built for packages-7.4.1 (packages)", "parameters": "--- File[/lib/systemd/system/opensearch_1@.service].orig\n+++ File[/lib/systemd/system/opensearch_1@.service]\n\n-    notify => Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]\n-    ensure => present\n-    mode   => 0444\n-    group  => root\n-    owner  => root\n"}, {"resource": "File[/etc/opensearch/production-search-omega-codfw/opensearch.yml]", "content": "--- /etc/opensearch/production-search-omega-codfw/opensearch.yml.orig\n+++ /etc/opensearch/production-search-omega-codfw/opensearch.yml\n@@ -90,18 +90,17 @@\n transport.tcp.port: 9500\n #\n # --------------------------------- Discovery ----------------------------------\n+# See also: https://docs.opensearch.org/2.19/tuning-your-cluster#step-4-configure-discovery-hosts-and-initial-cluster-manager-nodes-for-a-cluster\n #\n # Pass an initial list of hosts to perform discovery when this node is started:\n # The default list of hosts is [\"127.0.0.1\", \"[::1]\"]\n #\n #discovery.seed_hosts: [\"host1\", \"host2\"]\n-#\n-# Bootstrap the cluster using an initial set of master-eligible nodes:\n-#\n-#cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n-#\n-discovery.zen.ping.unicast.hosts: [\"cirrussearch2092.codfw.wmnet\", \"cirrussearch2100.codfw.wmnet\", \"cirrussearch2106.codfw.wmnet\", \"cirrussearch2073.codfw.wmnet\", \"cirrussearch2086.codfw.wmnet\"]\n-#\n+discovery.seed_hosts: [\"cirrussearch2092.codfw.wmnet\", \"cirrussearch2100.codfw.wmnet\", \"cirrussearch2106.codfw.wmnet\", \"cirrussearch2073.codfw.wmnet\", \"cirrussearch2086.codfw.wmnet\"]\n+\n+# Bootstrap the cluster using an initial set of cluster-manager-eligible node names\n+# Only set this during first cluster formation; remove it after the cluster forms\n+# cluster.initial_cluster_manager_nodes: [\"node-1\", \"node-2\"]\n # For more information, consult the discovery and cluster formation module documentation.\n #\n # ---------------------------------- Gateway -----------------------------------\n@@ -127,7 +126,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr,analysis-nori,analysis-sudachi,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: +apifeatureusage-*,+glent_*,-*\n@@ -140,14 +139,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes when this threshold is reached\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 75%\n-# Move shards off of nodes when this threshold is reached\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 80%\n-# Set indexes read-only with allocated shards on affected node when this threshold is reached\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n cluster.routing.allocation.disk.watermark.flood_stage: 95%\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +194,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n-    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n+    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n@@\n-    version         => 1.3.20\n+    version         => 2.19.5\n@@\n-    java_home       => /usr/lib/jvm/java-11-openjdk-amd64\n+    java_home       => /usr/lib/jvm/java-21-openjdk-amd64\n"}, {"resource": "Systemd::Unit[opensearch_2@-opensearch_2@]", "parameters": "--- Systemd::Unit[opensearch_2@-opensearch_2@].orig\n+++ Systemd::Unit[opensearch_2@-opensearch_2@]\n\n+    ensure            => present\n+    override          => True\n+    override_filename => opensearch_2@\n+    require           => ['Class[Systemd]']\n+    unit              => opensearch_2@\n+    restart           => False\n"}, {"resource": "Service[opensearch_1@production-search-omega-codfw]", "parameters": "--- Service[opensearch_1@production-search-omega-codfw].orig\n+++ Service[opensearch_1@production-search-omega-codfw]\n\n-    provider => systemd\n-    ensure   => running\n-    tag      => opensearch_services\n-    require  => ['Systemd::Unit[opensearch_1@.service]', 'File[/etc/opensearch/production-search-omega-codfw/opensearch.yml]', 'File[/etc/opensearch/production-search-omega-codfw/logging.yml]', 'File[/etc/opensearch/production-search-omega-codfw/log4j2.properties]', 'File[/etc/opensearch/production-search-omega-codfw/jvm.options]', 'File[/srv/opensearch/production-search-omega-codfw]', 'Systemd::Tmpfile[opensearch-production-search-omega-codfw]']\n-    enable   => True\n"}, {"resource": "File[/etc/opensearch/instances]", "content": "--- /etc/opensearch/instances.orig\n+++ /etc/opensearch/instances\n@@ -1,2 +1,2 @@\n-opensearch_1@production-search-codfw\n-opensearch_1@production-search-omega-codfw\n+opensearch_2@production-search-codfw\n+opensearch_2@production-search-omega-codfw"}, {"resource": "Apt::Repository[wikimedia-opensearch-plugins]", "parameters": "--- Apt::Repository[wikimedia-opensearch-plugins].orig\n+++ Apt::Repository[wikimedia-opensearch-plugins]\n\n@@\n-    components => component/opensearch13\n+    components => component/opensearch2\n"}, {"resource": "File[/etc/systemd/system/opensearch_2@.service.d]", "parameters": "--- File[/etc/systemd/system/opensearch_2@.service.d].orig\n+++ File[/etc/systemd/system/opensearch_2@.service.d]\n\n+    group  => root\n+    owner  => root\n+    ensure => directory\n+    mode   => 0555\n"}, {"resource": "Systemd::Override[opensearch_2@]", "parameters": "--- Systemd::Override[opensearch_2@].orig\n+++ Systemd::Override[opensearch_2@]\n\n+    ensure  => present\n+    unit    => opensearch_2@\n+    restart => False\n"}, {"resource": "Exec[install-opensearch]", "parameters": "--- Exec[install-opensearch].orig\n+++ Exec[install-opensearch]\n\n+    before      => Package[opensearch]\n+    command     => /usr/bin/apt-get -q -y -o DPkg::Options::=--force-confold install opensearch=2.19.5\n+    unless      => /usr/bin/dpkg-query -W -f='${Status} ${Version}\\n' opensearch 2>/dev/null | /bin/grep -Fxq 'install ok installed 2.19.5'\n+    timeout     => 300\n+    logoutput   => False\n+    environment => ['OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpensearchTemp1!', 'DEBIAN_FRONTEND=noninteractive']\n"}, {"resource": "Service[opensearch_2@production-search-codfw]", "parameters": "--- Service[opensearch_2@production-search-codfw].orig\n+++ Service[opensearch_2@production-search-codfw]\n\n+    provider => systemd\n+    ensure   => running\n+    tag      => opensearch_services\n+    require  => ['Systemd::Unit[opensearch_2@.service]', 'File[/etc/opensearch/production-search-codfw/opensearch.yml]', 'File[/etc/opensearch/production-search-codfw/logging.yml]', 'File[/etc/opensearch/production-search-codfw/log4j2.properties]', 'File[/etc/opensearch/production-search-codfw/jvm.options]', 'File[/srv/opensearch/production-search-codfw]', 'Systemd::Tmpfile[opensearch-production-search-codfw]']\n+    enable   => True\n"}, {"resource": "Opensearch::Instance[production-search-codfw]", "parameters": "--- Opensearch::Instance[production-search-codfw].orig\n+++ Opensearch::Instance[production-search-codfw]\n\n@@\n-    version           => 1.3.20\n+    version           => 2.19.5\n@@\n-    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n"}, {"resource": "Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "parameters": "--- Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)].orig\n+++ Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]\n\n-    refreshonly => True\n-    command     => /bin/systemctl daemon-reload\n"}, {"resource": "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "content": "--- /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf.orig\n+++ /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf\n@@ -0,0 +1,33 @@\n+# The OpenSearch packages come with a lot of plugins\n+# we don't want or need. This override helps us\n+# expose only the plugins we explicitly set via\n+# $plugins_mandatory var\n+[Service]\n+# Give OpenSearch its own mount namespace\n+PrivateMounts=yes\n+\n+# Replace plugins dir with an empty tmpfs\n+TemporaryFileSystem=/usr/share/opensearch/plugins:rw\n+\n+BindPaths=\n+BindPaths=/usr/share/opensearch/plugins/analysis-hebrew\n+BindPaths=/usr/share/opensearch/plugins/analysis-icu\n+BindPaths=/usr/share/opensearch/plugins/analysis-smartcn\n+BindPaths=/usr/share/opensearch/plugins/analysis-stempel\n+BindPaths=/usr/share/opensearch/plugins/analysis-ukrainian\n+BindPaths=/usr/share/opensearch/plugins/cirrus-highlighter\n+BindPaths=/usr/share/opensearch/plugins/opensearch-analysis-stconvert\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-esperanto\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-serbian\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-slovak\n+BindPaths=/usr/share/opensearch/plugins/ltr\n+BindPaths=/usr/share/opensearch/plugins/analysis-nori\n+BindPaths=/usr/share/opensearch/plugins/analysis-sudachi\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-homoglyph\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-khmer\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-textify\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-turkish\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-ukrainian\n+BindPaths=/usr/share/opensearch/plugins/opensearch-ltr\n+BindPaths=/usr/share/opensearch/plugins/repository-s3", "parameters": "--- File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf].orig\n+++ File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]\n\n+    notify => Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]\n+    ensure => present\n+    mode   => 0444\n+    group  => root\n+    owner  => root\n"}, {"resource": "Package[wmf-opensearch-search-plugins]", "parameters": "--- Package[wmf-opensearch-search-plugins].orig\n+++ Package[wmf-opensearch-search-plugins]\n\n@@\n-    before => ['Service[opensearch_1@production-search-codfw]', 'Service[opensearch_1@production-search-omega-codfw]']\n+    before => ['Service[opensearch_2@production-search-codfw]', 'Service[opensearch_2@production-search-omega-codfw]']\n"}, {"resource": "Systemd::Unit[opensearch_1@.service]", "parameters": "--- Systemd::Unit[opensearch_1@.service].orig\n+++ Systemd::Unit[opensearch_1@.service]\n\n-    ensure            => present\n-    override          => False\n-    override_filename => puppet-override.conf\n-    require           => ['Class[Systemd]']\n-    unit              => opensearch_1@.service\n-    restart           => False\n"}, {"resource": "File[/etc/opensearch/production-search-codfw/log4j2.properties]", "content": "--- /etc/opensearch/production-search-codfw/log4j2.properties.orig\n+++ /etc/opensearch/production-search-codfw/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Opensearch::Instance[production-search-omega-codfw]", "parameters": "--- Opensearch::Instance[production-search-omega-codfw].orig\n+++ Opensearch::Instance[production-search-omega-codfw]\n\n@@\n-    version           => 1.3.20\n+    version           => 2.19.5\n@@\n-    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n"}, {"resource": "Package[opensearch]", "parameters": "--- Package[opensearch].orig\n+++ Package[opensearch]\n\n@@\n-    ensure => 1.3.20\n+    ensure => 2.19.5\n"}, {"resource": "File[/lib/systemd/system/opensearch_2@.service]", "content": "--- /lib/systemd/system/opensearch_2@.service.orig\n+++ /lib/systemd/system/opensearch_2@.service\n@@ -0,0 +1,73 @@\n+[Unit]\n+Description=OpenSearch (cluster %i)\n+PartOf=opensearch.service\n+Documentation=https://opensearch.org/docs\n+Wants=network-online.target\n+After=network-online.target\n+\n+[Service]\n+Type=notify\n+RuntimeDirectory=opensearch\n+PrivateTmp=true\n+Environment=OPENSEARCH_HOME=/usr/share/opensearch\n+Environment=OPENSEARCH_PATH_CONF=/etc/opensearch/%i\n+Environment=OPENSEARCH_JVM_OPTIONS=/etc/opensearch/%i/jvm.options\n+Environment=PID_DIR=/run/opensearch-%i\n+Environment=OPENSEARCH_SD_NOTIFY=true\n+\n+Environment=LD_LIBRARY_PATH=/usr/share/opensearch/plugins/opensearch-knn/lib\n+Environment=JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64\n+\n+WorkingDirectory=/usr/share/opensearch\n+\n+User=opensearch\n+Group=opensearch\n+\n+# Force creation of /run/opensearch (see T276198)\n+ExecStartPre=/bin/systemd-tmpfiles --create /etc/tmpfiles.d/opensearch-%i.conf\n+ExecStart=/usr/share/opensearch/bin/opensearch \\\n+        -p ${PID_DIR}/%i.pid \\\n+        --quiet \\\n+        -Epath.logs=/var/log/opensearch \\\n+        -Epath.data=/srv/opensearch/%i\n+\n+# StandardOutput is configured to redirect to journalctl since\n+# some error messages may be logged in standard output before\n+# opensearch logging system is initialized. OpenSearch\n+# stores its logs in /var/log/opensearch and does not use\n+# journalctl by default. If you also want to enable journalctl\n+# logging, you can simply remove the \"quiet\" option from ExecStart.\n+StandardOutput=journal\n+StandardError=inherit\n+\n+# Specifies the maximum file descriptor number that can be opened by this process\n+LimitNOFILE=65535\n+\n+# Specifies the maximum number of processes\n+LimitNPROC=4096\n+\n+# Specifies the maximum size of virtual memory\n+LimitAS=infinity\n+\n+# Specifies the maximum file size\n+LimitFSIZE=infinity\n+\n+# Disable timeout logic and wait until process is stopped\n+TimeoutStopSec=0\n+\n+# SIGTERM signal is used to stop the Java process\n+KillSignal=SIGTERM\n+\n+# Send the signal only to the JVM rather than its control group\n+KillMode=process\n+\n+# Java process is never killed\n+SendSIGKILL=no\n+\n+# When a JVM receives a SIGTERM signal it exits with code 143\n+SuccessExitStatus=143\n+\n+[Install]\n+WantedBy=multi-user.target\n+\n+# Built for packages-7.4.1 (packages)", "parameters": "--- File[/lib/systemd/system/opensearch_2@.service].orig\n+++ File[/lib/systemd/system/opensearch_2@.service]\n\n+    notify => Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]\n+    ensure => present\n+    mode   => 0444\n+    group  => root\n+    owner  => root\n"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n-    version           => 1.3.20\n+    version           => 2.19.5\n@@\n-    instances         => {'production-search-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-codfw', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch2093.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2111.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2061.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2089.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2114.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 36, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-omega-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-omega-codfw', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9500, 'unicast_hosts': ['cirrussearch2092.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2086.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2114.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}\n+    instances         => {'production-search-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-codfw', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch2093.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2111.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2061.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2089.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2114.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 36, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-omega-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-omega-codfw', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9500, 'unicast_hosts': ['cirrussearch2092.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2086.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2114.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}\n@@\n-    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n@@\n-    java_home         => /usr/lib/jvm/java-11-openjdk-amd64\n+    java_home         => /usr/lib/jvm/java-21-openjdk-amd64\n"}], "perc_changed": "1.39%"}, "core": {"total": 3588, "only_in_self": ["Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "File[/lib/systemd/system/opensearch_1@.service]", "Service[opensearch_1@production-search-codfw]", "Service[opensearch_1@production-search-omega-codfw]"], "only_in_other": ["Exec[install-opensearch]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "File[/etc/systemd/system/opensearch_2@.service.d]", "File[/lib/systemd/system/opensearch_2@.service]", "Service[opensearch_2@production-search-codfw]", "Service[opensearch_2@production-search-omega-codfw]"], "resource_diffs": [{"resource": "File[/etc/opensearch/production-search-codfw/opensearch.yml]", "content": "--- /etc/opensearch/production-search-codfw/opensearch.yml.orig\n+++ /etc/opensearch/production-search-codfw/opensearch.yml\n@@ -90,18 +90,17 @@\n transport.tcp.port: 9300\n #\n # --------------------------------- Discovery ----------------------------------\n+# See also: https://docs.opensearch.org/2.19/tuning-your-cluster#step-4-configure-discovery-hosts-and-initial-cluster-manager-nodes-for-a-cluster\n #\n # Pass an initial list of hosts to perform discovery when this node is started:\n # The default list of hosts is [\"127.0.0.1\", \"[::1]\"]\n #\n #discovery.seed_hosts: [\"host1\", \"host2\"]\n-#\n-# Bootstrap the cluster using an initial set of master-eligible nodes:\n-#\n-#cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n-#\n-discovery.zen.ping.unicast.hosts: [\"cirrussearch2093.codfw.wmnet\", \"cirrussearch2061.codfw.wmnet\", \"cirrussearch2081.codfw.wmnet\", \"cirrussearch2084.codfw.wmnet\", \"cirrussearch2111.codfw.wmnet\"]\n-#\n+discovery.seed_hosts: [\"cirrussearch2093.codfw.wmnet\", \"cirrussearch2061.codfw.wmnet\", \"cirrussearch2081.codfw.wmnet\", \"cirrussearch2084.codfw.wmnet\", \"cirrussearch2111.codfw.wmnet\"]\n+\n+# Bootstrap the cluster using an initial set of cluster-manager-eligible node names\n+# Only set this during first cluster formation; remove it after the cluster forms\n+# cluster.initial_cluster_manager_nodes: [\"node-1\", \"node-2\"]\n # For more information, consult the discovery and cluster formation module documentation.\n #\n # ---------------------------------- Gateway -----------------------------------\n@@ -127,7 +126,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr,analysis-nori,analysis-sudachi,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: +apifeatureusage-*,+glent_*,-*\n@@ -140,14 +139,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes when this threshold is reached\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 75%\n-# Move shards off of nodes when this threshold is reached\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 80%\n-# Set indexes read-only with allocated shards on affected node when this threshold is reached\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n cluster.routing.allocation.disk.watermark.flood_stage: 95%\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +194,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/production-search-omega-codfw/opensearch.yml]", "content": "--- /etc/opensearch/production-search-omega-codfw/opensearch.yml.orig\n+++ /etc/opensearch/production-search-omega-codfw/opensearch.yml\n@@ -90,18 +90,17 @@\n transport.tcp.port: 9500\n #\n # --------------------------------- Discovery ----------------------------------\n+# See also: https://docs.opensearch.org/2.19/tuning-your-cluster#step-4-configure-discovery-hosts-and-initial-cluster-manager-nodes-for-a-cluster\n #\n # Pass an initial list of hosts to perform discovery when this node is started:\n # The default list of hosts is [\"127.0.0.1\", \"[::1]\"]\n #\n #discovery.seed_hosts: [\"host1\", \"host2\"]\n-#\n-# Bootstrap the cluster using an initial set of master-eligible nodes:\n-#\n-#cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n-#\n-discovery.zen.ping.unicast.hosts: [\"cirrussearch2092.codfw.wmnet\", \"cirrussearch2100.codfw.wmnet\", \"cirrussearch2106.codfw.wmnet\", \"cirrussearch2073.codfw.wmnet\", \"cirrussearch2086.codfw.wmnet\"]\n-#\n+discovery.seed_hosts: [\"cirrussearch2092.codfw.wmnet\", \"cirrussearch2100.codfw.wmnet\", \"cirrussearch2106.codfw.wmnet\", \"cirrussearch2073.codfw.wmnet\", \"cirrussearch2086.codfw.wmnet\"]\n+\n+# Bootstrap the cluster using an initial set of cluster-manager-eligible node names\n+# Only set this during first cluster formation; remove it after the cluster forms\n+# cluster.initial_cluster_manager_nodes: [\"node-1\", \"node-2\"]\n # For more information, consult the discovery and cluster formation module documentation.\n #\n # ---------------------------------- Gateway -----------------------------------\n@@ -127,7 +126,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr,analysis-nori,analysis-sudachi,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: +apifeatureusage-*,+glent_*,-*\n@@ -140,14 +139,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes when this threshold is reached\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 75%\n-# Move shards off of nodes when this threshold is reached\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 80%\n-# Set indexes read-only with allocated shards on affected node when this threshold is reached\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n cluster.routing.allocation.disk.watermark.flood_stage: 95%\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +194,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/instances]", "content": "--- /etc/opensearch/instances.orig\n+++ /etc/opensearch/instances\n@@ -1,2 +1,2 @@\n-opensearch_1@production-search-codfw\n-opensearch_1@production-search-omega-codfw\n+opensearch_2@production-search-codfw\n+opensearch_2@production-search-omega-codfw"}, {"resource": "File[/etc/opensearch/production-search-omega-codfw/log4j2.properties]", "content": "--- /etc/opensearch/production-search-omega-codfw/log4j2.properties.orig\n+++ /etc/opensearch/production-search-omega-codfw/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2"}, {"resource": "Package[wmf-opensearch-search-plugins]", "parameters": "--- Package[wmf-opensearch-search-plugins].orig\n+++ Package[wmf-opensearch-search-plugins]\n\n@@\n-    before => ['Service[opensearch_1@production-search-codfw]', 'Service[opensearch_1@production-search-omega-codfw]']\n+    before => ['Service[opensearch_2@production-search-codfw]', 'Service[opensearch_2@production-search-omega-codfw]']\n"}, {"resource": "File[/etc/opensearch/production-search-codfw/log4j2.properties]", "content": "--- /etc/opensearch/production-search-codfw/log4j2.properties.orig\n+++ /etc/opensearch/production-search-codfw/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2"}, {"resource": "Package[opensearch]", "parameters": "--- Package[opensearch].orig\n+++ Package[opensearch]\n\n@@\n-    ensure => 1.3.20\n+    ensure => 2.19.5\n"}], "perc_changed": "0.59%"}, "main": {"total": 3588, "only_in_self": ["Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]", "File[/lib/systemd/system/opensearch_1@.service]", "Service[opensearch_1@production-search-codfw]", "Service[opensearch_1@production-search-omega-codfw]", "Systemd::Unit[opensearch_1@.service]"], "only_in_other": ["Exec[install-opensearch]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]", "Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]", "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "File[/etc/systemd/system/opensearch_2@.service.d]", "File[/lib/systemd/system/opensearch_2@.service]", "Service[opensearch_2@production-search-codfw]", "Service[opensearch_2@production-search-omega-codfw]", "Systemd::Override[opensearch_2@]", "Systemd::Unit[opensearch_2@-opensearch_2@]", "Systemd::Unit[opensearch_2@.service]"], "resource_diffs": [{"resource": "Apt::Repository[wikimedia-opensearch]", "parameters": "--- Apt::Repository[wikimedia-opensearch].orig\n+++ Apt::Repository[wikimedia-opensearch]\n\n@@\n-    components => thirdparty/opensearch1\n+    components => thirdparty/opensearch2\n"}, {"resource": "File[/etc/opensearch/production-search-codfw/opensearch.yml]", "content": "--- /etc/opensearch/production-search-codfw/opensearch.yml.orig\n+++ /etc/opensearch/production-search-codfw/opensearch.yml\n@@ -90,18 +90,17 @@\n transport.tcp.port: 9300\n #\n # --------------------------------- Discovery ----------------------------------\n+# See also: https://docs.opensearch.org/2.19/tuning-your-cluster#step-4-configure-discovery-hosts-and-initial-cluster-manager-nodes-for-a-cluster\n #\n # Pass an initial list of hosts to perform discovery when this node is started:\n # The default list of hosts is [\"127.0.0.1\", \"[::1]\"]\n #\n #discovery.seed_hosts: [\"host1\", \"host2\"]\n-#\n-# Bootstrap the cluster using an initial set of master-eligible nodes:\n-#\n-#cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n-#\n-discovery.zen.ping.unicast.hosts: [\"cirrussearch2093.codfw.wmnet\", \"cirrussearch2061.codfw.wmnet\", \"cirrussearch2081.codfw.wmnet\", \"cirrussearch2084.codfw.wmnet\", \"cirrussearch2111.codfw.wmnet\"]\n-#\n+discovery.seed_hosts: [\"cirrussearch2093.codfw.wmnet\", \"cirrussearch2061.codfw.wmnet\", \"cirrussearch2081.codfw.wmnet\", \"cirrussearch2084.codfw.wmnet\", \"cirrussearch2111.codfw.wmnet\"]\n+\n+# Bootstrap the cluster using an initial set of cluster-manager-eligible node names\n+# Only set this during first cluster formation; remove it after the cluster forms\n+# cluster.initial_cluster_manager_nodes: [\"node-1\", \"node-2\"]\n # For more information, consult the discovery and cluster formation module documentation.\n #\n # ---------------------------------- Gateway -----------------------------------\n@@ -127,7 +126,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr,analysis-nori,analysis-sudachi,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: +apifeatureusage-*,+glent_*,-*\n@@ -140,14 +139,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes when this threshold is reached\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 75%\n-# Move shards off of nodes when this threshold is reached\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 80%\n-# Set indexes read-only with allocated shards on affected node when this threshold is reached\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n cluster.routing.allocation.disk.watermark.flood_stage: 95%\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +194,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "File[/etc/opensearch/production-search-omega-codfw/opensearch.yml]", "content": "--- /etc/opensearch/production-search-omega-codfw/opensearch.yml.orig\n+++ /etc/opensearch/production-search-omega-codfw/opensearch.yml\n@@ -90,18 +90,17 @@\n transport.tcp.port: 9500\n #\n # --------------------------------- Discovery ----------------------------------\n+# See also: https://docs.opensearch.org/2.19/tuning-your-cluster#step-4-configure-discovery-hosts-and-initial-cluster-manager-nodes-for-a-cluster\n #\n # Pass an initial list of hosts to perform discovery when this node is started:\n # The default list of hosts is [\"127.0.0.1\", \"[::1]\"]\n #\n #discovery.seed_hosts: [\"host1\", \"host2\"]\n-#\n-# Bootstrap the cluster using an initial set of master-eligible nodes:\n-#\n-#cluster.initial_master_nodes: [\"node-1\", \"node-2\"]\n-#\n-discovery.zen.ping.unicast.hosts: [\"cirrussearch2092.codfw.wmnet\", \"cirrussearch2100.codfw.wmnet\", \"cirrussearch2106.codfw.wmnet\", \"cirrussearch2073.codfw.wmnet\", \"cirrussearch2086.codfw.wmnet\"]\n-#\n+discovery.seed_hosts: [\"cirrussearch2092.codfw.wmnet\", \"cirrussearch2100.codfw.wmnet\", \"cirrussearch2106.codfw.wmnet\", \"cirrussearch2073.codfw.wmnet\", \"cirrussearch2086.codfw.wmnet\"]\n+\n+# Bootstrap the cluster using an initial set of cluster-manager-eligible node names\n+# Only set this during first cluster formation; remove it after the cluster forms\n+# cluster.initial_cluster_manager_nodes: [\"node-1\", \"node-2\"]\n # For more information, consult the discovery and cluster formation module documentation.\n #\n # ---------------------------------- Gateway -----------------------------------\n@@ -127,7 +126,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr,analysis-nori,analysis-sudachi,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: +apifeatureusage-*,+glent_*,-*\n@@ -140,14 +139,16 @@\n # 1k is too low to allow mwgrep to operate.\n action.search.shard_count.limit: 5000\n \n+# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n cluster.routing.allocation.disk.threshold_enabled: true\n-# Stop allocating new shards to nodes when this threshold is reached\n+# Stop allocating shards to nodes (default 80%)\n cluster.routing.allocation.disk.watermark.low: 75%\n-# Move shards off of nodes when this threshold is reached\n+# Begin evicting shards off the node (default 90%)\n cluster.routing.allocation.disk.watermark.high: 80%\n-# Set indexes read-only with allocated shards on affected node when this threshold is reached\n+# Set indexes read-only with allocated shards on affected node (default 95%)\n cluster.routing.allocation.disk.watermark.flood_stage: 95%\n+\n # Check disk utilization every 60 seconds\n cluster.info.update.interval: 60s\n \n@@ -193,3 +194,4 @@\n ltr.caches.max_mem: 100mb\n \n \n+"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n-    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n+    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%'}\n@@\n-    version         => 1.3.20\n+    version         => 2.19.5\n@@\n-    java_home       => /usr/lib/jvm/java-11-openjdk-amd64\n+    java_home       => /usr/lib/jvm/java-21-openjdk-amd64\n"}, {"resource": "File[/etc/opensearch/instances]", "content": "--- /etc/opensearch/instances.orig\n+++ /etc/opensearch/instances\n@@ -1,2 +1,2 @@\n-opensearch_1@production-search-codfw\n-opensearch_1@production-search-omega-codfw\n+opensearch_2@production-search-codfw\n+opensearch_2@production-search-omega-codfw"}, {"resource": "File[/etc/opensearch/production-search-omega-codfw/log4j2.properties]", "content": "--- /etc/opensearch/production-search-omega-codfw/log4j2.properties.orig\n+++ /etc/opensearch/production-search-omega-codfw/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "Opensearch::Cross_cluster_settings[production-search-omega-codfw]", "parameters": "--- Opensearch::Cross_cluster_settings[production-search-omega-codfw].orig\n+++ Opensearch::Cross_cluster_settings[production-search-omega-codfw]\n\n@@\n-    settings => {'production-search-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-codfw', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch2093.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2111.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2061.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2089.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2114.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 36, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-omega-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-omega-codfw', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9500, 'unicast_hosts': ['cirrussearch2092.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2086.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2114.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}, 'production-search-psi-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-psi-codfw', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9700, 'unicast_hosts': ['cirrussearch2108.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2089.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}\n+    settings => {'production-search-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-codfw', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch2093.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2111.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2061.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2089.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2114.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 36, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-omega-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-omega-codfw', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9500, 'unicast_hosts': ['cirrussearch2092.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2086.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2114.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}, 'production-search-psi-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-psi-codfw', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9700, 'unicast_hosts': ['cirrussearch2108.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2089.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}\n"}, {"resource": "Apt::Repository[wikimedia-opensearch-plugins]", "parameters": "--- Apt::Repository[wikimedia-opensearch-plugins].orig\n+++ Apt::Repository[wikimedia-opensearch-plugins]\n\n@@\n-    components => component/opensearch13\n+    components => component/opensearch2\n"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2"}, {"resource": "Opensearch::Instance[production-search-codfw]", "parameters": "--- Opensearch::Instance[production-search-codfw].orig\n+++ Opensearch::Instance[production-search-codfw]\n\n@@\n-    version           => 1.3.20\n+    version           => 2.19.5\n@@\n-    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n"}, {"resource": "Package[wmf-opensearch-search-plugins]", "parameters": "--- Package[wmf-opensearch-search-plugins].orig\n+++ Package[wmf-opensearch-search-plugins]\n\n@@\n-    before => ['Service[opensearch_1@production-search-codfw]', 'Service[opensearch_1@production-search-omega-codfw]']\n+    before => ['Service[opensearch_2@production-search-codfw]', 'Service[opensearch_2@production-search-omega-codfw]']\n"}, {"resource": "File[/etc/opensearch/production-search-codfw/log4j2.properties]", "content": "--- /etc/opensearch/production-search-codfw/log4j2.properties.orig\n+++ /etc/opensearch/production-search-codfw/log4j2.properties\n@@ -185,7 +185,6 @@\n rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash\n logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n-logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash\n \n # Define Logstash appender\n appender.ship_to_logstash.name=ship_to_logstash"}, {"resource": "File[/etc/apt/sources.list.d/wikimedia-opensearch.list]", "content": "--- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig\n+++ /etc/apt/sources.list.d/wikimedia-opensearch.list\n@@ -1,2 +1,2 @@\n-deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n-deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1\n+deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2\n+deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2"}, {"resource": "Opensearch::Instance[production-search-omega-codfw]", "parameters": "--- Opensearch::Instance[production-search-omega-codfw].orig\n+++ Opensearch::Instance[production-search-omega-codfw]\n\n@@\n-    version           => 1.3.20\n+    version           => 2.19.5\n@@\n-    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n"}, {"resource": "Class[Profile::Opensearch::Cirrus::Server]", "parameters": "--- Class[Profile::Opensearch::Cirrus::Server].orig\n+++ Class[Profile::Opensearch::Cirrus::Server]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "Package[opensearch]", "parameters": "--- Package[opensearch].orig\n+++ Package[opensearch]\n\n@@\n-    ensure => 1.3.20\n+    ensure => 2.19.5\n"}, {"resource": "Class[Opensearch::Packages]", "parameters": "--- Class[Opensearch::Packages].orig\n+++ Class[Opensearch::Packages]\n\n@@\n-    version => 1.3.20\n+    version => 2.19.5\n"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n-    version           => 1.3.20\n+    version           => 2.19.5\n@@\n-    instances         => {'production-search-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-codfw', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch2093.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2111.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2061.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2089.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2114.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 36, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-omega-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-omega-codfw', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9500, 'unicast_hosts': ['cirrussearch2092.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2086.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2114.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}\n+    instances         => {'production-search-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '30G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-codfw', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9300, 'unicast_hosts': ['cirrussearch2093.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2111.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2061.codfw.wmnet', 'cirrussearch2062.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2064.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2066.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2068.codfw.wmnet', 'cirrussearch2069.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2071.codfw.wmnet', 'cirrussearch2072.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2075.codfw.wmnet', 'cirrussearch2076.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2078.codfw.wmnet', 'cirrussearch2079.codfw.wmnet', 'cirrussearch2080.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2082.codfw.wmnet', 'cirrussearch2083.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2085.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2089.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2091.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2094.codfw.wmnet', 'cirrussearch2095.codfw.wmnet', 'cirrussearch2096.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2101.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2103.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2107.codfw.wmnet', 'cirrussearch2108.codfw.wmnet', 'cirrussearch2109.codfw.wmnet', 'cirrussearch2110.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2113.codfw.wmnet', 'cirrussearch2114.codfw.wmnet', 'cirrussearch2115.codfw.wmnet'], 'expected_nodes': 36, 'recover_after_nodes': 24, 'indices_to_monitor': ['commonswiki_file', 'wikidatawiki_content', 'enwiki_general', 'enwiki_content', 'viwiki_general', 'commonswiki_general', 'cebwiki_content', 'metawiki_general', 'dewiki_content', 'frwiki_content']}, 'production-search-omega-codfw': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'watermarks': {'low': '75%', 'high': '80%', 'flood_stage': '95%'}, 'heap_memory': '10G', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'auto_create_index': '+apifeatureusage-*,+glent_*,-*', 'script_max_compilations_per_minute': 10000, 'bulk_thread_pool_executors': 6, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'cluster_name': 'production-search-omega-codfw', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'certificate_name': 'search.discovery.wmnet', 'transport_tcp_port': 9500, 'unicast_hosts': ['cirrussearch2092.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2086.codfw.wmnet'], 'cluster_hosts': ['cirrussearch2087.codfw.wmnet', 'cirrussearch2088.codfw.wmnet', 'cirrussearch2061.codfw.wmnet', 'cirrussearch2073.codfw.wmnet', 'cirrussearch2074.codfw.wmnet', 'cirrussearch2090.codfw.wmnet', 'cirrussearch2111.codfw.wmnet', 'cirrussearch2092.codfw.wmnet', 'cirrussearch2093.codfw.wmnet', 'cirrussearch2063.codfw.wmnet', 'cirrussearch2070.codfw.wmnet', 'cirrussearch2077.codfw.wmnet', 'cirrussearch2097.codfw.wmnet', 'cirrussearch2098.codfw.wmnet', 'cirrussearch2099.codfw.wmnet', 'cirrussearch2100.codfw.wmnet', 'cirrussearch2065.codfw.wmnet', 'cirrussearch2081.codfw.wmnet', 'cirrussearch2102.codfw.wmnet', 'cirrussearch2112.codfw.wmnet', 'cirrussearch2104.codfw.wmnet', 'cirrussearch2105.codfw.wmnet', 'cirrussearch2106.codfw.wmnet', 'cirrussearch2067.codfw.wmnet', 'cirrussearch2084.codfw.wmnet', 'cirrussearch2086.codfw.wmnet', 'cirrussearch2114.codfw.wmnet'], 'expected_nodes': 18, 'recover_after_nodes': 11}}\n@@\n-    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr']\n+    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'ltr', 'analysis-nori', 'analysis-sudachi', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n@@\n-    java_home         => /usr/lib/jvm/java-11-openjdk-amd64\n+    java_home         => /usr/lib/jvm/java-21-openjdk-amd64\n"}], "perc_changed": "0.95%"}}}