{"host": "cloudelastic1007.eqiad.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3688, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "Systemd::Unit[opensearch_2@-opensearch_2@]"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -28,7 +28,7 @@\n #\n # Row/rack awareness attributes\n #\n-\n+cluster.routing.allocation.awareness.attributes: row\n node.attr.row: eqiad-row-a\n node.attr.rack: A2\n #\n@@ -83,7 +83,7 @@\n \n # Set both 'bind_host' and 'publish_host':\n #\n-network.host: [ _local_,_site_ ]\n+network.host: [ 0.0.0.0 ]\n \n # Set a custom port for the node to node communication (9300 by default):\n #\n@@ -118,7 +118,7 @@\n #\n #\n gateway:\n-    recover_after_time: 1s\n+    recover_after_time: 5m\n     expected_nodes: 6\n #\n # ---------------------------------- Various -----------------------------------\n@@ -131,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-#plugin.mandatory: mapper-attachments\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-nori,analysis-smartcn,analysis-stempel,analysis-sudachi,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,9 +140,9 @@\n # still close/delete individual indices.\n action.destructive_requires_name: true\n \n-# Allow up to 1000 shards to be queried at a time. The default\n+# Allow up to 5000 shards to be queried at a time. The default\n # 1k is too low to allow mwgrep to operate.\n-action.search.shard_count.limit: 1000\n+action.search.shard_count.limit: 5000\n \n # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n@@ -180,13 +180,23 @@\n ##\n # Filter cache size\n ##\n-indices.queries.cache.size: 10%\n-\n-\n-\n-\n-\n-\n-\n-\n-\n+indices.queries.cache.size: 20%\n+\n+##\n+# Thread pool settings\n+##\n+thread_pool:\n+    write:\n+        size: 18\n+        queue_size: 1000\n+\n+\n+\n+\n+\n+##\n+# Size the LTR cache to prevent churn under normal and AB testing scenarios\n+ltr.caches.max_mem: 100mb\n+\n+\n+"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n-    plugins_mandatory => []\n+    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n@@\n-    instances         => {'cloudelastic-chi-eqiad': {'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'heap_memory': '55G', 'tune_gc_new_size_ratio': 3, 'certificate_name': 'cloudelastic.wikimedia.org'}, 'cloudelastic-omega-eqiad': {'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500, 'heap_memory': '14G', 'certificate_name': 'cloudelastic.wikimedia.org'}, 'cloudelastic-psi-eqiad': {'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700, 'heap_memory': '14G', 'certificate_name': 'cloudelastic.wikimedia.org'}}\n+    instances         => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -28,7 +28,7 @@\n #\n # Row/rack awareness attributes\n #\n-\n+cluster.routing.allocation.awareness.attributes: row\n node.attr.row: eqiad-row-a\n node.attr.rack: A2\n #\n@@ -83,7 +83,7 @@\n \n # Set both 'bind_host' and 'publish_host':\n #\n-network.host: [ _local_,_site_ ]\n+network.host: [ 0.0.0.0 ]\n \n # Set a custom port for the node to node communication (9300 by default):\n #\n@@ -118,7 +118,7 @@\n #\n #\n gateway:\n-    recover_after_time: 1s\n+    recover_after_time: 5m\n     expected_nodes: 6\n #\n # ---------------------------------- Various -----------------------------------\n@@ -131,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-#plugin.mandatory: mapper-attachments\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-nori,analysis-smartcn,analysis-stempel,analysis-sudachi,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,9 +140,9 @@\n # still close/delete individual indices.\n action.destructive_requires_name: true\n \n-# Allow up to 1000 shards to be queried at a time. The default\n+# Allow up to 5000 shards to be queried at a time. The default\n # 1k is too low to allow mwgrep to operate.\n-action.search.shard_count.limit: 1000\n+action.search.shard_count.limit: 5000\n \n # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n@@ -180,13 +180,23 @@\n ##\n # Filter cache size\n ##\n-indices.queries.cache.size: 10%\n-\n-\n-\n-\n-\n-\n-\n-\n-\n+indices.queries.cache.size: 20%\n+\n+##\n+# Thread pool settings\n+##\n+thread_pool:\n+    write:\n+        size: 18\n+        queue_size: 1000\n+\n+\n+\n+\n+\n+##\n+# Size the LTR cache to prevent churn under normal and AB testing scenarios\n+ltr.caches.max_mem: 100mb\n+\n+\n+"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -28,7 +28,7 @@\n #\n # Row/rack awareness attributes\n #\n-\n+cluster.routing.allocation.awareness.attributes: row\n node.attr.row: eqiad-row-a\n node.attr.rack: A2\n #\n@@ -83,7 +83,7 @@\n \n # Set both 'bind_host' and 'publish_host':\n #\n-network.host: [ _local_,_site_ ]\n+network.host: [ 0.0.0.0 ]\n \n # Set a custom port for the node to node communication (9300 by default):\n #\n@@ -118,7 +118,7 @@\n #\n #\n gateway:\n-    recover_after_time: 1s\n+    recover_after_time: 5m\n     expected_nodes: 6\n #\n # ---------------------------------- Various -----------------------------------\n@@ -131,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-#plugin.mandatory: mapper-attachments\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-nori,analysis-smartcn,analysis-stempel,analysis-sudachi,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,9 +140,9 @@\n # still close/delete individual indices.\n action.destructive_requires_name: true\n \n-# Allow up to 1000 shards to be queried at a time. The default\n+# Allow up to 5000 shards to be queried at a time. The default\n # 1k is too low to allow mwgrep to operate.\n-action.search.shard_count.limit: 1000\n+action.search.shard_count.limit: 5000\n \n # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n@@ -180,13 +180,23 @@\n ##\n # Filter cache size\n ##\n-indices.queries.cache.size: 10%\n-\n-\n-\n-\n-\n-\n-\n-\n-\n+indices.queries.cache.size: 20%\n+\n+##\n+# Thread pool settings\n+##\n+thread_pool:\n+    write:\n+        size: 18\n+        queue_size: 1000\n+\n+\n+\n+\n+\n+##\n+# Size the LTR cache to prevent churn under normal and AB testing scenarios\n+ltr.caches.max_mem: 100mb\n+\n+\n+"}, {"resource": "Opensearch::Instance[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-chi-eqiad]\n\n+    ltr_cache_size             => 100mb\n+    plugins_mandatory          => ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n+    bulk_thread_pool_executors => 18\n+    awareness_attributes       => row\n+    bulk_thread_pool_capacity  => 1000\n@@\n-    recover_after_time         => 1s\n+    recover_after_time         => 5m\n@@\n-    filter_cache_size          => 10%\n+    filter_cache_size          => 20%\n@@\n-    bind_networks              => ['_local_', '_site_']\n+    bind_networks              => ['0.0.0.0']\n@@\n-    search_shard_count_limit   => 1000\n+    search_shard_count_limit   => 5000\n"}, {"resource": "Opensearch::Instance[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-psi-eqiad]\n\n+    ltr_cache_size             => 100mb\n+    plugins_mandatory          => ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n+    bulk_thread_pool_executors => 18\n+    awareness_attributes       => row\n+    bulk_thread_pool_capacity  => 1000\n@@\n-    recover_after_time         => 1s\n+    recover_after_time         => 5m\n@@\n-    filter_cache_size          => 10%\n+    filter_cache_size          => 20%\n@@\n-    bind_networks              => ['_local_', '_site_']\n+    bind_networks              => ['0.0.0.0']\n@@\n-    search_shard_count_limit   => 1000\n+    search_shard_count_limit   => 5000\n"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n-    common_settings => {'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}}\n+    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}}\n"}, {"resource": "Systemd::Override[opensearch_2@]"}, {"resource": "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "content": "--- /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf.orig\n+++ /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf\n@@ -10,3 +10,23 @@\n TemporaryFileSystem=/usr/share/opensearch/plugins:rw\n \n BindPaths=\n+BindPaths=/usr/share/opensearch/plugins/analysis-hebrew\n+BindPaths=/usr/share/opensearch/plugins/analysis-icu\n+BindPaths=/usr/share/opensearch/plugins/analysis-nori\n+BindPaths=/usr/share/opensearch/plugins/analysis-smartcn\n+BindPaths=/usr/share/opensearch/plugins/analysis-stempel\n+BindPaths=/usr/share/opensearch/plugins/analysis-sudachi\n+BindPaths=/usr/share/opensearch/plugins/analysis-ukrainian\n+BindPaths=/usr/share/opensearch/plugins/cirrus-highlighter\n+BindPaths=/usr/share/opensearch/plugins/opensearch-analysis-stconvert\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-esperanto\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-homoglyph\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-khmer\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-serbian\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-slovak\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-textify\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-turkish\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-ukrainian\n+BindPaths=/usr/share/opensearch/plugins/opensearch-ltr\n+BindPaths=/usr/share/opensearch/plugins/repository-s3"}, {"resource": "Opensearch::Instance[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-omega-eqiad]\n\n+    ltr_cache_size             => 100mb\n+    plugins_mandatory          => ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n+    bulk_thread_pool_executors => 18\n+    awareness_attributes       => row\n+    bulk_thread_pool_capacity  => 1000\n@@\n-    recover_after_time         => 1s\n+    recover_after_time         => 5m\n@@\n-    filter_cache_size          => 10%\n+    filter_cache_size          => 20%\n@@\n-    bind_networks              => ['_local_', '_site_']\n+    bind_networks              => ['0.0.0.0']\n@@\n-    search_shard_count_limit   => 1000\n+    search_shard_count_limit   => 5000\n"}], "perc_changed": "0.30%"}, "core": {"total": 3688, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -28,7 +28,7 @@\n #\n # Row/rack awareness attributes\n #\n-\n+cluster.routing.allocation.awareness.attributes: row\n node.attr.row: eqiad-row-a\n node.attr.rack: A2\n #\n@@ -83,7 +83,7 @@\n \n # Set both 'bind_host' and 'publish_host':\n #\n-network.host: [ _local_,_site_ ]\n+network.host: [ 0.0.0.0 ]\n \n # Set a custom port for the node to node communication (9300 by default):\n #\n@@ -118,7 +118,7 @@\n #\n #\n gateway:\n-    recover_after_time: 1s\n+    recover_after_time: 5m\n     expected_nodes: 6\n #\n # ---------------------------------- Various -----------------------------------\n@@ -131,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-#plugin.mandatory: mapper-attachments\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-nori,analysis-smartcn,analysis-stempel,analysis-sudachi,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,9 +140,9 @@\n # still close/delete individual indices.\n action.destructive_requires_name: true\n \n-# Allow up to 1000 shards to be queried at a time. The default\n+# Allow up to 5000 shards to be queried at a time. The default\n # 1k is too low to allow mwgrep to operate.\n-action.search.shard_count.limit: 1000\n+action.search.shard_count.limit: 5000\n \n # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n@@ -180,13 +180,23 @@\n ##\n # Filter cache size\n ##\n-indices.queries.cache.size: 10%\n-\n-\n-\n-\n-\n-\n-\n-\n-\n+indices.queries.cache.size: 20%\n+\n+##\n+# Thread pool settings\n+##\n+thread_pool:\n+    write:\n+        size: 18\n+        queue_size: 1000\n+\n+\n+\n+\n+\n+##\n+# Size the LTR cache to prevent churn under normal and AB testing scenarios\n+ltr.caches.max_mem: 100mb\n+\n+\n+"}, {"resource": "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "content": "--- /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf.orig\n+++ /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf\n@@ -10,3 +10,23 @@\n TemporaryFileSystem=/usr/share/opensearch/plugins:rw\n \n BindPaths=\n+BindPaths=/usr/share/opensearch/plugins/analysis-hebrew\n+BindPaths=/usr/share/opensearch/plugins/analysis-icu\n+BindPaths=/usr/share/opensearch/plugins/analysis-nori\n+BindPaths=/usr/share/opensearch/plugins/analysis-smartcn\n+BindPaths=/usr/share/opensearch/plugins/analysis-stempel\n+BindPaths=/usr/share/opensearch/plugins/analysis-sudachi\n+BindPaths=/usr/share/opensearch/plugins/analysis-ukrainian\n+BindPaths=/usr/share/opensearch/plugins/cirrus-highlighter\n+BindPaths=/usr/share/opensearch/plugins/opensearch-analysis-stconvert\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-esperanto\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-homoglyph\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-khmer\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-serbian\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-slovak\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-textify\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-turkish\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-ukrainian\n+BindPaths=/usr/share/opensearch/plugins/opensearch-ltr\n+BindPaths=/usr/share/opensearch/plugins/repository-s3"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -28,7 +28,7 @@\n #\n # Row/rack awareness attributes\n #\n-\n+cluster.routing.allocation.awareness.attributes: row\n node.attr.row: eqiad-row-a\n node.attr.rack: A2\n #\n@@ -83,7 +83,7 @@\n \n # Set both 'bind_host' and 'publish_host':\n #\n-network.host: [ _local_,_site_ ]\n+network.host: [ 0.0.0.0 ]\n \n # Set a custom port for the node to node communication (9300 by default):\n #\n@@ -118,7 +118,7 @@\n #\n #\n gateway:\n-    recover_after_time: 1s\n+    recover_after_time: 5m\n     expected_nodes: 6\n #\n # ---------------------------------- Various -----------------------------------\n@@ -131,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-#plugin.mandatory: mapper-attachments\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-nori,analysis-smartcn,analysis-stempel,analysis-sudachi,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,9 +140,9 @@\n # still close/delete individual indices.\n action.destructive_requires_name: true\n \n-# Allow up to 1000 shards to be queried at a time. The default\n+# Allow up to 5000 shards to be queried at a time. The default\n # 1k is too low to allow mwgrep to operate.\n-action.search.shard_count.limit: 1000\n+action.search.shard_count.limit: 5000\n \n # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n@@ -180,13 +180,23 @@\n ##\n # Filter cache size\n ##\n-indices.queries.cache.size: 10%\n-\n-\n-\n-\n-\n-\n-\n-\n-\n+indices.queries.cache.size: 20%\n+\n+##\n+# Thread pool settings\n+##\n+thread_pool:\n+    write:\n+        size: 18\n+        queue_size: 1000\n+\n+\n+\n+\n+\n+##\n+# Size the LTR cache to prevent churn under normal and AB testing scenarios\n+ltr.caches.max_mem: 100mb\n+\n+\n+"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -28,7 +28,7 @@\n #\n # Row/rack awareness attributes\n #\n-\n+cluster.routing.allocation.awareness.attributes: row\n node.attr.row: eqiad-row-a\n node.attr.rack: A2\n #\n@@ -83,7 +83,7 @@\n \n # Set both 'bind_host' and 'publish_host':\n #\n-network.host: [ _local_,_site_ ]\n+network.host: [ 0.0.0.0 ]\n \n # Set a custom port for the node to node communication (9300 by default):\n #\n@@ -118,7 +118,7 @@\n #\n #\n gateway:\n-    recover_after_time: 1s\n+    recover_after_time: 5m\n     expected_nodes: 6\n #\n # ---------------------------------- Various -----------------------------------\n@@ -131,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-#plugin.mandatory: mapper-attachments\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-nori,analysis-smartcn,analysis-stempel,analysis-sudachi,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,9 +140,9 @@\n # still close/delete individual indices.\n action.destructive_requires_name: true\n \n-# Allow up to 1000 shards to be queried at a time. The default\n+# Allow up to 5000 shards to be queried at a time. The default\n # 1k is too low to allow mwgrep to operate.\n-action.search.shard_count.limit: 1000\n+action.search.shard_count.limit: 5000\n \n # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n@@ -180,13 +180,23 @@\n ##\n # Filter cache size\n ##\n-indices.queries.cache.size: 10%\n-\n-\n-\n-\n-\n-\n-\n-\n-\n+indices.queries.cache.size: 20%\n+\n+##\n+# Thread pool settings\n+##\n+thread_pool:\n+    write:\n+        size: 18\n+        queue_size: 1000\n+\n+\n+\n+\n+\n+##\n+# Size the LTR cache to prevent churn under normal and AB testing scenarios\n+ltr.caches.max_mem: 100mb\n+\n+\n+"}], "perc_changed": "0.11%"}, "main": {"total": 3688, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "Opensearch::Instance[cloudelastic-chi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-chi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-chi-eqiad]\n\n+    ltr_cache_size             => 100mb\n+    plugins_mandatory          => ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n+    bulk_thread_pool_executors => 18\n+    awareness_attributes       => row\n+    bulk_thread_pool_capacity  => 1000\n@@\n-    recover_after_time         => 1s\n+    recover_after_time         => 5m\n@@\n-    filter_cache_size          => 10%\n+    filter_cache_size          => 20%\n@@\n-    bind_networks              => ['_local_', '_site_']\n+    bind_networks              => ['0.0.0.0']\n@@\n-    search_shard_count_limit   => 1000\n+    search_shard_count_limit   => 5000\n"}, {"resource": "Opensearch::Instance[cloudelastic-psi-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-psi-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-psi-eqiad]\n\n+    ltr_cache_size             => 100mb\n+    plugins_mandatory          => ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n+    bulk_thread_pool_executors => 18\n+    awareness_attributes       => row\n+    bulk_thread_pool_capacity  => 1000\n@@\n-    recover_after_time         => 1s\n+    recover_after_time         => 5m\n@@\n-    filter_cache_size          => 10%\n+    filter_cache_size          => 20%\n@@\n-    bind_networks              => ['_local_', '_site_']\n+    bind_networks              => ['0.0.0.0']\n@@\n-    search_shard_count_limit   => 1000\n+    search_shard_count_limit   => 5000\n"}, {"resource": "Class[Profile::Opensearch::Server]", "parameters": "--- Class[Profile::Opensearch::Server].orig\n+++ Class[Profile::Opensearch::Server]\n\n@@\n-    common_settings => {'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}}\n+    common_settings => {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '30G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}}\n"}, {"resource": "Systemd::Unit[opensearch_2@-opensearch_2@]"}, {"resource": "Systemd::Override[opensearch_2@]"}, {"resource": "File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml\n@@ -28,7 +28,7 @@\n #\n # Row/rack awareness attributes\n #\n-\n+cluster.routing.allocation.awareness.attributes: row\n node.attr.row: eqiad-row-a\n node.attr.rack: A2\n #\n@@ -83,7 +83,7 @@\n \n # Set both 'bind_host' and 'publish_host':\n #\n-network.host: [ _local_,_site_ ]\n+network.host: [ 0.0.0.0 ]\n \n # Set a custom port for the node to node communication (9300 by default):\n #\n@@ -118,7 +118,7 @@\n #\n #\n gateway:\n-    recover_after_time: 1s\n+    recover_after_time: 5m\n     expected_nodes: 6\n #\n # ---------------------------------- Various -----------------------------------\n@@ -131,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-#plugin.mandatory: mapper-attachments\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-nori,analysis-smartcn,analysis-stempel,analysis-sudachi,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,9 +140,9 @@\n # still close/delete individual indices.\n action.destructive_requires_name: true\n \n-# Allow up to 1000 shards to be queried at a time. The default\n+# Allow up to 5000 shards to be queried at a time. The default\n # 1k is too low to allow mwgrep to operate.\n-action.search.shard_count.limit: 1000\n+action.search.shard_count.limit: 5000\n \n # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n@@ -180,13 +180,23 @@\n ##\n # Filter cache size\n ##\n-indices.queries.cache.size: 10%\n-\n-\n-\n-\n-\n-\n-\n-\n-\n+indices.queries.cache.size: 20%\n+\n+##\n+# Thread pool settings\n+##\n+thread_pool:\n+    write:\n+        size: 18\n+        queue_size: 1000\n+\n+\n+\n+\n+\n+##\n+# Size the LTR cache to prevent churn under normal and AB testing scenarios\n+ltr.caches.max_mem: 100mb\n+\n+\n+"}, {"resource": "File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]", "content": "--- /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf.orig\n+++ /etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf\n@@ -10,3 +10,23 @@\n TemporaryFileSystem=/usr/share/opensearch/plugins:rw\n \n BindPaths=\n+BindPaths=/usr/share/opensearch/plugins/analysis-hebrew\n+BindPaths=/usr/share/opensearch/plugins/analysis-icu\n+BindPaths=/usr/share/opensearch/plugins/analysis-nori\n+BindPaths=/usr/share/opensearch/plugins/analysis-smartcn\n+BindPaths=/usr/share/opensearch/plugins/analysis-stempel\n+BindPaths=/usr/share/opensearch/plugins/analysis-sudachi\n+BindPaths=/usr/share/opensearch/plugins/analysis-ukrainian\n+BindPaths=/usr/share/opensearch/plugins/cirrus-highlighter\n+BindPaths=/usr/share/opensearch/plugins/opensearch-analysis-stconvert\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-esperanto\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-homoglyph\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-khmer\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-serbian\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-slovak\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-textify\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-turkish\n+BindPaths=/usr/share/opensearch/plugins/opensearch-extra-analysis-ukrainian\n+BindPaths=/usr/share/opensearch/plugins/opensearch-ltr\n+BindPaths=/usr/share/opensearch/plugins/repository-s3"}, {"resource": "Opensearch::Instance[cloudelastic-omega-eqiad]", "parameters": "--- Opensearch::Instance[cloudelastic-omega-eqiad].orig\n+++ Opensearch::Instance[cloudelastic-omega-eqiad]\n\n+    ltr_cache_size             => 100mb\n+    plugins_mandatory          => ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n+    bulk_thread_pool_executors => 18\n+    awareness_attributes       => row\n+    bulk_thread_pool_capacity  => 1000\n@@\n-    recover_after_time         => 1s\n+    recover_after_time         => 5m\n@@\n-    filter_cache_size          => 10%\n+    filter_cache_size          => 20%\n@@\n-    bind_networks              => ['_local_', '_site_']\n+    bind_networks              => ['0.0.0.0']\n@@\n-    search_shard_count_limit   => 1000\n+    search_shard_count_limit   => 5000\n"}, {"resource": "Class[Opensearch]", "parameters": "--- Class[Opensearch].orig\n+++ Class[Opensearch]\n\n@@\n-    plugins_mandatory => []\n+    plugins_mandatory => ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3']\n@@\n-    instances         => {'cloudelastic-chi-eqiad': {'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'heap_memory': '55G', 'tune_gc_new_size_ratio': 3, 'certificate_name': 'cloudelastic.wikimedia.org'}, 'cloudelastic-omega-eqiad': {'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500, 'heap_memory': '14G', 'certificate_name': 'cloudelastic.wikimedia.org'}, 'cloudelastic-psi-eqiad': {'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700, 'heap_memory': '14G', 'certificate_name': 'cloudelastic.wikimedia.org'}}\n+    instances         => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}\n"}, {"resource": "File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml\n@@ -28,7 +28,7 @@\n #\n # Row/rack awareness attributes\n #\n-\n+cluster.routing.allocation.awareness.attributes: row\n node.attr.row: eqiad-row-a\n node.attr.rack: A2\n #\n@@ -83,7 +83,7 @@\n \n # Set both 'bind_host' and 'publish_host':\n #\n-network.host: [ _local_,_site_ ]\n+network.host: [ 0.0.0.0 ]\n \n # Set a custom port for the node to node communication (9300 by default):\n #\n@@ -118,7 +118,7 @@\n #\n #\n gateway:\n-    recover_after_time: 1s\n+    recover_after_time: 5m\n     expected_nodes: 6\n #\n # ---------------------------------- Various -----------------------------------\n@@ -131,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-#plugin.mandatory: mapper-attachments\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-nori,analysis-smartcn,analysis-stempel,analysis-sudachi,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,9 +140,9 @@\n # still close/delete individual indices.\n action.destructive_requires_name: true\n \n-# Allow up to 1000 shards to be queried at a time. The default\n+# Allow up to 5000 shards to be queried at a time. The default\n # 1k is too low to allow mwgrep to operate.\n-action.search.shard_count.limit: 1000\n+action.search.shard_count.limit: 5000\n \n # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n@@ -180,13 +180,23 @@\n ##\n # Filter cache size\n ##\n-indices.queries.cache.size: 10%\n-\n-\n-\n-\n-\n-\n-\n-\n-\n+indices.queries.cache.size: 20%\n+\n+##\n+# Thread pool settings\n+##\n+thread_pool:\n+    write:\n+        size: 18\n+        queue_size: 1000\n+\n+\n+\n+\n+\n+##\n+# Size the LTR cache to prevent churn under normal and AB testing scenarios\n+ltr.caches.max_mem: 100mb\n+\n+\n+"}, {"resource": "File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]", "content": "--- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig\n+++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml\n@@ -28,7 +28,7 @@\n #\n # Row/rack awareness attributes\n #\n-\n+cluster.routing.allocation.awareness.attributes: row\n node.attr.row: eqiad-row-a\n node.attr.rack: A2\n #\n@@ -83,7 +83,7 @@\n \n # Set both 'bind_host' and 'publish_host':\n #\n-network.host: [ _local_,_site_ ]\n+network.host: [ 0.0.0.0 ]\n \n # Set a custom port for the node to node communication (9300 by default):\n #\n@@ -118,7 +118,7 @@\n #\n #\n gateway:\n-    recover_after_time: 1s\n+    recover_after_time: 5m\n     expected_nodes: 6\n #\n # ---------------------------------- Various -----------------------------------\n@@ -131,7 +131,7 @@\n #\n # If a plugin listed here is not installed for current node, the node will not start.\n #\n-#plugin.mandatory: mapper-attachments\n+plugin.mandatory: analysis-hebrew,analysis-icu,analysis-nori,analysis-smartcn,analysis-stempel,analysis-sudachi,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-homoglyph,opensearch-extra-analysis-khmer,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-extra-analysis-textify,opensearch-extra-analysis-turkish,opensearch-extra-analysis-ukrainian,opensearch-ltr,repository-s3\n \n # Specify the pattern of index names allowed to be created automatically\n action.auto_create_index: false\n@@ -140,9 +140,9 @@\n # still close/delete individual indices.\n action.destructive_requires_name: true\n \n-# Allow up to 1000 shards to be queried at a time. The default\n+# Allow up to 5000 shards to be queried at a time. The default\n # 1k is too low to allow mwgrep to operate.\n-action.search.shard_count.limit: 1000\n+action.search.shard_count.limit: 5000\n \n # https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation\n # Enable the disk space aware shard allocator\n@@ -180,13 +180,23 @@\n ##\n # Filter cache size\n ##\n-indices.queries.cache.size: 10%\n-\n-\n-\n-\n-\n-\n-\n-\n-\n+indices.queries.cache.size: 20%\n+\n+##\n+# Thread pool settings\n+##\n+thread_pool:\n+    write:\n+        size: 18\n+        queue_size: 1000\n+\n+\n+\n+\n+\n+##\n+# Size the LTR cache to prevent churn under normal and AB testing scenarios\n+ltr.caches.max_mem: 100mb\n+\n+\n+"}], "perc_changed": "0.30%"}}}