--- Class[Opensearch].orig
+++ Class[Opensearch]
@@
- instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}
+ instances => {'cloudelastic-chi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '55G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-chi-eqiad', 'short_cluster_name': 'chi', 'http_port': 9200, 'tls_port': 9243, 'tls_ro_port': 8243, 'transport_tcp_port': 9300, 'tune_gc_new_size_ratio': 3}, 'cloudelastic-omega-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-omega-eqiad', 'short_cluster_name': 'omega', 'http_port': 9400, 'tls_port': 9443, 'tls_ro_port': 8443, 'transport_tcp_port': 9500}, 'cloudelastic-psi-eqiad': {'awareness_attributes': 'row', 'plugins_mandatory': ['analysis-hebrew', 'analysis-icu', 'analysis-nori', 'analysis-smartcn', 'analysis-stempel', 'analysis-sudachi', 'analysis-ukrainian', 'cirrus-highlighter', 'opensearch-analysis-stconvert', 'opensearch-extra', 'opensearch-extra-analysis-esperanto', 'opensearch-extra-analysis-homoglyph', 'opensearch-extra-analysis-khmer', 'opensearch-extra-analysis-serbian', 'opensearch-extra-analysis-slovak', 'opensearch-extra-analysis-textify', 'opensearch-extra-analysis-turkish', 'opensearch-extra-analysis-ukrainian', 'opensearch-ltr', 'repository-s3'], 'heap_memory': '14G', 'bind_networks': ['0.0.0.0'], 'certificate_name': 'cloudelastic.wikimedia.org', 'recover_after_time': '5m', 'search_shard_count_limit': 5000, 'ltr_cache_size': '100mb', 'bulk_thread_pool_executors': 18, 'bulk_thread_pool_capacity': 1000, 'filter_cache_size': '20%', 'watermarks': {'low': '100gb', 'high': '50gb', 'flood_stage': '10gb'}, 'unicast_hosts': ['cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet'], 'cluster_hosts': ['cloudelastic1007.eqiad.wmnet', 'cloudelastic1008.eqiad.wmnet', 'cloudelastic1009.eqiad.wmnet', 'cloudelastic1010.eqiad.wmnet', 'cloudelastic1011.eqiad.wmnet', 'cloudelastic1012.eqiad.wmnet'], 'expected_nodes': 6, 'recover_after_nodes': 3, 'cluster_name': 'cloudelastic-psi-eqiad', 'short_cluster_name': 'psi', 'http_port': 9600, 'tls_port': 9643, 'tls_ro_port': 8643, 'transport_tcp_port': 9700}}
Class[Lvs::Realserver]
- Parameters differences:
--- Class[Lvs::Realserver].orig
+++ Class[Lvs::Realserver]
@@
- realserver_ips => ['10.2.2.30']
+ realserver_ips => ['208.80.154.241', '2620:0:861:ed1a::3:241']
- Systemd::Timer::Job[prometheus_ferm_mss]
- Parameters differences:
--- Systemd::Timer::Job[prometheus_ferm_mss].orig
+++ Systemd::Timer::Job[prometheus_ferm_mss]
@@
- command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443
+ command => /usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643
- File[/etc/ferm/conf.d/10_clamp-mss-ipv4]
- Content differences:
--- /etc/ferm/conf.d/10_clamp-mss-ipv4.orig
+++ /etc/ferm/conf.d/10_clamp-mss-ipv4
@@ -5,7 +5,7 @@
domain (ip) {
table filter {
chain OUTPUT {
- outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;
+ outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;
}
}
}
- File[/etc/conftool/local_services.yaml]
- Content differences:
--- /etc/conftool/local_services.yaml.orig
+++ /etc/conftool/local_services.yaml
@@ -1,19 +1,19 @@
---
-search:
- cluster: elasticsearch
- service: elasticsearch
+cloudelastic-chi-https:
+ cluster: cloudelastic
+ service: cloudelastic-chi-ssl
servers:
- - pybal-low-traffic.svc.eqiad.wmnet
- port: 9200
-search-https:
- cluster: elasticsearch
- service: elasticsearch-ssl
+ - pybal-high-traffic2-eqiad.wikimedia.org
+ port: 9243
+cloudelastic-omega-https:
+ cluster: cloudelastic
+ service: cloudelastic-omega-ssl
servers:
- - pybal-low-traffic.svc.eqiad.wmnet
- port: 9243
-search-omega-https:
- cluster: elasticsearch
- service: elasticsearch-omega-ssl
+ - pybal-high-traffic2-eqiad.wikimedia.org
+ port: 9443
+cloudelastic-psi-https:
+ cluster: cloudelastic
+ service: cloudelastic-psi-ssl
servers:
- - pybal-low-traffic.svc.eqiad.wmnet
- port: 9443
+ - pybal-high-traffic2-eqiad.wikimedia.org
+ port: 9643
- File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]
- Content differences:
--- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig
+++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml
@@ -148,11 +148,11 @@
# Enable the disk space aware shard allocator
cluster.routing.allocation.disk.threshold_enabled: true
# Stop allocating shards to nodes (default 80%)
-cluster.routing.allocation.disk.watermark.low: 0.75
+cluster.routing.allocation.disk.watermark.low: 100gb
# Begin evicting shards off the node (default 90%)
-cluster.routing.allocation.disk.watermark.high: 0.80
+cluster.routing.allocation.disk.watermark.high: 50gb
# Set indexes read-only with allocated shards on affected node (default 95%)
-cluster.routing.allocation.disk.watermark.flood_stage: 0.95
+cluster.routing.allocation.disk.watermark.flood_stage: 10gb
# Check disk utilization every 60 seconds
cluster.info.update.interval: 60s
- File[/lib/systemd/system/tcp-mss-clamper.service]
- Content differences:
--- /lib/systemd/system/tcp-mss-clamper.service.orig
+++ /lib/systemd/system/tcp-mss-clamper.service
@@ -7,5 +7,5 @@
[Service]
LimitMEMLOCK=infinity
-ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s "10.2.2.30:9200,10.2.2.30:9243,10.2.2.30:9443" -i enp152s0f0np0,lo
+ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s "208.80.154.241:9243,208.80.154.241:9443,208.80.154.241:9643,[2620:0:861:ed1a::3:241]:9243,[2620:0:861:ed1a::3:241]:9443,[2620:0:861:ed1a::3:241]:9643" -i enp152s0f0np0,lo
Restart=on-failure
- Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]
- Parameters differences:
--- Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport].orig
+++ Prometheus::Node_lvs_realserver_mss[lvs_clamped_ipport]
@@
- clamped_ipport => ['10.2.2.30:9200', '10.2.2.30:9243', '10.2.2.30:9443']
+ clamped_ipport => ['208.80.154.241:9243', '208.80.154.241:9443', '208.80.154.241:9643', '[2620:0:861:ed1a::3:241]:9243', '[2620:0:861:ed1a::3:241]:9443', '[2620:0:861:ed1a::3:241]:9643']
- File[/lib/systemd/system/prometheus_ferm_mss.service]
- Content differences:
--- /lib/systemd/system/prometheus_ferm_mss.service.orig
+++ /lib/systemd/system/prometheus_ferm_mss.service
@@ -5,4 +5,4 @@
[Service]
Type=oneshot
User=root
-ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443
+ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643
- File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]
- Content differences:
--- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig
+++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml
@@ -148,11 +148,11 @@
# Enable the disk space aware shard allocator
cluster.routing.allocation.disk.threshold_enabled: true
# Stop allocating shards to nodes (default 80%)
-cluster.routing.allocation.disk.watermark.low: 0.75
+cluster.routing.allocation.disk.watermark.low: 100gb
# Begin evicting shards off the node (default 90%)
-cluster.routing.allocation.disk.watermark.high: 0.80
+cluster.routing.allocation.disk.watermark.high: 50gb
# Set indexes read-only with allocated shards on affected node (default 95%)
-cluster.routing.allocation.disk.watermark.flood_stage: 0.95
+cluster.routing.allocation.disk.watermark.flood_stage: 10gb
# Check disk utilization every 60 seconds
cluster.info.update.interval: 60s
- Systemd::Unit[prometheus_ferm_mss.service]
- Systemd::Service[tcp-mss-clamper]
Relevant files