Compilation results for cloudelastic1011.eqiad.wmnet: System changes detected

Catalog differences

Summary

Resources only in the new catalog

• File[/usr/local/bin/depool-opensearch_2@production-search-eqiad]
• File[/usr/local/sbin/restart-opensearch_2@production-search-omega-eqiad]
• File[/usr/local/sbin/restart-nginx]
• Service[opensearch_2@cloudelastic-chi-eqiad]
• File[/usr/local/bin/pool-nginx]
• Exec[install-opensearch]
• File[/usr/local/bin/depool-opensearch_2@production-search-omega-eqiad]
• Service[opensearch_2@cloudelastic-psi-eqiad]
• File[/usr/local/bin/pool-opensearch_2@production-search-omega-eqiad]
• Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@.service)]
• File[/etc/systemd/system/opensearch_2@.service.d/opensearch_2@.conf]
• File[/usr/local/sbin/restart-opensearch_2@production-search-eqiad]
• File[/usr/local/bin/depool-nginx]
• Service[opensearch_2@cloudelastic-omega-eqiad]
• Exec[systemd daemon-reload for opensearch_2@.service (opensearch_2@-opensearch_2@)]
• File[/etc/systemd/system/opensearch_2@.service.d]
• File[/lib/systemd/system/opensearch_2@.service]
• File[/usr/local/bin/pool-opensearch_2@production-search-eqiad]

Resources only in the old catalog

• File[/usr/local/bin/depool-opensearch_1@cloudelastic-omega-eqiad.service]
• File[/usr/local/bin/depool-opensearch_1@cloudelastic-chi-eqiad.service]
• File[/usr/local/bin/pool-opensearch_1@cloudelastic-chi-eqiad.service]
• Service[opensearch_1@cloudelastic-psi-eqiad]
• File[/lib/systemd/system/opensearch_1@.service]
• Service[opensearch_1@cloudelastic-omega-eqiad]
• File[/usr/local/bin/depool-opensearch_1@cloudelastic-psi-eqiad.service]
• File[/usr/local/sbin/restart-opensearch_1@cloudelastic-psi-eqiad.service]
• Service[opensearch_1@cloudelastic-chi-eqiad]
• File[/usr/local/sbin/restart-opensearch_1@cloudelastic-omega-eqiad.service]
• File[/usr/local/sbin/restart-opensearch_1@cloudelastic-chi-eqiad.service]
• File[/usr/local/bin/pool-opensearch_1@cloudelastic-omega-eqiad.service]
• File[/usr/local/bin/pool-opensearch_1@cloudelastic-psi-eqiad.service]
• Exec[systemd daemon-reload for opensearch_1@.service (opensearch_1@.service)]

Resources modified

• File[/lib/systemd/system/tcp-mss-clamper.service]

  Content differences:

  --- /lib/systemd/system/tcp-mss-clamper.service.orig
  +++ /lib/systemd/system/tcp-mss-clamper.service
  @@ -7,5 +7,5 @@
   
   [Service]
   LimitMEMLOCK=infinity
  -ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s "208.80.154.241:9243,208.80.154.241:9443,208.80.154.241:9643,[2620:0:861:ed1a::3:241]:9243,[2620:0:861:ed1a::3:241]:9443,[2620:0:861:ed1a::3:241]:9643" -i enp152s0f0np0,lo
  +ExecStart=/usr/bin/tcp-mss-clamper --ipv4-mss 1440 --ipv6-mss 1400 -p :2200 -s "10.2.2.30:9200,10.2.2.30:9243,10.2.2.30:9443" -i enp152s0f0np0,lo
   Restart=on-failure

• File[/etc/ferm/conf.d/10_clamp-mss-ipv6]

  Content differences:

  --- /etc/ferm/conf.d/10_clamp-mss-ipv6.orig
  +++ /etc/ferm/conf.d/10_clamp-mss-ipv6
  @@ -5,7 +5,7 @@
   domain (ip6) {
   	table filter {
   		chain OUTPUT {
  -			outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1400;
  +			outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1400;
   		}
   	}
   }

• File[/lib/systemd/system/prometheus_lvs_realserver_mss.service]

  Content differences:

  --- /lib/systemd/system/prometheus_lvs_realserver_mss.service.orig
  +++ /lib/systemd/system/prometheus_lvs_realserver_mss.service
  @@ -5,4 +5,4 @@
   [Service]
   Type=oneshot
   User=root
  -ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643
  +ExecStart=/usr/local/bin/prometheus-lvs-realserver-mss -o /var/lib/prometheus/node.d/lvs-realserver-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443

• File[/etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml]

  Content differences:

  --- /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml.orig
  +++ /etc/opensearch/cloudelastic-chi-eqiad/opensearch.yml
  @@ -100,6 +100,10 @@
   #
   #cluster.initial_master_nodes: ["node-1", "node-2"]
   #
  +# Set to ensure a node sees N other master eligible nodes to be considered
  +# operational within the cluster. Its recommended to set it to a higher value
  +# than 1 when running more than 2 nodes in the cluster.
  +#
   discovery.zen.ping.unicast.hosts: ["cloudelastic1009.eqiad.wmnet", "cloudelastic1010.eqiad.wmnet", "cloudelastic1011.eqiad.wmnet"]
   #
   # For more information, consult the discovery and cluster formation module documentation.
  @@ -127,7 +131,7 @@
   #
   # If a plugin listed here is not installed for current node, the node will not start.
   #
  -plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr
  +plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3
   
   # Specify the pattern of index names allowed to be created automatically
   action.auto_create_index: false
  @@ -140,12 +144,16 @@
   # 1k is too low to allow mwgrep to operate.
   action.search.shard_count.limit: 5000
   
  +# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation
   # Enable the disk space aware shard allocator
   cluster.routing.allocation.disk.threshold_enabled: true
  -# Stop allocating new shards to nodes with 75% disk utilization
  +# Stop allocating shards to nodes (default 80%)
   cluster.routing.allocation.disk.watermark.low: 0.75
  -# Move shards off of nodes with 80% disk utilization
  +# Begin evicting shards off the node (default 90%)
   cluster.routing.allocation.disk.watermark.high: 0.80
  +# Set indexes read-only with allocated shards on affected node (default 95%)
  +cluster.routing.allocation.disk.watermark.flood_stage: 0.95
  +
   # Check disk utilization every 60 seconds
   cluster.info.update.interval: 60s
   
  @@ -191,3 +199,4 @@
   ltr.caches.max_mem: 100mb
   
   
  +

• File[/etc/default/wikimedia-lvs-realserver]

  Content differences:

  --- /etc/default/wikimedia-lvs-realserver.orig
  +++ /etc/default/wikimedia-lvs-realserver
  @@ -7,4 +7,4 @@
   
   # LVS service IPs to be bound to the loopback interface,
   # separate using spaces
  -LVS_SERVICE_IPS="208.80.154.241 2620:0:861:ed1a::3:241"
  +LVS_SERVICE_IPS="10.2.2.30"

• Package[wmf-opensearch-search-plugins]

  Parameters differences:

  --- Package[wmf-opensearch-search-plugins].orig
  +++ Package[wmf-opensearch-search-plugins]
  
  @@
  -    before => ['Service[opensearch_1@cloudelastic-chi-eqiad]', 'Service[opensearch_1@cloudelastic-omega-eqiad]', 'Service[opensearch_1@cloudelastic-psi-eqiad]']
  +    before => ['Service[opensearch_2@cloudelastic-chi-eqiad]', 'Service[opensearch_2@cloudelastic-omega-eqiad]', 'Service[opensearch_2@cloudelastic-psi-eqiad]']
  

• File[/etc/conftool/local_services.yaml]

  Content differences:

  --- /etc/conftool/local_services.yaml.orig
  +++ /etc/conftool/local_services.yaml
  @@ -1,19 +1,19 @@
   ---
  -cloudelastic-chi-https:
  -  cluster: cloudelastic
  -  service: cloudelastic-chi-ssl
  +search:
  +  cluster: elasticsearch
  +  service: elasticsearch
     servers:
  -  - pybal-high-traffic2-eqiad.wikimedia.org
  +  - pybal-low-traffic.svc.eqiad.wmnet
  +  port: 9200
  +search-https:
  +  cluster: elasticsearch
  +  service: elasticsearch-ssl
  +  servers:
  +  - pybal-low-traffic.svc.eqiad.wmnet
     port: 9243
  -cloudelastic-omega-https:
  -  cluster: cloudelastic
  -  service: cloudelastic-omega-ssl
  +search-omega-https:
  +  cluster: elasticsearch
  +  service: elasticsearch-omega-ssl
     servers:
  -  - pybal-high-traffic2-eqiad.wikimedia.org
  +  - pybal-low-traffic.svc.eqiad.wmnet
     port: 9443
  -cloudelastic-psi-https:
  -  cluster: cloudelastic
  -  service: cloudelastic-psi-ssl
  -  servers:
  -  - pybal-high-traffic2-eqiad.wikimedia.org
  -  port: 9643

• File[/etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml]

  Content differences:

  --- /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml.orig
  +++ /etc/opensearch/cloudelastic-psi-eqiad/opensearch.yml
  @@ -100,6 +100,10 @@
   #
   #cluster.initial_master_nodes: ["node-1", "node-2"]
   #
  +# Set to ensure a node sees N other master eligible nodes to be considered
  +# operational within the cluster. Its recommended to set it to a higher value
  +# than 1 when running more than 2 nodes in the cluster.
  +#
   discovery.zen.ping.unicast.hosts: ["cloudelastic1009.eqiad.wmnet", "cloudelastic1010.eqiad.wmnet", "cloudelastic1011.eqiad.wmnet"]
   #
   # For more information, consult the discovery and cluster formation module documentation.
  @@ -127,7 +131,7 @@
   #
   # If a plugin listed here is not installed for current node, the node will not start.
   #
  -plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr
  +plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3
   
   # Specify the pattern of index names allowed to be created automatically
   action.auto_create_index: false
  @@ -140,12 +144,16 @@
   # 1k is too low to allow mwgrep to operate.
   action.search.shard_count.limit: 5000
   
  +# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation
   # Enable the disk space aware shard allocator
   cluster.routing.allocation.disk.threshold_enabled: true
  -# Stop allocating new shards to nodes with 75% disk utilization
  +# Stop allocating shards to nodes (default 80%)
   cluster.routing.allocation.disk.watermark.low: 0.75
  -# Move shards off of nodes with 80% disk utilization
  +# Begin evicting shards off the node (default 90%)
   cluster.routing.allocation.disk.watermark.high: 0.80
  +# Set indexes read-only with allocated shards on affected node (default 95%)
  +cluster.routing.allocation.disk.watermark.flood_stage: 0.95
  +
   # Check disk utilization every 60 seconds
   cluster.info.update.interval: 60s
   
  @@ -191,3 +199,4 @@
   ltr.caches.max_mem: 100mb
   
   
  +

• File[/etc/ferm/conf.d/10_clamp-mss-ipv4]

  Content differences:

  --- /etc/ferm/conf.d/10_clamp-mss-ipv4.orig
  +++ /etc/ferm/conf.d/10_clamp-mss-ipv4
  @@ -5,7 +5,7 @@
   domain (ip) {
   	table filter {
   		chain OUTPUT {
  -			outerface (enp152s0f0np0 lo) saddr @ipfilter((208.80.154.241 2620:0:861:ed1a::3:241])) proto tcp sport (9243 9443 9643) tcp-flags (SYN) SYN TCPMSS set-mss 1440;
  +			outerface (enp152s0f0np0 lo) saddr @ipfilter(10.2.2.30) proto tcp sport (9200 9243 9443) tcp-flags (SYN) SYN TCPMSS set-mss 1440;
   		}
   	}
   }

• File[/lib/systemd/system/prometheus_ferm_mss.service]

  Content differences:

  --- /lib/systemd/system/prometheus_ferm_mss.service.orig
  +++ /lib/systemd/system/prometheus_ferm_mss.service
  @@ -5,4 +5,4 @@
   [Service]
   Type=oneshot
   User=root
  -ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 208.80.154.241:9243 -e 208.80.154.241:9443 -e 208.80.154.241:9643 -e [2620:0:861:ed1a::3:241]:9243 -e [2620:0:861:ed1a::3:241]:9443 -e [2620:0:861:ed1a::3:241]:9643
  +ExecStart=/usr/local/bin/prometheus-ferm-mss -o /var/lib/prometheus/node.d/ferm-mss.prom -e 10.2.2.30:9200 -e 10.2.2.30:9243 -e 10.2.2.30:9443

• File[/etc/apt/sources.list.d/wikimedia-opensearch.list]

  Content differences:

  --- /etc/apt/sources.list.d/wikimedia-opensearch.list.orig
  +++ /etc/apt/sources.list.d/wikimedia-opensearch.list
  @@ -1,2 +1,2 @@
  -deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1
  -deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch1
  +deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2
  +deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia thirdparty/opensearch2

• File[/etc/opensearch/instances]

  Content differences:

  --- /etc/opensearch/instances.orig
  +++ /etc/opensearch/instances
  @@ -1,3 +1,3 @@
  -opensearch_1@cloudelastic-chi-eqiad
  -opensearch_1@cloudelastic-omega-eqiad
  -opensearch_1@cloudelastic-psi-eqiad
  +opensearch_2@cloudelastic-chi-eqiad
  +opensearch_2@cloudelastic-omega-eqiad
  +opensearch_2@cloudelastic-psi-eqiad

• File[/etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties]

  Content differences:

  --- /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties.orig
  +++ /etc/opensearch/cloudelastic-psi-eqiad/log4j2.properties
  @@ -185,7 +185,6 @@
   rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash
   logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash
   logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
  -logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
   
   # Define Logstash appender
   appender.ship_to_logstash.name=ship_to_logstash

• File[/etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties]

  Content differences:

  --- /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties.orig
  +++ /etc/opensearch/cloudelastic-omega-eqiad/log4j2.properties
  @@ -185,7 +185,6 @@
   rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash
   logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash
   logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
  -logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
   
   # Define Logstash appender
   appender.ship_to_logstash.name=ship_to_logstash

• Package[opensearch]

  Parameters differences:

  --- Package[opensearch].orig
  +++ Package[opensearch]
  
  @@
  -    ensure => 1.3.20
  +    ensure => 2.19.5
  

• File[/etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml]

  Content differences:

  --- /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml.orig
  +++ /etc/opensearch/cloudelastic-omega-eqiad/opensearch.yml
  @@ -100,6 +100,10 @@
   #
   #cluster.initial_master_nodes: ["node-1", "node-2"]
   #
  +# Set to ensure a node sees N other master eligible nodes to be considered
  +# operational within the cluster. Its recommended to set it to a higher value
  +# than 1 when running more than 2 nodes in the cluster.
  +#
   discovery.zen.ping.unicast.hosts: ["cloudelastic1009.eqiad.wmnet", "cloudelastic1010.eqiad.wmnet", "cloudelastic1011.eqiad.wmnet"]
   #
   # For more information, consult the discovery and cluster formation module documentation.
  @@ -127,7 +131,7 @@
   #
   # If a plugin listed here is not installed for current node, the node will not start.
   #
  -plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,ltr
  +plugin.mandatory: analysis-hebrew,analysis-icu,analysis-smartcn,analysis-stempel,analysis-ukrainian,cirrus-highlighter,opensearch-analysis-stconvert,opensearch-extra,opensearch-extra-analysis-esperanto,opensearch-extra-analysis-serbian,opensearch-extra-analysis-slovak,opensearch-ltr,repository-s3
   
   # Specify the pattern of index names allowed to be created automatically
   action.auto_create_index: false
  @@ -140,12 +144,16 @@
   # 1k is too low to allow mwgrep to operate.
   action.search.shard_count.limit: 5000
   
  +# https://www.elastic.co/guide/en/elasticsearch/reference/7.12/modules-cluster.html#disk-based-shard-allocation
   # Enable the disk space aware shard allocator
   cluster.routing.allocation.disk.threshold_enabled: true
  -# Stop allocating new shards to nodes with 75% disk utilization
  +# Stop allocating shards to nodes (default 80%)
   cluster.routing.allocation.disk.watermark.low: 0.75
  -# Move shards off of nodes with 80% disk utilization
  +# Begin evicting shards off the node (default 90%)
   cluster.routing.allocation.disk.watermark.high: 0.80
  +# Set indexes read-only with allocated shards on affected node (default 95%)
  +cluster.routing.allocation.disk.watermark.flood_stage: 0.95
  +
   # Check disk utilization every 60 seconds
   cluster.info.update.interval: 60s
   
  @@ -191,3 +199,4 @@
   ltr.caches.max_mem: 100mb
   
   
  +

• File[/etc/apt/sources.list.d/wikimedia-opensearch-plugins.list]

  Content differences:

  --- /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list.orig
  +++ /etc/apt/sources.list.d/wikimedia-opensearch-plugins.list
  @@ -1,2 +1,2 @@
  -deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13
  -deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch13
  +deb http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2
  +deb-src http://apt.wikimedia.org/wikimedia bullseye-wikimedia component/opensearch2

• File[/etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties]

  Content differences:

  --- /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties.orig
  +++ /etc/opensearch/cloudelastic-chi-eqiad/log4j2.properties
  @@ -185,7 +185,6 @@
   rootLogger.appenderRef.ship_to_logstash.ref = ship_to_logstash
   logger.index_indexing_slowlog.appenderRef.ship_to_logstash.ref = ship_to_logstash
   logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
  -logger.index_search_slowlog_rolling.appenderRef.ship_to_logstash.ref = ship_to_logstash
   
   # Define Logstash appender
   appender.ship_to_logstash.name=ship_to_logstash

Relevant files

• Production catalog
• Change catalog
• Production errors/warnings
• Change errors/warnings
• Standard Diff
• Full Diff