Compilation results for logstash2030.codfw.wmnet: System changes detected

You can retrieve this result from host.json.

Catalog differences

Summary

Total Resources:	3348
Resources added:	6
Resources removed:	0
Resources modified:	8
Change percentage:	0.42%

Resources only in the new catalog

Logstash::Input::Kafka[webrequest-dumps-codfw]
File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf]
Logstash::Conf[input-kafka-webrequest-dumps-eqiad]
File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf]
Logstash::Input::Kafka[webrequest-dumps-eqiad]
Logstash::Conf[input-kafka-webrequest-dumps-codfw]

Resources modified

Logstash::Conf[input-kafka-webrequest-dumps-codfw]

Parameters differences:

--- Logstash::Conf[input-kafka-webrequest-dumps-codfw].orig
+++ Logstash::Conf[input-kafka-webrequest-dumps-codfw]

+    priority => 10
+    ensure   => present

Logstash::Input::Kafka[webrequest-dumps-codfw]

Parameters differences:

--- Logstash::Input::Kafka[webrequest-dumps-codfw].orig
+++ Logstash::Input::Kafka[webrequest-dumps-codfw]

+    priority                              => 10
+    ensure                                => present
+    tags                                  => ['input-kafka-webrequest-dumps-codfw', 'kafka', 'es']
+    security_protocol                     => SSL
+    topic                                 => codfw.webrequest.dumps.dev0
+    codec                                 => json
+    ssl_truststore_password               => changeit
+    ssl_truststore_location               => /etc/ssl/localcerts/wmf-java-cacerts
+    consumer_threads                      => 3
+    plugin_id                             => input/kafka/webrequest-dumps-codfw
+    ssl_endpoint_identification_algorithm => 
+    group_id                              => logstash7-codfw
+    kafka_cluster_name                    => logging-codfw
+    type                                  => webrequest-dumps
+    manage_truststore                     => False

Logstash::Conf[input-kafka-webrequest-dumps-eqiad]

Parameters differences:

--- Logstash::Conf[input-kafka-webrequest-dumps-eqiad].orig
+++ Logstash::Conf[input-kafka-webrequest-dumps-eqiad]

+    priority => 10
+    ensure   => present

File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf]

Parameters differences:

--- File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf].orig
+++ File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf]

+    notify       => Service[logstash]
+    ensure       => present
+    owner        => logstash
+    validate_cmd => /usr/local/bin/logstash-config-test %
+    mode         => 0440
+    group        => logstash

Content differences:

--- /etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf.orig
+++ /etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf
@@ -0,0 +1,18 @@
+input {
+  kafka {
+    # Some options have not been added, if you want to add more options, see:
+    # https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html
+    tags                    => ["input-kafka-webrequest-dumps-eqiad", "kafka", "es"]
+    topics                  => ["eqiad.webrequest.dumps.dev0"]
+    group_id                => "logstash7-codfw"
+    type                    => "webrequest-dumps"
+    codec                   => "json"
+    consumer_threads        => 3
+    bootstrap_servers       => "kafka-logging1001.eqiad.wmnet:9093,kafka-logging1002.eqiad.wmnet:9093,kafka-logging1003.eqiad.wmnet:9093,kafka-logging1004.eqiad.wmnet:9093,kafka-logging1005.eqiad.wmnet:9093"
+    id                      => "input/kafka/webrequest-dumps-eqiad"
+    security_protocol       => "SSL"
+    ssl_truststore_location => "/etc/ssl/localcerts/wmf-java-cacerts"
+    ssl_truststore_password => "changeit"
+    ssl_endpoint_identification_algorithm => ""
+  }
+}

Logstash::Input::Kafka[webrequest-dumps-eqiad]

Parameters differences:

--- Logstash::Input::Kafka[webrequest-dumps-eqiad].orig
+++ Logstash::Input::Kafka[webrequest-dumps-eqiad]

+    priority                              => 10
+    ensure                                => present
+    tags                                  => ['input-kafka-webrequest-dumps-eqiad', 'kafka', 'es']
+    security_protocol                     => SSL
+    topic                                 => eqiad.webrequest.dumps.dev0
+    codec                                 => json
+    ssl_truststore_password               => changeit
+    ssl_truststore_location               => /etc/ssl/localcerts/wmf-java-cacerts
+    consumer_threads                      => 3
+    plugin_id                             => input/kafka/webrequest-dumps-eqiad
+    ssl_endpoint_identification_algorithm => 
+    group_id                              => logstash7-codfw
+    kafka_cluster_name                    => logging-eqiad
+    type                                  => webrequest-dumps
+    manage_truststore                     => False

File[/etc/rsyslog.d/30-output-kafka.conf]

Content differences:

--- /etc/rsyslog.d/30-output-kafka.conf.orig
+++ /etc/rsyslog.d/30-output-kafka.conf
@@ -13,6 +13,19 @@
 
 # define a template to be used by omkafka dynatopic
 template(name="kafka_topic" type="string" string="rsyslog-%syslogseverity-text%")
+
+# Event Platform support (T291645): messages that carry a 'meta.stream' field are
+# Event Platform events (e.g. ECS formatted logs). Produce them to the
+# '<datacenter>.<meta.stream>' topic so they can be ingested into the Data Lake.
+# The datacenter prefix is supplied by puppet; the stream suffix is read from the
+# parsed json message via the '%!meta!stream%' property.
+template(name="event_platform_topic" type="string" string="codfw.%!meta!stream%")
+
+# Emit the parsed json message verbatim (with $schema, meta and dt intact) so the
+# event remains valid for Event Platform / Data Lake ingestion.
+template(name="event_platform_json" type="list") {
+  property(name="$!all-json")
+}
 
 # send to kafka if lookup table contains "kafka" for relevant programname
 # $.log_outputs defined by lookup table in lookup_output.conf
@@ -42,6 +55,31 @@
     # unfortunately rsyslog doesn't allow variables to be used as template
     # names, so the kafka action is duplicated here.
     if $parsesuccess == "OK" then {
+        # Event Platform events carry a 'meta.stream' field. Produce these only to
+        # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this
+        # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.
+        if ($!meta!stream != "") then {
+            action(type="omkafka"
+                   name="omkafka_event_platform"
+                   broker=["kafka-logging2001.codfw.wmnet:9093","kafka-logging2002.codfw.wmnet:9093","kafka-logging2003.codfw.wmnet:9093","kafka-logging2004.codfw.wmnet:9093","kafka-logging2005.codfw.wmnet:9093"]
+                   topic="event_platform_topic"
+                   dynatopic="on"
+                   dynatopic.cachesize="1000"
+                   partitions.auto="on"
+                   template="event_platform_json"
+                   queue.type="LinkedList" queue.size="10000" queue.filename="output_kafka_event_platform"
+                   queue.highWatermark="7000" queue.lowWatermark="6000"
+                   queue.checkpointInterval="5"
+                   queue.maxDiskSpace="40960000"
+                   confParam=[ "security.protocol=ssl",
+                               "ssl.ca.location=/etc/ssl/certs/wmf-ca-certificates.crt",
+                               "compression.codec=snappy",
+                               "socket.timeout.ms=10000",
+                               "socket.keepalive.enable=true",
+                               "queue.buffering.max.ms=50",
+                               "batch.num.messages=1000" ]
+            )
+        } else {
         action(type="omkafka"
                name="omkafka_syslog_cee"
                broker=["kafka-logging2001.codfw.wmnet:9093","kafka-logging2002.codfw.wmnet:9093","kafka-logging2003.codfw.wmnet:9093","kafka-logging2004.codfw.wmnet:9093","kafka-logging2005.codfw.wmnet:9093"]
@@ -62,6 +100,7 @@
                            "queue.buffering.max.ms=50",
                            "batch.num.messages=1000" ]
         )
+        }
     } else {
         # if ecs_170 in log_outputs, use that template to format
         # non-json-formatted syslog events into an ecs-compatible form

Rsyslog::Conf[output_kafka]

File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf]

Parameters differences:

--- File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf].orig
+++ File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf]

+    notify       => Service[logstash]
+    ensure       => present
+    owner        => logstash
+    validate_cmd => /usr/local/bin/logstash-config-test %
+    mode         => 0440
+    group        => logstash

Content differences:

--- /etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf.orig
+++ /etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf
@@ -0,0 +1,18 @@
+input {
+  kafka {
+    # Some options have not been added, if you want to add more options, see:
+    # https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html
+    tags                    => ["input-kafka-webrequest-dumps-codfw", "kafka", "es"]
+    topics                  => ["codfw.webrequest.dumps.dev0"]
+    group_id                => "logstash7-codfw"
+    type                    => "webrequest-dumps"
+    codec                   => "json"
+    consumer_threads        => 3
+    bootstrap_servers       => "kafka-logging2001.codfw.wmnet:9093,kafka-logging2002.codfw.wmnet:9093,kafka-logging2003.codfw.wmnet:9093,kafka-logging2004.codfw.wmnet:9093,kafka-logging2005.codfw.wmnet:9093"
+    id                      => "input/kafka/webrequest-dumps-codfw"
+    security_protocol       => "SSL"
+    ssl_truststore_location => "/etc/ssl/localcerts/wmf-java-cacerts"
+    ssl_truststore_password => "changeit"
+    ssl_endpoint_identification_algorithm => ""
+  }
+}

Compilation results for logstash2030.codfw.wmnet: System changes detected

Catalog differences

Summary

Resources only in the new catalog

Resources modified

Relevant files