{"host": "clouddumps1002.wikimedia.org", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3735, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/rsyslog.d/30-output-kafka.conf]", "content": "--- /etc/rsyslog.d/30-output-kafka.conf.orig\n+++ /etc/rsyslog.d/30-output-kafka.conf\n@@ -13,6 +13,19 @@\n \n # define a template to be used by omkafka dynatopic\n template(name=\"kafka_topic\" type=\"string\" string=\"rsyslog-%syslogseverity-text%\")\n+\n+# Event Platform support (T291645): messages that carry a 'meta.stream' field are\n+# Event Platform events (e.g. ECS formatted logs). Produce them to the\n+# '<datacenter>.<meta.stream>' topic so they can be ingested into the Data Lake.\n+# The datacenter prefix is supplied by puppet; the stream suffix is read from the\n+# parsed json message via the '%!meta!stream%' property.\n+template(name=\"event_platform_topic\" type=\"string\" string=\"eqiad.%!meta!stream%\")\n+\n+# Emit the parsed json message verbatim (with $schema, meta and dt intact) so the\n+# event remains valid for Event Platform / Data Lake ingestion.\n+template(name=\"event_platform_json\" type=\"list\") {\n+  property(name=\"$!all-json\")\n+}\n \n # send to kafka if lookup table contains \"kafka\" for relevant programname\n # $.log_outputs defined by lookup table in lookup_output.conf\n@@ -42,6 +55,31 @@\n     # unfortunately rsyslog doesn't allow variables to be used as template\n     # names, so the kafka action is duplicated here.\n     if $parsesuccess == \"OK\" then {\n+        # Event Platform events carry a 'meta.stream' field. Produce these only to\n+        # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this\n+        # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.\n+        if ($!meta!stream != \"\") then {\n+            action(type=\"omkafka\"\n+                   name=\"omkafka_event_platform\"\n+                   broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]\n+                   topic=\"event_platform_topic\"\n+                   dynatopic=\"on\"\n+                   dynatopic.cachesize=\"1000\"\n+                   partitions.auto=\"on\"\n+                   template=\"event_platform_json\"\n+                   queue.type=\"LinkedList\" queue.size=\"10000\" queue.filename=\"output_kafka_event_platform\"\n+                   queue.highWatermark=\"7000\" queue.lowWatermark=\"6000\"\n+                   queue.checkpointInterval=\"5\"\n+                   queue.maxDiskSpace=\"40960000\"\n+                   confParam=[ \"security.protocol=ssl\",\n+                               \"ssl.ca.location=/etc/ssl/certs/wmf-ca-certificates.crt\",\n+                               \"compression.codec=snappy\",\n+                               \"socket.timeout.ms=10000\",\n+                               \"socket.keepalive.enable=true\",\n+                               \"queue.buffering.max.ms=50\",\n+                               \"batch.num.messages=1000\" ]\n+            )\n+        } else {\n         action(type=\"omkafka\"\n                name=\"omkafka_syslog_cee\"\n                broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]\n@@ -62,6 +100,7 @@\n                            \"queue.buffering.max.ms=50\",\n                            \"batch.num.messages=1000\" ]\n         )\n+        }\n     } else {\n         # if ecs_170 in log_outputs, use that template to format\n         # non-json-formatted syslog events into an ecs-compatible form"}, {"resource": "Rsyslog::Conf[output_kafka]"}], "perc_changed": "0.05%"}, "core": {"total": 3735, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/rsyslog.d/30-output-kafka.conf]", "content": "--- /etc/rsyslog.d/30-output-kafka.conf.orig\n+++ /etc/rsyslog.d/30-output-kafka.conf\n@@ -13,6 +13,19 @@\n \n # define a template to be used by omkafka dynatopic\n template(name=\"kafka_topic\" type=\"string\" string=\"rsyslog-%syslogseverity-text%\")\n+\n+# Event Platform support (T291645): messages that carry a 'meta.stream' field are\n+# Event Platform events (e.g. ECS formatted logs). Produce them to the\n+# '<datacenter>.<meta.stream>' topic so they can be ingested into the Data Lake.\n+# The datacenter prefix is supplied by puppet; the stream suffix is read from the\n+# parsed json message via the '%!meta!stream%' property.\n+template(name=\"event_platform_topic\" type=\"string\" string=\"eqiad.%!meta!stream%\")\n+\n+# Emit the parsed json message verbatim (with $schema, meta and dt intact) so the\n+# event remains valid for Event Platform / Data Lake ingestion.\n+template(name=\"event_platform_json\" type=\"list\") {\n+  property(name=\"$!all-json\")\n+}\n \n # send to kafka if lookup table contains \"kafka\" for relevant programname\n # $.log_outputs defined by lookup table in lookup_output.conf\n@@ -42,6 +55,31 @@\n     # unfortunately rsyslog doesn't allow variables to be used as template\n     # names, so the kafka action is duplicated here.\n     if $parsesuccess == \"OK\" then {\n+        # Event Platform events carry a 'meta.stream' field. Produce these only to\n+        # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this\n+        # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.\n+        if ($!meta!stream != \"\") then {\n+            action(type=\"omkafka\"\n+                   name=\"omkafka_event_platform\"\n+                   broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]\n+                   topic=\"event_platform_topic\"\n+                   dynatopic=\"on\"\n+                   dynatopic.cachesize=\"1000\"\n+                   partitions.auto=\"on\"\n+                   template=\"event_platform_json\"\n+                   queue.type=\"LinkedList\" queue.size=\"10000\" queue.filename=\"output_kafka_event_platform\"\n+                   queue.highWatermark=\"7000\" queue.lowWatermark=\"6000\"\n+                   queue.checkpointInterval=\"5\"\n+                   queue.maxDiskSpace=\"40960000\"\n+                   confParam=[ \"security.protocol=ssl\",\n+                               \"ssl.ca.location=/etc/ssl/certs/wmf-ca-certificates.crt\",\n+                               \"compression.codec=snappy\",\n+                               \"socket.timeout.ms=10000\",\n+                               \"socket.keepalive.enable=true\",\n+                               \"queue.buffering.max.ms=50\",\n+                               \"batch.num.messages=1000\" ]\n+            )\n+        } else {\n         action(type=\"omkafka\"\n                name=\"omkafka_syslog_cee\"\n                broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]\n@@ -62,6 +100,7 @@\n                            \"queue.buffering.max.ms=50\",\n                            \"batch.num.messages=1000\" ]\n         )\n+        }\n     } else {\n         # if ecs_170 in log_outputs, use that template to format\n         # non-json-formatted syslog events into an ecs-compatible form"}], "perc_changed": "0.03%"}, "main": {"total": 3735, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/rsyslog.d/30-output-kafka.conf]", "content": "--- /etc/rsyslog.d/30-output-kafka.conf.orig\n+++ /etc/rsyslog.d/30-output-kafka.conf\n@@ -13,6 +13,19 @@\n \n # define a template to be used by omkafka dynatopic\n template(name=\"kafka_topic\" type=\"string\" string=\"rsyslog-%syslogseverity-text%\")\n+\n+# Event Platform support (T291645): messages that carry a 'meta.stream' field are\n+# Event Platform events (e.g. ECS formatted logs). Produce them to the\n+# '<datacenter>.<meta.stream>' topic so they can be ingested into the Data Lake.\n+# The datacenter prefix is supplied by puppet; the stream suffix is read from the\n+# parsed json message via the '%!meta!stream%' property.\n+template(name=\"event_platform_topic\" type=\"string\" string=\"eqiad.%!meta!stream%\")\n+\n+# Emit the parsed json message verbatim (with $schema, meta and dt intact) so the\n+# event remains valid for Event Platform / Data Lake ingestion.\n+template(name=\"event_platform_json\" type=\"list\") {\n+  property(name=\"$!all-json\")\n+}\n \n # send to kafka if lookup table contains \"kafka\" for relevant programname\n # $.log_outputs defined by lookup table in lookup_output.conf\n@@ -42,6 +55,31 @@\n     # unfortunately rsyslog doesn't allow variables to be used as template\n     # names, so the kafka action is duplicated here.\n     if $parsesuccess == \"OK\" then {\n+        # Event Platform events carry a 'meta.stream' field. Produce these only to\n+        # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this\n+        # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.\n+        if ($!meta!stream != \"\") then {\n+            action(type=\"omkafka\"\n+                   name=\"omkafka_event_platform\"\n+                   broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]\n+                   topic=\"event_platform_topic\"\n+                   dynatopic=\"on\"\n+                   dynatopic.cachesize=\"1000\"\n+                   partitions.auto=\"on\"\n+                   template=\"event_platform_json\"\n+                   queue.type=\"LinkedList\" queue.size=\"10000\" queue.filename=\"output_kafka_event_platform\"\n+                   queue.highWatermark=\"7000\" queue.lowWatermark=\"6000\"\n+                   queue.checkpointInterval=\"5\"\n+                   queue.maxDiskSpace=\"40960000\"\n+                   confParam=[ \"security.protocol=ssl\",\n+                               \"ssl.ca.location=/etc/ssl/certs/wmf-ca-certificates.crt\",\n+                               \"compression.codec=snappy\",\n+                               \"socket.timeout.ms=10000\",\n+                               \"socket.keepalive.enable=true\",\n+                               \"queue.buffering.max.ms=50\",\n+                               \"batch.num.messages=1000\" ]\n+            )\n+        } else {\n         action(type=\"omkafka\"\n                name=\"omkafka_syslog_cee\"\n                broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]\n@@ -62,6 +100,7 @@\n                            \"queue.buffering.max.ms=50\",\n                            \"batch.num.messages=1000\" ]\n         )\n+        }\n     } else {\n         # if ecs_170 in log_outputs, use that template to format\n         # non-json-formatted syslog events into an ecs-compatible form"}, {"resource": "Rsyslog::Conf[output_kafka]"}], "perc_changed": "0.05%"}}}