{"host": "logstash2030.codfw.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3348, "only_in_self": [], "only_in_other": ["File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf]", "File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf]", "Logstash::Conf[input-kafka-webrequest-dumps-codfw]", "Logstash::Conf[input-kafka-webrequest-dumps-eqiad]", "Logstash::Input::Kafka[webrequest-dumps-codfw]", "Logstash::Input::Kafka[webrequest-dumps-eqiad]"], "resource_diffs": [{"resource": "Logstash::Conf[input-kafka-webrequest-dumps-eqiad]", "parameters": "--- Logstash::Conf[input-kafka-webrequest-dumps-eqiad].orig\n+++ Logstash::Conf[input-kafka-webrequest-dumps-eqiad]\n\n+    priority => 10\n+    ensure   => present\n"}, {"resource": "File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf]", "content": "--- /etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf.orig\n+++ /etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf\n@@ -0,0 +1,18 @@\n+input {\n+  kafka {\n+    # Some options have not been added, if you want to add more options, see:\n+    # https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html\n+    tags                    => [\"input-kafka-webrequest-dumps-codfw\", \"kafka\", \"es\"]\n+    topics                  => [\"codfw.webrequest.dumps.dev0\"]\n+    group_id                => \"logstash7-codfw\"\n+    type                    => \"webrequest-dumps\"\n+    codec                   => \"json\"\n+    consumer_threads        => 3\n+    bootstrap_servers       => \"kafka-logging2001.codfw.wmnet:9093,kafka-logging2002.codfw.wmnet:9093,kafka-logging2003.codfw.wmnet:9093,kafka-logging2004.codfw.wmnet:9093,kafka-logging2005.codfw.wmnet:9093\"\n+    id                      => \"input/kafka/webrequest-dumps-codfw\"\n+    security_protocol       => \"SSL\"\n+    ssl_truststore_location => \"/etc/ssl/localcerts/wmf-java-cacerts\"\n+    ssl_truststore_password => \"changeit\"\n+    ssl_endpoint_identification_algorithm => \"\"\n+  }\n+}", "parameters": "--- File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf].orig\n+++ File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf]\n\n+    validate_cmd => /usr/local/bin/logstash-config-test %\n+    mode         => 0440\n+    owner        => logstash\n+    notify       => Service[logstash]\n+    ensure       => present\n+    group        => logstash\n"}, {"resource": "Logstash::Input::Kafka[webrequest-dumps-eqiad]", "parameters": "--- Logstash::Input::Kafka[webrequest-dumps-eqiad].orig\n+++ Logstash::Input::Kafka[webrequest-dumps-eqiad]\n\n+    topic                                 => eqiad.webrequest.dumps.dev0\n+    ssl_truststore_password               => changeit\n+    tags                                  => ['input-kafka-webrequest-dumps-eqiad', 'kafka', 'es']\n+    priority                              => 10\n+    type                                  => webrequest-dumps\n+    consumer_threads                      => 3\n+    ensure                                => present\n+    plugin_id                             => input/kafka/webrequest-dumps-eqiad\n+    codec                                 => json\n+    security_protocol                     => SSL\n+    ssl_truststore_location               => /etc/ssl/localcerts/wmf-java-cacerts\n+    kafka_cluster_name                    => logging-eqiad\n+    manage_truststore                     => False\n+    group_id                              => logstash7-codfw\n+    ssl_endpoint_identification_algorithm => \n"}, {"resource": "Logstash::Input::Kafka[webrequest-dumps-codfw]", "parameters": "--- Logstash::Input::Kafka[webrequest-dumps-codfw].orig\n+++ Logstash::Input::Kafka[webrequest-dumps-codfw]\n\n+    topic                                 => codfw.webrequest.dumps.dev0\n+    ssl_truststore_password               => changeit\n+    tags                                  => ['input-kafka-webrequest-dumps-codfw', 'kafka', 'es']\n+    priority                              => 10\n+    type                                  => webrequest-dumps\n+    consumer_threads                      => 3\n+    ensure                                => present\n+    plugin_id                             => input/kafka/webrequest-dumps-codfw\n+    codec                                 => json\n+    security_protocol                     => SSL\n+    ssl_truststore_location               => /etc/ssl/localcerts/wmf-java-cacerts\n+    kafka_cluster_name                    => logging-codfw\n+    manage_truststore                     => False\n+    group_id                              => logstash7-codfw\n+    ssl_endpoint_identification_algorithm => \n"}, {"resource": "Logstash::Conf[input-kafka-webrequest-dumps-codfw]", "parameters": "--- Logstash::Conf[input-kafka-webrequest-dumps-codfw].orig\n+++ Logstash::Conf[input-kafka-webrequest-dumps-codfw]\n\n+    priority => 10\n+    ensure   => present\n"}, {"resource": "Rsyslog::Conf[output_kafka]"}, {"resource": "File[/etc/rsyslog.d/30-output-kafka.conf]", "content": "--- /etc/rsyslog.d/30-output-kafka.conf.orig\n+++ /etc/rsyslog.d/30-output-kafka.conf\n@@ -13,6 +13,19 @@\n \n # define a template to be used by omkafka dynatopic\n template(name=\"kafka_topic\" type=\"string\" string=\"rsyslog-%syslogseverity-text%\")\n+\n+# Event Platform support (T291645): messages that carry a 'meta.stream' field are\n+# Event Platform events (e.g. ECS formatted logs). Produce them to the\n+# '<datacenter>.<meta.stream>' topic so they can be ingested into the Data Lake.\n+# The datacenter prefix is supplied by puppet; the stream suffix is read from the\n+# parsed json message via the '%!meta!stream%' property.\n+template(name=\"event_platform_topic\" type=\"string\" string=\"codfw.%!meta!stream%\")\n+\n+# Emit the parsed json message verbatim (with $schema, meta and dt intact) so the\n+# event remains valid for Event Platform / Data Lake ingestion.\n+template(name=\"event_platform_json\" type=\"list\") {\n+  property(name=\"$!all-json\")\n+}\n \n # send to kafka if lookup table contains \"kafka\" for relevant programname\n # $.log_outputs defined by lookup table in lookup_output.conf\n@@ -42,6 +55,31 @@\n     # unfortunately rsyslog doesn't allow variables to be used as template\n     # names, so the kafka action is duplicated here.\n     if $parsesuccess == \"OK\" then {\n+        # Event Platform events carry a 'meta.stream' field. Produce these only to\n+        # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this\n+        # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.\n+        if ($!meta!stream != \"\") then {\n+            action(type=\"omkafka\"\n+                   name=\"omkafka_event_platform\"\n+                   broker=[\"kafka-logging2001.codfw.wmnet:9093\",\"kafka-logging2002.codfw.wmnet:9093\",\"kafka-logging2003.codfw.wmnet:9093\",\"kafka-logging2004.codfw.wmnet:9093\",\"kafka-logging2005.codfw.wmnet:9093\"]\n+                   topic=\"event_platform_topic\"\n+                   dynatopic=\"on\"\n+                   dynatopic.cachesize=\"1000\"\n+                   partitions.auto=\"on\"\n+                   template=\"event_platform_json\"\n+                   queue.type=\"LinkedList\" queue.size=\"10000\" queue.filename=\"output_kafka_event_platform\"\n+                   queue.highWatermark=\"7000\" queue.lowWatermark=\"6000\"\n+                   queue.checkpointInterval=\"5\"\n+                   queue.maxDiskSpace=\"40960000\"\n+                   confParam=[ \"security.protocol=ssl\",\n+                               \"ssl.ca.location=/etc/ssl/certs/wmf-ca-certificates.crt\",\n+                               \"compression.codec=snappy\",\n+                               \"socket.timeout.ms=10000\",\n+                               \"socket.keepalive.enable=true\",\n+                               \"queue.buffering.max.ms=50\",\n+                               \"batch.num.messages=1000\" ]\n+            )\n+        } else {\n         action(type=\"omkafka\"\n                name=\"omkafka_syslog_cee\"\n                broker=[\"kafka-logging2001.codfw.wmnet:9093\",\"kafka-logging2002.codfw.wmnet:9093\",\"kafka-logging2003.codfw.wmnet:9093\",\"kafka-logging2004.codfw.wmnet:9093\",\"kafka-logging2005.codfw.wmnet:9093\"]\n@@ -62,6 +100,7 @@\n                            \"queue.buffering.max.ms=50\",\n                            \"batch.num.messages=1000\" ]\n         )\n+        }\n     } else {\n         # if ecs_170 in log_outputs, use that template to format\n         # non-json-formatted syslog events into an ecs-compatible form"}, {"resource": "File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf]", "content": "--- /etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf.orig\n+++ /etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf\n@@ -0,0 +1,18 @@\n+input {\n+  kafka {\n+    # Some options have not been added, if you want to add more options, see:\n+    # https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html\n+    tags                    => [\"input-kafka-webrequest-dumps-eqiad\", \"kafka\", \"es\"]\n+    topics                  => [\"eqiad.webrequest.dumps.dev0\"]\n+    group_id                => \"logstash7-codfw\"\n+    type                    => \"webrequest-dumps\"\n+    codec                   => \"json\"\n+    consumer_threads        => 3\n+    bootstrap_servers       => \"kafka-logging1001.eqiad.wmnet:9093,kafka-logging1002.eqiad.wmnet:9093,kafka-logging1003.eqiad.wmnet:9093,kafka-logging1004.eqiad.wmnet:9093,kafka-logging1005.eqiad.wmnet:9093\"\n+    id                      => \"input/kafka/webrequest-dumps-eqiad\"\n+    security_protocol       => \"SSL\"\n+    ssl_truststore_location => \"/etc/ssl/localcerts/wmf-java-cacerts\"\n+    ssl_truststore_password => \"changeit\"\n+    ssl_endpoint_identification_algorithm => \"\"\n+  }\n+}", "parameters": "--- File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf].orig\n+++ File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf]\n\n+    validate_cmd => /usr/local/bin/logstash-config-test %\n+    mode         => 0440\n+    owner        => logstash\n+    notify       => Service[logstash]\n+    ensure       => present\n+    group        => logstash\n"}], "perc_changed": "0.42%"}, "core": {"total": 3348, "only_in_self": [], "only_in_other": ["File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf]", "File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf]"], "resource_diffs": [{"resource": "File[/etc/rsyslog.d/30-output-kafka.conf]", "content": "--- /etc/rsyslog.d/30-output-kafka.conf.orig\n+++ /etc/rsyslog.d/30-output-kafka.conf\n@@ -13,6 +13,19 @@\n \n # define a template to be used by omkafka dynatopic\n template(name=\"kafka_topic\" type=\"string\" string=\"rsyslog-%syslogseverity-text%\")\n+\n+# Event Platform support (T291645): messages that carry a 'meta.stream' field are\n+# Event Platform events (e.g. ECS formatted logs). Produce them to the\n+# '<datacenter>.<meta.stream>' topic so they can be ingested into the Data Lake.\n+# The datacenter prefix is supplied by puppet; the stream suffix is read from the\n+# parsed json message via the '%!meta!stream%' property.\n+template(name=\"event_platform_topic\" type=\"string\" string=\"codfw.%!meta!stream%\")\n+\n+# Emit the parsed json message verbatim (with $schema, meta and dt intact) so the\n+# event remains valid for Event Platform / Data Lake ingestion.\n+template(name=\"event_platform_json\" type=\"list\") {\n+  property(name=\"$!all-json\")\n+}\n \n # send to kafka if lookup table contains \"kafka\" for relevant programname\n # $.log_outputs defined by lookup table in lookup_output.conf\n@@ -42,6 +55,31 @@\n     # unfortunately rsyslog doesn't allow variables to be used as template\n     # names, so the kafka action is duplicated here.\n     if $parsesuccess == \"OK\" then {\n+        # Event Platform events carry a 'meta.stream' field. Produce these only to\n+        # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this\n+        # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.\n+        if ($!meta!stream != \"\") then {\n+            action(type=\"omkafka\"\n+                   name=\"omkafka_event_platform\"\n+                   broker=[\"kafka-logging2001.codfw.wmnet:9093\",\"kafka-logging2002.codfw.wmnet:9093\",\"kafka-logging2003.codfw.wmnet:9093\",\"kafka-logging2004.codfw.wmnet:9093\",\"kafka-logging2005.codfw.wmnet:9093\"]\n+                   topic=\"event_platform_topic\"\n+                   dynatopic=\"on\"\n+                   dynatopic.cachesize=\"1000\"\n+                   partitions.auto=\"on\"\n+                   template=\"event_platform_json\"\n+                   queue.type=\"LinkedList\" queue.size=\"10000\" queue.filename=\"output_kafka_event_platform\"\n+                   queue.highWatermark=\"7000\" queue.lowWatermark=\"6000\"\n+                   queue.checkpointInterval=\"5\"\n+                   queue.maxDiskSpace=\"40960000\"\n+                   confParam=[ \"security.protocol=ssl\",\n+                               \"ssl.ca.location=/etc/ssl/certs/wmf-ca-certificates.crt\",\n+                               \"compression.codec=snappy\",\n+                               \"socket.timeout.ms=10000\",\n+                               \"socket.keepalive.enable=true\",\n+                               \"queue.buffering.max.ms=50\",\n+                               \"batch.num.messages=1000\" ]\n+            )\n+        } else {\n         action(type=\"omkafka\"\n                name=\"omkafka_syslog_cee\"\n                broker=[\"kafka-logging2001.codfw.wmnet:9093\",\"kafka-logging2002.codfw.wmnet:9093\",\"kafka-logging2003.codfw.wmnet:9093\",\"kafka-logging2004.codfw.wmnet:9093\",\"kafka-logging2005.codfw.wmnet:9093\"]\n@@ -62,6 +100,7 @@\n                            \"queue.buffering.max.ms=50\",\n                            \"batch.num.messages=1000\" ]\n         )\n+        }\n     } else {\n         # if ecs_170 in log_outputs, use that template to format\n         # non-json-formatted syslog events into an ecs-compatible form"}], "perc_changed": "0.09%"}, "main": {"total": 3348, "only_in_self": [], "only_in_other": ["File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-codfw.conf]", "File[/etc/logstash/conf.d/10-input-kafka-webrequest-dumps-eqiad.conf]", "Logstash::Conf[input-kafka-webrequest-dumps-codfw]", "Logstash::Conf[input-kafka-webrequest-dumps-eqiad]", "Logstash::Input::Kafka[webrequest-dumps-codfw]", "Logstash::Input::Kafka[webrequest-dumps-eqiad]"], "resource_diffs": [{"resource": "Rsyslog::Conf[output_kafka]"}, {"resource": "File[/etc/rsyslog.d/30-output-kafka.conf]", "content": "--- /etc/rsyslog.d/30-output-kafka.conf.orig\n+++ /etc/rsyslog.d/30-output-kafka.conf\n@@ -13,6 +13,19 @@\n \n # define a template to be used by omkafka dynatopic\n template(name=\"kafka_topic\" type=\"string\" string=\"rsyslog-%syslogseverity-text%\")\n+\n+# Event Platform support (T291645): messages that carry a 'meta.stream' field are\n+# Event Platform events (e.g. ECS formatted logs). Produce them to the\n+# '<datacenter>.<meta.stream>' topic so they can be ingested into the Data Lake.\n+# The datacenter prefix is supplied by puppet; the stream suffix is read from the\n+# parsed json message via the '%!meta!stream%' property.\n+template(name=\"event_platform_topic\" type=\"string\" string=\"codfw.%!meta!stream%\")\n+\n+# Emit the parsed json message verbatim (with $schema, meta and dt intact) so the\n+# event remains valid for Event Platform / Data Lake ingestion.\n+template(name=\"event_platform_json\" type=\"list\") {\n+  property(name=\"$!all-json\")\n+}\n \n # send to kafka if lookup table contains \"kafka\" for relevant programname\n # $.log_outputs defined by lookup table in lookup_output.conf\n@@ -42,6 +55,31 @@\n     # unfortunately rsyslog doesn't allow variables to be used as template\n     # names, so the kafka action is duplicated here.\n     if $parsesuccess == \"OK\" then {\n+        # Event Platform events carry a 'meta.stream' field. Produce these only to\n+        # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this\n+        # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.\n+        if ($!meta!stream != \"\") then {\n+            action(type=\"omkafka\"\n+                   name=\"omkafka_event_platform\"\n+                   broker=[\"kafka-logging2001.codfw.wmnet:9093\",\"kafka-logging2002.codfw.wmnet:9093\",\"kafka-logging2003.codfw.wmnet:9093\",\"kafka-logging2004.codfw.wmnet:9093\",\"kafka-logging2005.codfw.wmnet:9093\"]\n+                   topic=\"event_platform_topic\"\n+                   dynatopic=\"on\"\n+                   dynatopic.cachesize=\"1000\"\n+                   partitions.auto=\"on\"\n+                   template=\"event_platform_json\"\n+                   queue.type=\"LinkedList\" queue.size=\"10000\" queue.filename=\"output_kafka_event_platform\"\n+                   queue.highWatermark=\"7000\" queue.lowWatermark=\"6000\"\n+                   queue.checkpointInterval=\"5\"\n+                   queue.maxDiskSpace=\"40960000\"\n+                   confParam=[ \"security.protocol=ssl\",\n+                               \"ssl.ca.location=/etc/ssl/certs/wmf-ca-certificates.crt\",\n+                               \"compression.codec=snappy\",\n+                               \"socket.timeout.ms=10000\",\n+                               \"socket.keepalive.enable=true\",\n+                               \"queue.buffering.max.ms=50\",\n+                               \"batch.num.messages=1000\" ]\n+            )\n+        } else {\n         action(type=\"omkafka\"\n                name=\"omkafka_syslog_cee\"\n                broker=[\"kafka-logging2001.codfw.wmnet:9093\",\"kafka-logging2002.codfw.wmnet:9093\",\"kafka-logging2003.codfw.wmnet:9093\",\"kafka-logging2004.codfw.wmnet:9093\",\"kafka-logging2005.codfw.wmnet:9093\"]\n@@ -62,6 +100,7 @@\n                            \"queue.buffering.max.ms=50\",\n                            \"batch.num.messages=1000\" ]\n         )\n+        }\n     } else {\n         # if ecs_170 in log_outputs, use that template to format\n         # non-json-formatted syslog events into an ecs-compatible form"}], "perc_changed": "0.24%"}}}