{"host": "clouddumps1002.wikimedia.org", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 3735, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/rsyslog.d/30-output-kafka.conf]", "content": "--- /etc/rsyslog.d/30-output-kafka.conf.orig\n+++ /etc/rsyslog.d/30-output-kafka.conf\n@@ -58,7 +58,10 @@\n         # Event Platform events carry a 'meta.stream' field. Produce these only to\n         # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this\n         # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.\n+        # We also need to unset the $!msg field here, which is left over from the\n+        # failed mmjsonparse_kafka_raw parsing attempt.\n         if ($!meta!stream != \"\") then {\n+            unset $!msg;\n             action(type=\"omkafka\"\n                    name=\"omkafka_event_platform\"\n                    broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]"}, {"resource": "Nginx::Site[xmldumps]"}, {"resource": "File[/etc/nginx/sites-available/xmldumps]", "content": "--- /etc/nginx/sites-available/xmldumps.orig\n+++ /etc/nginx/sites-available/xmldumps\n@@ -97,8 +97,10 @@\n    # The 'dumps_http' tag wil match with an entry in /etc/rsyslog.lookup.d/lookup_table_output.json\n    # so these logs will be sent to the kafka-logging cluster. nginx only permits\n    # alphanumeric characters and underscores in the syslog tag, so we cannot use a hyphen.\n+   # We also require the 'nohostname' option, otherwise the hostname is prepended to the message and\n+   # this is where rsyslog expects to find the tag, or programname.\n    access_log /var/log/nginx/access.log; # duplicate of the standard configuration in the http block.\n-   access_log syslog:server=unix:/dev/log,facility=local7,tag=dumps_http,severity=info ecs_json;\n+   access_log syslog:server=unix:/dev/log,nohostname,facility=local7,tag=dumps_http,severity=info ecs_json;\n \n    add_header Server 'clouddumps1002.wikimedia.org' always;\n "}, {"resource": "Rsyslog::Conf[output_kafka]"}], "perc_changed": "0.11%"}, "core": {"total": 3735, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/rsyslog.d/30-output-kafka.conf]", "content": "--- /etc/rsyslog.d/30-output-kafka.conf.orig\n+++ /etc/rsyslog.d/30-output-kafka.conf\n@@ -58,7 +58,10 @@\n         # Event Platform events carry a 'meta.stream' field. Produce these only to\n         # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this\n         # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.\n+        # We also need to unset the $!msg field here, which is left over from the\n+        # failed mmjsonparse_kafka_raw parsing attempt.\n         if ($!meta!stream != \"\") then {\n+            unset $!msg;\n             action(type=\"omkafka\"\n                    name=\"omkafka_event_platform\"\n                    broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]"}, {"resource": "File[/etc/nginx/sites-available/xmldumps]", "content": "--- /etc/nginx/sites-available/xmldumps.orig\n+++ /etc/nginx/sites-available/xmldumps\n@@ -97,8 +97,10 @@\n    # The 'dumps_http' tag wil match with an entry in /etc/rsyslog.lookup.d/lookup_table_output.json\n    # so these logs will be sent to the kafka-logging cluster. nginx only permits\n    # alphanumeric characters and underscores in the syslog tag, so we cannot use a hyphen.\n+   # We also require the 'nohostname' option, otherwise the hostname is prepended to the message and\n+   # this is where rsyslog expects to find the tag, or programname.\n    access_log /var/log/nginx/access.log; # duplicate of the standard configuration in the http block.\n-   access_log syslog:server=unix:/dev/log,facility=local7,tag=dumps_http,severity=info ecs_json;\n+   access_log syslog:server=unix:/dev/log,nohostname,facility=local7,tag=dumps_http,severity=info ecs_json;\n \n    add_header Server 'clouddumps1002.wikimedia.org' always;\n "}], "perc_changed": "0.05%"}, "main": {"total": 3735, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/rsyslog.d/30-output-kafka.conf]", "content": "--- /etc/rsyslog.d/30-output-kafka.conf.orig\n+++ /etc/rsyslog.d/30-output-kafka.conf\n@@ -58,7 +58,10 @@\n         # Event Platform events carry a 'meta.stream' field. Produce these only to\n         # the '<datacenter>.<meta.stream>' topic (T291645). Logstash consumes this\n         # topic via an explicit kafka input rather than the 'rsyslog-*' pattern.\n+        # We also need to unset the $!msg field here, which is left over from the\n+        # failed mmjsonparse_kafka_raw parsing attempt.\n         if ($!meta!stream != \"\") then {\n+            unset $!msg;\n             action(type=\"omkafka\"\n                    name=\"omkafka_event_platform\"\n                    broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]"}, {"resource": "Nginx::Site[xmldumps]"}, {"resource": "File[/etc/nginx/sites-available/xmldumps]", "content": "--- /etc/nginx/sites-available/xmldumps.orig\n+++ /etc/nginx/sites-available/xmldumps\n@@ -97,8 +97,10 @@\n    # The 'dumps_http' tag wil match with an entry in /etc/rsyslog.lookup.d/lookup_table_output.json\n    # so these logs will be sent to the kafka-logging cluster. nginx only permits\n    # alphanumeric characters and underscores in the syslog tag, so we cannot use a hyphen.\n+   # We also require the 'nohostname' option, otherwise the hostname is prepended to the message and\n+   # this is where rsyslog expects to find the tag, or programname.\n    access_log /var/log/nginx/access.log; # duplicate of the standard configuration in the http block.\n-   access_log syslog:server=unix:/dev/log,facility=local7,tag=dumps_http,severity=info ecs_json;\n+   access_log syslog:server=unix:/dev/log,nohostname,facility=local7,tag=dumps_http,severity=info ecs_json;\n \n    add_header Server 'clouddumps1002.wikimedia.org' always;\n "}, {"resource": "Rsyslog::Conf[output_kafka]"}], "perc_changed": "0.11%"}}}