{"host": "ml-serve1015.eqiad.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 2865, "only_in_self": ["Class[Role::Ml_k8s::Insetup_gpu]", "File[/etc/update-motd.d/05-ml-k8s--insetup-gpu]", "Motd::Message[ml_k8s::insetup_gpu]", "Motd::Script[ml_k8s::insetup_gpu]", "Node[__node_regexp__ml-serve10145.eqiad.]"], "only_in_other": ["Apt::Package_from_bpo[firmware-amd-graphics-trixie-bpo]", "Apt::Package_from_bpo[linux-6.16-trixie]", "Apt::Package_from_component[calico329]", "Apt::Package_from_component[istio115]", "Apt::Package_from_component[kubernetes131]", "Apt::Pin[apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo]", "Apt::Pin[apt_pin_linux-6.16-trixie_trixie-bpo]", "Apt::Repository[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Apt::Repository[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Apt::Repository[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Cfssl::Cert[discovery__ml-serve1015_eqiad_wmnet]", "Cfssl::Cert[mlserve__amdgpu-node-labeller]", "Cfssl::Cert[mlserve__calico-cni]", "Cfssl::Cert[mlserve__calicoctl]", "Cfssl::Cert[mlserve__istio-cni]", "Cfssl::Cert[mlserve__kubelet_server]", "Cfssl::Cert[mlserve__rsyslog]", "Cfssl::Cert[mlserve__system_kube-proxy]", "Cfssl::Cert[mlserve__system_node_ml-serve1015_eqiad_wmnet]", "Cfssl::Csr[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__calico-cni.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__calicoctl.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__istio-cni.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__kubelet_server.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__rsyslog.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "Class[Apparmor]", "Class[Base::Sysctl::Inotify]", "Class[Calico]", "Class[Containerd::Configuration]", "Class[Containerd::Nerdctl]", "Class[Containerd]", "Class[Cpufrequtils]", "Class[Dragonfly::Dfdaemon]", "Class[K8s::Base_dirs]", "Class[K8s::Clusters]", "Class[K8s::Kubelet::Cni::Base]", "Class[K8s::Kubelet]", "Class[K8s::Proxy]", "Class[Lvs::Realserver]", "Class[Profile::Calico::Kubernetes]", "Class[Profile::Containerd]", "Class[Profile::Dragonfly::Dfdaemon]", "Class[Profile::Kubernetes::Container_runtime]", "Class[Profile::Kubernetes::Node]", "Class[Profile::Lvs::Configuration]", "Class[Profile::Lvs::Realserver]", "Class[Profile::Rsyslog::Kubernetes]", "Class[Profile::Rsyslog::Shellbox]", "Class[Role::Ml_k8s::Worker]", "Class[Toil::Rsyslog_imfile_remedy]", "Class[Wmflib::Service::Catalog]", "Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_file[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_file[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_file[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Docker::Credentials[/var/lib/kubelet/config.json]", "Exec[/sbin/modprobe overlay]", "Exec[/usr/sbin/dpkg-reconfigure -p critical -f noninteractive wikimedia-lvs-realserver]", "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh on intermediate ca change]", "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh]", "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet]", "Exec[Generate cert mlserve__amdgpu-node-labeller refresh on intermediate ca change]", "Exec[Generate cert mlserve__amdgpu-node-labeller refresh]", "Exec[Generate cert mlserve__amdgpu-node-labeller]", "Exec[Generate cert mlserve__calico-cni refresh on intermediate ca change]", "Exec[Generate cert mlserve__calico-cni refresh]", "Exec[Generate cert mlserve__calico-cni]", "Exec[Generate cert mlserve__calicoctl refresh on intermediate ca change]", "Exec[Generate cert mlserve__calicoctl refresh]", "Exec[Generate cert mlserve__calicoctl]", "Exec[Generate cert mlserve__istio-cni refresh on intermediate ca change]", "Exec[Generate cert mlserve__istio-cni refresh]", "Exec[Generate cert mlserve__istio-cni]", "Exec[Generate cert mlserve__kubelet_server refresh on intermediate ca change]", "Exec[Generate cert mlserve__kubelet_server refresh]", "Exec[Generate cert mlserve__kubelet_server]", "Exec[Generate cert mlserve__rsyslog refresh on intermediate ca change]", "Exec[Generate cert mlserve__rsyslog refresh]", "Exec[Generate cert mlserve__rsyslog]", "Exec[Generate cert mlserve__system_kube-proxy refresh on intermediate ca change]", "Exec[Generate cert mlserve__system_kube-proxy refresh]", "Exec[Generate cert mlserve__system_kube-proxy]", "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh on intermediate ca change]", "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh]", "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet]", "Exec[apt_package_from_component_calico329]", "Exec[apt_package_from_component_istio115]", "Exec[apt_package_from_component_kubernetes131]", "Exec[apt_pin_apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo]", "Exec[apt_pin_apt_pin_linux-6.16-trixie_trixie-bpo]", "Exec[apt_repository_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[apt_repository_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[apt_repository_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[cpupower_reload]", "Exec[create chained cert /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]", "Exec[create chained cert /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__calico-cni.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__calicoctl.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__istio-cni.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]", "Exec[exec-apt-get-update-firmware-amd-graphics-trixie-bpo_trixie-bpo]", "Exec[exec-apt-get-update-linux-6.16-trixie_trixie-bpo]", "Exec[renew certificate - discovery__ml-serve1015_eqiad_wmnet]", "Exec[renew certificate - mlserve__amdgpu-node-labeller]", "Exec[renew certificate - mlserve__calico-cni]", "Exec[renew certificate - mlserve__calicoctl]", "Exec[renew certificate - mlserve__istio-cni]", "Exec[renew certificate - mlserve__kubelet_server]", "Exec[renew certificate - mlserve__rsyslog]", "Exec[renew certificate - mlserve__system_kube-proxy]", "Exec[renew certificate - mlserve__system_node_ml-serve1015_eqiad_wmnet]", "Exec[systemd daemon-reload for amd-k8s-node-labeller.service (amd-k8s-node-labeller-amd-devplugin-after-labeller)]", "Exec[systemd daemon-reload for cpupower.service (cpupower)]", "Exec[systemd daemon-reload for ferm.service (ferm-ferm-service-auto-restart)]", "Exec[systemd daemon-reload for kube-proxy.service (kube-proxy)]", "Exec[systemd daemon-reload for kubelet.service (kubelet-container-runtime)]", "Exec[systemd daemon-reload for rsyslog-imfile-remedy.service (rsyslog-imfile-remedy.service)]", "Exec[systemd daemon-reload for rsyslog-imfile-remedy.timer (rsyslog-imfile-remedy.timer)]", "Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.service (rsyslog-release-deleted-inotify-watches.service)]", "Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.timer (rsyslog-release-deleted-inotify-watches.timer)]", "Ferm::Service[calico-bird]", "Ferm::Service[calico_typha]", "Ferm::Service[dragonfly_dfget]", "Ferm::Service[kubelet-http]", "File[/etc/amd/node-labeller-kubeconfig]", "File[/etc/amd]", "File[/etc/apparmor.d/abstractions]", "File[/etc/apt/preferences.d/apt_pin_firmware_amd_graphics_trixie_bpo_trixie_bpo.pref]", "File[/etc/apt/preferences.d/apt_pin_linux_6_16_trixie_trixie_bpo.pref]", "File[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "File[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "File[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "File[/etc/calico/calicoctl-kubeconfig]", "File[/etc/calico/calicoctl.cfg]", "File[/etc/calico/pki]", "File[/etc/calico]", "File[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]", "File[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]", "File[/etc/cfssl/csr/mlserve__calico-cni.csr]", "File[/etc/cfssl/csr/mlserve__calicoctl.csr]", "File[/etc/cfssl/csr/mlserve__istio-cni.csr]", "File[/etc/cfssl/csr/mlserve__kubelet_server.csr]", "File[/etc/cfssl/csr/mlserve__rsyslog.csr]", "File[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]", "File[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog-key.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chained.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.csr]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog]", "File[/etc/cni/net.d/10-calico.conflist]", "File[/etc/cni/net.d/calico-kubeconfig]", "File[/etc/cni/net.d/istio-kubeconfig]", "File[/etc/cni/net.d]", "File[/etc/cni]", "File[/etc/containerd/config.toml]", "File[/etc/containerd]", "File[/etc/default/cpupower]", "File[/etc/default/kube-proxy]", "File[/etc/default/kubelet]", "File[/etc/default/wikimedia-lvs-realserver]", "File[/etc/dragonfly/dfdaemon.yml]", "File[/etc/dragonfly/dfget.yml]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.csr]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem]", "File[/etc/dragonfly]", "File[/etc/ferm/conf.d/10_calico-bird]", "File[/etc/ferm/conf.d/10_calico_typha]", "File[/etc/ferm/conf.d/10_dragonfly_dfget]", "File[/etc/ferm/conf.d/10_kubelet-http]", "File[/etc/kubernetes/kube-proxy-config.yaml]", "File[/etc/kubernetes/kubelet-config.yaml]", "File[/etc/kubernetes/kubelet.conf]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller-key.pem]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chained.pem]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.csr]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni-key.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni.chain.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni.chained.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni.csr]", "File[/etc/kubernetes/pki/mlserve__calico-cni.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl-key.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl.chain.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl.chained.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl.csr]", "File[/etc/kubernetes/pki/mlserve__calicoctl.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni-key.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni.chain.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni.chained.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni.csr]", "File[/etc/kubernetes/pki/mlserve__istio-cni.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server-key.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.chained.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.csr]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy-key.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chained.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.csr]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet-key.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chained.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem]", "File[/etc/kubernetes/pki]", "File[/etc/kubernetes/proxy.conf]", "File[/etc/kubernetes]", "File[/etc/logrotate.d/rsyslog-release-deleted-inotify-watches]", "File[/etc/modules-load.d/overlay.conf]", "File[/etc/nerdctl/nerdctl.toml]", "File[/etc/nerdctl]", "File[/etc/rsyslog.d/00-imfile.conf]", "File[/etc/rsyslog.d/08-input-file-kubernetes-json.conf]", "File[/etc/rsyslog.d/09-kubernetes.conf]", "File[/etc/rsyslog.d/10-kubernetes-node-filters.conf]", "File[/etc/rsyslog.d/20-shellbox.conf]", "File[/etc/rsyslog.d/35-output-kafka-k8s.conf]", "File[/etc/rsyslog.d/40-rsyslog-release-deleted-inotify-watches.conf]", "File[/etc/sysctl.d/70-increase_inotify_limits.conf]", "File[/etc/sysctl.d/70-ipv6-fowarding-accept-ra.conf]", "File[/etc/sysctl.d/75-kube_proxy_conntrack.conf]", "File[/etc/sysctl.d/75-kube_proxy_icmp.conf]", "File[/etc/systemd/system/amd-k8s-node-labeller.service.d/amd-devplugin-after-labeller.conf]", "File[/etc/systemd/system/amd-k8s-node-labeller.service.d]", "File[/etc/systemd/system/ferm.service.d/ferm-service-auto-restart.conf]", "File[/etc/systemd/system/kube-proxy.service.d/puppet-override.conf]", "File[/etc/systemd/system/kube-proxy.service.d]", "File[/etc/systemd/system/kubelet.service.d/container-runtime.conf]", "File[/etc/systemd/system/kubelet.service.d]", "File[/etc/udev/rules.d/70-kfd.rules]", "File[/etc/udev/rules.d/70-render.rules]", "File[/etc/udev/rules.d/75-kube_proxy_conntrack.rules]", "File[/etc/update-motd.d/05-ml-k8s--worker]", "File[/lib/systemd/system/cpupower.service]", "File[/lib/systemd/system/rsyslog-imfile-remedy.service]", "File[/lib/systemd/system/rsyslog-imfile-remedy.timer]", "File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.service]", "File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.timer]", "File[/usr/libexec/cpupower]", "File[/usr/local/sbin/rsyslog-release-deleted-inotify-watches]", "File[/var/lib/kubelet/config.json]", "File[/var/lib/kubelet]", "File[/var/log/rsyslog-release-deleted-inotify-watches]", "File[/var/run/kubernetes]", "Firewall::Service[calico-typha]", "Firewall::Service[dragonfly_dfget]", "Group[kube]", "K8s::Kubeconfig[/etc/amd/node-labeller-kubeconfig]", "K8s::Kubeconfig[/etc/calico/calicoctl-kubeconfig]", "K8s::Kubeconfig[/etc/cni/net.d/calico-kubeconfig]", "K8s::Kubeconfig[/etc/cni/net.d/istio-kubeconfig]", "K8s::Kubeconfig[/etc/kubernetes/kubelet.conf]", "K8s::Kubeconfig[/etc/kubernetes/proxy.conf]", "K8s::Kubelet::Cni[calico]", "K8s::Package[kubelet]", "K8s::Package[proxy]", "Kmod::Module[overlay]", "Logrotate::Conf[rsyslog-release-deleted-inotify-watches]", "Motd::Message[ml_k8s::worker]", "Motd::Script[ml_k8s::worker]", "Node[__node_regexp__ml-serve1001-91012345.eqiad.]", "Package[amd-k8s-device-plugin]", "Package[amd-k8s-node-labeller]", "Package[apparmor]", "Package[calico-cni]", "Package[calicoctl]", "Package[containerd]", "Package[crictl]", "Package[dragonfly-dfdaemon]", "Package[dragonfly-dfget]", "Package[istio-cni]", "Package[kubernetes-node]", "Package[linux-cpupower]", "Package[linux-image-6.16.3+deb13-amd64]", "Package[nerdctl]", "Package[rsyslog-kubernetes]", "Package[socat]", "Package[wikimedia-lvs-realserver]", "Rsyslog::Conf[imfile]", "Rsyslog::Conf[input-file-kubernetes-json]", "Rsyslog::Conf[kubernetes-node-filters]", "Rsyslog::Conf[kubernetes]", "Rsyslog::Conf[output_kafka_k8s]", "Rsyslog::Conf[rsyslog-release-deleted-inotify-watches]", "Rsyslog::Conf[shellbox]", "Rsyslog::Input::File[kubernetes-json]", "Service[apparmor]", "Service[containerd]", "Service[cpupower]", "Service[dragonfly-dfdaemon]", "Service[kube-proxy]", "Service[kubelet]", "Service[rsyslog-imfile-remedy.timer]", "Service[rsyslog-release-deleted-inotify-watches.timer]", "Sysctl::Conffile[increase_inotify_limits]", "Sysctl::Conffile[ipv6-fowarding-accept-ra]", "Sysctl::Conffile[kube_proxy_conntrack]", "Sysctl::Conffile[kube_proxy_icmp]", "Sysctl::Parameters[increase_inotify_limits]", "Sysctl::Parameters[ipv6-fowarding-accept-ra]", "Sysctl::Parameters[kube_proxy_conntrack]", "Sysctl::Parameters[kube_proxy_icmp]", "Systemd::Override[amd-devplugin-after-labeller]", "Systemd::Override[container-runtime]", "Systemd::Override[ferm-service-auto-restart]", "Systemd::Service[cpupower]", "Systemd::Service[kube-proxy]", "Systemd::Service[rsyslog-imfile-remedy]", "Systemd::Service[rsyslog-release-deleted-inotify-watches]", "Systemd::Syslog[rsyslog-release-deleted-inotify-watches]", "Systemd::Timer::Job[rsyslog-imfile-remedy]", "Systemd::Timer::Job[rsyslog-release-deleted-inotify-watches]", "Systemd::Timer[rsyslog-imfile-remedy]", "Systemd::Timer[rsyslog-release-deleted-inotify-watches]", "Systemd::Unit[amd-k8s-node-labeller-amd-devplugin-after-labeller]", "Systemd::Unit[cpupower]", "Systemd::Unit[ferm-ferm-service-auto-restart]", "Systemd::Unit[kube-proxy]", "Systemd::Unit[kubelet-container-runtime]", "Systemd::Unit[rsyslog-imfile-remedy.service]", "Systemd::Unit[rsyslog-imfile-remedy.timer]", "Systemd::Unit[rsyslog-release-deleted-inotify-watches.service]", "Systemd::Unit[rsyslog-release-deleted-inotify-watches.timer]", "Udev::Rule[kube_proxy_conntrack]", "User[kube]"], "resource_diffs": [{"resource": "Apt::Repository[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "parameters": "--- Apt::Repository[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Apt::Repository[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    trust_repo               => False\n+    components               => component/calico329\n+    dist                     => trixie-wikimedia\n+    keyfile                  => puppet:///modules/install_server/autoinstall/keyring/wikimedia-archive-keyring.gpg\n+    uri                      => http://apt.wikimedia.org/wikimedia\n+    allow_releaseinfo_change => False\n+    bin                      => True\n+    source                   => True\n+    ensure                   => present\n"}, {"resource": "Exec[create chained cert /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]", "parameters": "--- Exec[create chained cert /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem].orig\n+++ Exec[create chained cert /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]\n\n+    command   => /bin/cat /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem > /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chained.pem\n+    unless    => /usr/bin/test \"$(/bin/cat /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem | sha512sum)\" == \"$(/bin/cat /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chained.pem | sha512sum)\"\n\n+    subscribe => ['Exec[renew certificate - mlserve__amdgpu-node-labeller]', 'File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]', 'File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem]']\n+    require   => Exec[Generate cert mlserve__amdgpu-node-labeller refresh on intermediate ca change]\n"}, {"resource": "File[/etc/dragonfly/dfdaemon.yml]", "content": "--- /etc/dragonfly/dfdaemon.yml.orig\n+++ /etc/dragonfly/dfdaemon.yml\n@@ -0,0 +1,49 @@\n+# This is only used if dfdaemon is configured in registry-mirrors of\n+# /etc/docker/daemon.json.\n+# It is not used in our configuration, but I'll keep it here as the default is\n+# the upstream docker hub and that looks very phishy in logs.\n+registry_mirror:\n+  remote: \"https://docker-registry.discovery.wmnet\"\n+\n+# dfdaemon will listen on TCP 65001 by default for connections to be proxied to the registry.\n+# If certpem and keypem is set, dfdaemon will only accept HTTPS connections.\n+# Prometheus metrics are served as well using this port, so we need to bind to 0.0.0.0.\n+hostIp: 0.0.0.0\n+port: 65001\n+# We can't use HTTP even though we're localhost only as docker will refuse to send credentials\n+# for a registry via HTTPS. So this needs to be a certificate for: 127.0.0.1, ::1, localhost and $(hostname -f) (for scraping metrics).\n+certpem: \"/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem\"\n+keypem: \"/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem\"\n+\n+# Requests will be handled by the first matching rule\n+# \"use_https: true\" can be used to upgrade incoming HTTP requests to HTTPS (this means connections\n+# from dfdaemon to the source registry will always use HTTPS).\n+proxies:\n+  - regx: \"wikimedia/machinelearning-liftwing.*/blobs/sha256.*\"\n+    use_https: true\n+  - regx: \"amd-pytorch.*/blobs/sha256.*\"\n+    use_https: true\n+\n+# If an https request's host matches any of the hijacking rules, dfdaemon will\n+# decrypt the request with given key pair and proxy it with the proxy rules.\n+hijack_https:\n+  # Cert and key of docker-registry.discovery.wmnet\n+  cert: \"/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem\"\n+  key: \"/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem\"\n+  hosts:\n+    - regx: \"docker-registry.discovery.wmnet\"\n+      # Puppet-ca here, for validation of the cert from source\n+      certs: [\"/etc/ssl/certs/wmf-ca-certificates.crt\"]\n+\n+# Configure dfget to use a specific home directory (will be created if it does not exist).\n+#\n+# The IP used here is not only a listen IP but the IP annonced to the supernode. Using 0.0.0.0\n+# leads to the network still functioning but every client will ask it's own dfget server for parts\n+# which will make them fetch the parts from the docker-registry istead of peers.\n+#\n+# Port needs to be specified here as dfget will choose a random one (per invocation) if not.\n+dfget_flags: [\"--home\", \"/var/lib/dragonfly-dfdaemon/dfget\", \"--ip\", \"10.64.167.6\", \"--port\", \"15001\"]\n+\n+# Network bandwith rate limit, will actually be used as \"--locallimit\" and \"--totallimit\" for dfget.\n+# In format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte.\n+ratelimit: 100M", "parameters": "--- File[/etc/dragonfly/dfdaemon.yml].orig\n+++ File[/etc/dragonfly/dfdaemon.yml]\n\n+    group  => root\n+    mode   => 0644\n+    ensure => file\n+    notify => Service[dragonfly-dfdaemon]\n+    owner  => root\n"}, {"resource": "Package[linux-image-6.16.3+deb13-amd64]", "parameters": "--- Package[linux-image-6.16.3+deb13-amd64].orig\n+++ Package[linux-image-6.16.3+deb13-amd64]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__kubelet_server.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__kubelet_server.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__kubelet_server.pem]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => kube\n"}, {"resource": "Class[Profile::Monitoring]", "parameters": "--- Class[Profile::Monitoring].orig\n+++ Class[Profile::Monitoring]\n\n@@\n-    nrpe_check_disk_options => -w 6% -c 3% -W 6% -K 3% -l -e -A -i \"/srv/sd[a-b][1-3]\" -i \"/srv/nvme[0-9]n[0-9]p[0-9]\" --exclude-type=fuse --exclude-type=fuse.fuse_dfs --exclude-type=tracefs\n+    nrpe_check_disk_options => -w 10% -c 5% -W 6% -K 3% -l -e -A -i '/(var/lib|run)/(containerd|kubelet)/*' --exclude-type=tracefs\n"}, {"resource": "Exec[renew certificate - mlserve__kubelet_server]", "parameters": "--- Exec[renew certificate - mlserve__kubelet_server].orig\n+++ Exec[renew certificate - mlserve__kubelet_server]\n\n+    command     => /usr/bin/cfssl sign -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve -profile server /etc/kubernetes/pki/mlserve__kubelet_server.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__kubelet_server\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Exec[Generate cert mlserve__kubelet_server]\n+    unless      => /usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__kubelet_server.pem -checkend 952200\n+    notify      => ['Service[kubelet]']\n"}, {"resource": "File[/etc/default/cpupower]", "content": "--- /etc/default/cpupower.orig\n+++ /etc/default/cpupower\n@@ -0,0 +1 @@\n+GOVERNOR=performance", "parameters": "--- File[/etc/default/cpupower].orig\n+++ File[/etc/default/cpupower]\n\n+    owner   => root\n+    require => Package[linux-cpupower]\n+    group   => root\n"}, {"resource": "Service[cpupower]", "parameters": "--- Service[cpupower].orig\n+++ Service[cpupower]\n\n+    enable => True\n+    ensure => running\n"}, {"resource": "Logrotate::Conf[rsyslog-release-deleted-inotify-watches]", "parameters": "--- Logrotate::Conf[rsyslog-release-deleted-inotify-watches].orig\n+++ Logrotate::Conf[rsyslog-release-deleted-inotify-watches]\n\n+    ensure => absent\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_disk_space.cfg]", "content": "--- /etc/nagios/nrpe.d/check_disk_space.cfg.orig\n+++ /etc/nagios/nrpe.d/check_disk_space.cfg\n@@ -1,2 +1,2 @@\n # File generated by puppet. DO NOT edit by hand\n-command[check_disk_space]=/usr/lib/nagios/plugins/check_disk -w 6% -c 3% -W 6% -K 3% -l -e -A -i \"/srv/sd[a-b][1-3]\" -i \"/srv/nvme[0-9]n[0-9]p[0-9]\" --exclude-type=fuse --exclude-type=fuse.fuse_dfs --exclude-type=tracefs\n+command[check_disk_space]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -W 6% -K 3% -l -e -A -i '/(var/lib|run)/(containerd|kubelet)/*' --exclude-type=tracefs"}, {"resource": "File[/etc/containerd/config.toml]", "content": "--- /etc/containerd/config.toml.orig\n+++ /etc/containerd/config.toml\n@@ -0,0 +1,47 @@\n+# SPDX-License-Identifier: Apache-2.0\n+# This is based on the config shipped with the containerd package in Debian (1.6.20~ds1-1+b1)\n+#\n+# All possible config values including their defaults can be found by running:\n+# containerd config default\n+version = 2\n+\n+[plugins]\n+  [plugins.\"io.containerd.grpc.v1.cri\"]\n+    # Define our sandbox image\n+    sandbox_image = \"docker-registry.discovery.wmnet/pause:3.6-1\"\n+    # max_container_log_line_size is the maximum log line size in bytes for a container.\n+    # Log line longer than the limit will be split into multiple lines. -1 means no\n+    # limit.\n+    max_container_log_line_size = -1\n+    # By default docker does set net.ipv4.ip_unprivileged_port_start=0 allowing containers to bind to ports\n+    # below 1024 without explicit NET_BIND_SERVICE capability.\n+    # It also sets net.ipv4.ping_group_range=\"0 2147483647\", allowing ICMP sockets without CAP_NET_RAW.\n+    # The following two options ensure compatibility with current workloads.\n+    #\n+    # enable_unprivileged_ports configures net.ipv4.ip_unprivileged_port_start=0\n+    # for all containers which are not using host network and if it is not overwritten by PodSandboxConfig\n+    # Note that currently default is set to disabled but target change it in future, see:\n+    # https://github.com/kubernetes/kubernetes/issues/102612\n+    enable_unprivileged_ports = true\n+    # enable_unprivileged_icmp configures net.ipv4.ping_group_range=\"0 2147483647\"\n+    # for all containers which are not using host network, are not running in user namespace and if it is not\n+    # overwritten by PodSandboxConfig.\n+    # Note that currently default is set to disabled but target change it in future together with enable_unprivileged_ports\n+    enable_unprivileged_icmp = true\n+    [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n+      # Re-define the runtime type as defining runc.options would shadow the default setting.\n+      # Without this kubelet will fail to run containers with the following error:\n+      # failed to create containerd container: create container failed validation: container.Runtime.Name must be set: invalid argument\n+      runtime_type = \"io.containerd.runc.v2\"\n+      [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n+        # With cgroup v2 we need to use the systemd cgroup driver\n+        SystemdCgroup = true\n+    # If dragonfly is enabled, configure the local dfget as registry mirror\n+    # https://d7y.io/docs/v2.0.2/setup/runtime/containerd/mirror\n+    [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker-registry.discovery.wmnet\"]\n+      endpoint = [\"https://127.0.0.1:65001\",\"https://docker-registry.discovery.wmnet\"]\n+    [plugins.\"io.containerd.grpc.v1.cri\".cni]\n+      bin_dir = \"/opt/cni/bin\"\n+  [plugins.\"io.containerd.internal.v1.opt\"]\n+    # Debian overrides path from /opt/containerd\n+    path = \"/var/lib/containerd/opt\"", "parameters": "--- File[/etc/containerd/config.toml].orig\n+++ File[/etc/containerd/config.toml]\n\n+    group  => root\n+    mode   => 0440\n+    ensure => file\n+    notify => Service[containerd]\n+    owner  => root\n"}, {"resource": "Exec[/sbin/modprobe overlay]", "parameters": "--- Exec[/sbin/modprobe overlay].orig\n+++ Exec[/sbin/modprobe overlay]\n\n+    refreshonly => True\n+    unless      => /bin/lsmod | /bin/grep -q '^overlay '\n"}, {"resource": "Systemd::Service[kube-proxy]", "parameters": "--- Systemd::Service[kube-proxy].orig\n+++ Systemd::Service[kube-proxy]\n\n+    monitoring_enabled       => False\n+    subscribe                => File[/etc/kubernetes/proxy.conf]\n+    override                 => True\n+    monitoring_critical      => False\n+    restart                  => True\n+    service_params           => {}\n+    monitoring_contact_group => admins\n+    migration_task           => T407130\n+    ensure                   => present\n+    unit_type                => service\n"}, {"resource": "File[/var/run/kubernetes]", "parameters": "--- File[/var/run/kubernetes].orig\n+++ File[/var/run/kubernetes]\n\n+    mode   => 0700\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "Package[amd-k8s-node-labeller]", "parameters": "--- Package[amd-k8s-node-labeller].orig\n+++ Package[amd-k8s-node-labeller]\n\n+    ensure   => present\n+    provider => apt\n+    require  => K8s::Kubeconfig[/etc/amd/node-labeller-kubeconfig]\n"}, {"resource": "Systemd::Unit[ferm-ferm-service-auto-restart]", "parameters": "--- Systemd::Unit[ferm-ferm-service-auto-restart].orig\n+++ Systemd::Unit[ferm-ferm-service-auto-restart]\n\n+    override_filename => ferm-service-auto-restart\n+    override          => True\n+    restart           => False\n+    require           => ['Class[Systemd]']\n+    unit              => ferm\n+    source            => puppet:///modules/profile/kubernetes/node/ferm_systemd_override\n+    ensure            => present\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chained.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chained.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chained.pem]\n\n+    group   => root\n+    ensure  => file\n+    require => Exec[create chained cert /etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]\n+    owner   => kube\n"}, {"resource": "Exec[Generate cert mlserve__calico-cni]", "parameters": "--- Exec[Generate cert mlserve__calico-cni].orig\n+++ Exec[Generate cert mlserve__calico-cni]\n\n+    unless      => /usr/bin/test \"$(/usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__calico-cni.pem -noout -pubkey 2>&1)\" == \"$(/usr/bin/openssl pkey -pubout -in /etc/kubernetes/pki/mlserve__calico-cni-key.pem 2>&1)\"\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__calico-cni.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__calico-cni\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Cfssl::Csr[/etc/cfssl/csr/mlserve__calico-cni.csr]\n"}, {"resource": "Rsyslog::Conf[kubernetes]", "parameters": "--- Rsyslog::Conf[kubernetes].orig\n+++ Rsyslog::Conf[kubernetes]\n\n+    priority => 9\n+    mode     => 0444\n+    ensure   => present\n"}, {"resource": "Rsyslog::Conf[shellbox]", "parameters": "--- Rsyslog::Conf[shellbox].orig\n+++ Rsyslog::Conf[shellbox]\n\n+    source   => puppet:///modules/profile/rsyslog/shellbox.rsyslog.conf\n+    priority => 20\n+    mode     => 0444\n+    ensure   => present\n"}, {"resource": "File[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "parameters": "--- File[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.list].orig\n+++ File[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]\n\n+    owner  => root\n+    ensure => absent\n+    group  => root\n"}, {"resource": "File[/etc/sysctl.d/70-ipv6-fowarding-accept-ra.conf]", "content": "--- /etc/sysctl.d/70-ipv6-fowarding-accept-ra.conf.orig\n+++ /etc/sysctl.d/70-ipv6-fowarding-accept-ra.conf\n@@ -0,0 +1,3 @@\n+# sysctl parameters managed by Puppet.\n+net.ipv6.conf.all.forwarding = 1\n+net.ipv6.conf.ens11f1np1.accept_ra = 2", "parameters": "--- File[/etc/sysctl.d/70-ipv6-fowarding-accept-ra.conf].orig\n+++ File[/etc/sysctl.d/70-ipv6-fowarding-accept-ra.conf]\n\n+    group  => root\n+    ensure => present\n+    notify => Exec[update_sysctl]\n+    owner  => root\n"}, {"resource": "Exec[systemd daemon-reload for kube-proxy.service (kube-proxy)]", "parameters": "--- Exec[systemd daemon-reload for kube-proxy.service (kube-proxy)].orig\n+++ Exec[systemd daemon-reload for kube-proxy.service (kube-proxy)]\n\n+    command     => /bin/systemctl daemon-reload\n+    notify      => ['Service[kube-proxy]']\n+    refreshonly => True\n"}, {"resource": "Concat_file[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "parameters": "--- Concat_file[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources].orig\n+++ Concat_file[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]\n\n+    show_diff      => True\n+    tag            => _etc_apt_sources.list.d_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    format         => plain\n+    group          => root\n+    mode           => 0444\n+    order          => alpha\n+    owner          => root\n+    replace        => True\n+    backup         => puppet\n+    ensure_newline => False\n+    force          => False\n"}, {"resource": "Apt::Package_from_component[calico329]", "parameters": "--- Apt::Package_from_component[calico329].orig\n+++ Apt::Package_from_component[calico329]\n\n+    distro          => trixie-wikimedia\n+    component       => component/calico329\n+    ensure_packages => True\n+    packages        => {'calicoctl': '>=3.29 <3.30', 'calico-cni': '>=3.29 <3.30'}\n+    priority        => 1001\n+    uri             => http://apt.wikimedia.org/wikimedia\n+    ensure          => present\n"}, {"resource": "Systemd::Unit[amd-k8s-node-labeller-amd-devplugin-after-labeller]", "parameters": "--- Systemd::Unit[amd-k8s-node-labeller-amd-devplugin-after-labeller].orig\n+++ Systemd::Unit[amd-k8s-node-labeller-amd-devplugin-after-labeller]\n\n+    override_filename => amd-devplugin-after-labeller\n+    override          => True\n+    restart           => True\n+    require           => ['Class[Systemd]']\n+    unit              => amd-k8s-node-labeller\n+    ensure            => present\n"}, {"resource": "Exec[Generate cert mlserve__rsyslog refresh]", "parameters": "--- Exec[Generate cert mlserve__rsyslog refresh].orig\n+++ Exec[Generate cert mlserve__rsyslog refresh]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__rsyslog.csr | /usr/bin/cfssljson -bare /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/csr/mlserve__rsyslog.csr]\n+    notify      => ['Service[rsyslog]']\n"}, {"resource": "Exec[systemd daemon-reload for rsyslog-imfile-remedy.service (rsyslog-imfile-remedy.service)]", "parameters": "--- Exec[systemd daemon-reload for rsyslog-imfile-remedy.service (rsyslog-imfile-remedy.service)].orig\n+++ Exec[systemd daemon-reload for rsyslog-imfile-remedy.service (rsyslog-imfile-remedy.service)]\n\n+    command     => /bin/systemctl daemon-reload\n+    refreshonly => True\n"}, {"resource": "File[/etc/kubernetes/proxy.conf]", "content": "--- /etc/kubernetes/proxy.conf.orig\n+++ /etc/kubernetes/proxy.conf\n@@ -0,0 +1,18 @@\n+apiVersion: v1\n+kind: Config\n+preferences: {}\n+current-context: default-system\n+contexts:\n+- name: default-system\n+  context:\n+    cluster: default-cluster\n+    user: default-proxy\n+clusters:\n+- name: default-cluster\n+  cluster:\n+    server: https://ml-ctrl.svc.eqiad.wmnet:6443\n+users:\n+- name: default-proxy\n+  user:\n+    client-certificate: /etc/kubernetes/pki/mlserve__system_kube-proxy.pem\n+    client-key: /etc/kubernetes/pki/mlserve__system_kube-proxy-key.pem", "parameters": "--- File[/etc/kubernetes/proxy.conf].orig\n+++ File[/etc/kubernetes/proxy.conf]\n\n+    mode   => 0400\n+    group  => kube\n+    ensure => present\n+    owner  => kube\n"}, {"resource": "Class[Containerd::Nerdctl]", "parameters": "--- Class[Containerd::Nerdctl].orig\n+++ Class[Containerd::Nerdctl]\n\n+    ensure    => present\n+    namespace => k8s.io\n"}, {"resource": "Exec[Generate cert mlserve__calicoctl refresh]", "parameters": "--- Exec[Generate cert mlserve__calicoctl refresh].orig\n+++ Exec[Generate cert mlserve__calicoctl refresh]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__calicoctl.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__calicoctl\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/csr/mlserve__calicoctl.csr]\n"}, {"resource": "File[/etc/ferm/conf.d/10_calico_typha]", "content": "--- /etc/ferm/conf.d/10_calico_typha.orig\n+++ /etc/ferm/conf.d/10_calico_typha\n@@ -0,0 +1,6 @@\n+# Autogenerated by puppet. DO NOT EDIT BY HAND!\n+#\n+# \n+&R_SERVICE(tcp, 5473, $DOMAIN_NETWORKS);\n+\n+", "parameters": "--- File[/etc/ferm/conf.d/10_calico_typha].orig\n+++ File[/etc/ferm/conf.d/10_calico_typha]\n\n+    tag     => ferm\n+    require => File[/etc/ferm/conf.d]\n+    group   => root\n+    mode    => 0400\n+    ensure  => present\n+    notify  => Service[ferm]\n+    owner   => root\n"}, {"resource": "Ferm::Service[calico-bird]", "parameters": "--- Ferm::Service[calico-bird].orig\n+++ Ferm::Service[calico-bird]\n\n+    prio                => 10\n+    srange              => ($NETWORK_INFRA 10.64.167.1)\n+    notrack             => False\n+    desc                => \n+    port                => 179\n+    proto               => tcp\n+    ensure              => present\n+    unrestricted_access => False\n"}, {"resource": "Cfssl::Cert[mlserve__calico-cni]", "parameters": "--- Cfssl::Cert[mlserve__calico-cni].orig\n+++ Cfssl::Cert[mlserve__calico-cni]\n\n+    label           => mlserve\n+    notify_services => []\n+    common_name     => calico-cni\n+    outdir          => /etc/kubernetes/pki\n+    provide_chain   => True\n+    key             => {'algo': 'ecdsa', 'size': 256}\n+    group           => root\n+    renew_seconds   => 952200\n+    mode            => 0740\n+    owner           => root\n+    before_services => []\n+    hosts           => []\n+    environment     => ['GODEBUG=x509ignoreCN=0']\n+    ensure          => present\n+    names           => []\n+    auto_renew      => True\n"}, {"resource": "Rsyslog::Input::File[kubernetes-json]", "parameters": "--- Rsyslog::Input::File[kubernetes-json].orig\n+++ Rsyslog::Input::File[kubernetes-json]\n\n+    addceetag          => on\n+    syslog_tag         => kubernetes\n+    reopen_on_truncate => on\n+    syslog_tag_prefix  => input-file\n+    priority           => 8\n+    ensure             => present\n+    addmetadata        => on\n+    path               => /var/log/containers/*.log\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__kubelet_server.csr]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__kubelet_server.csr].orig\n+++ File[/etc/kubernetes/pki/mlserve__kubelet_server.csr]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => kube\n"}, {"resource": "File[/var/lib/kubelet]", "parameters": "--- File[/var/lib/kubelet].orig\n+++ File[/var/lib/kubelet]\n\n+    mode   => 0700\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "File[/etc/udev/rules.d/70-render.rules]", "content": "--- /etc/udev/rules.d/70-render.rules.orig\n+++ /etc/udev/rules.d/70-render.rules\n@@ -0,0 +1 @@\n+SUBSYSTEM==\"drm\", KERNEL==\"renderD*\", MODE=\"0666\"", "parameters": "--- File[/etc/udev/rules.d/70-render.rules].orig\n+++ File[/etc/udev/rules.d/70-render.rules]\n\n+    group => root\n+    mode  => 0544\n+    owner => root\n"}, {"resource": "File[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "parameters": "--- File[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.list].orig\n+++ File[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]\n\n+    owner  => root\n+    ensure => absent\n+    group  => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calicoctl.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calicoctl.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__calicoctl.pem]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Cfssl::Cert[mlserve__system_node_ml-serve1015_eqiad_wmnet]", "parameters": "--- Cfssl::Cert[mlserve__system_node_ml-serve1015_eqiad_wmnet].orig\n+++ Cfssl::Cert[mlserve__system_node_ml-serve1015_eqiad_wmnet]\n\n+    label           => mlserve\n+    notify_services => ['kubelet']\n+    common_name     => system:node:ml-serve1015.eqiad.wmnet\n+    outdir          => /etc/kubernetes/pki\n+    provide_chain   => True\n+    key             => {'algo': 'ecdsa', 'size': 256}\n+    group           => root\n+    renew_seconds   => 952200\n+    mode            => 0740\n+    owner           => kube\n+    before_services => []\n+    hosts           => []\n+    environment     => ['GODEBUG=x509ignoreCN=0']\n+    ensure          => present\n+    names           => [{'organisation': 'system:nodes'}]\n+    auto_renew      => True\n"}, {"resource": "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem]", "parameters": "--- File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem].orig\n+++ File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => dragonfly\n"}, {"resource": "Cfssl::Csr[/etc/cfssl/csr/mlserve__rsyslog.csr]", "parameters": "--- Cfssl::Csr[/etc/cfssl/csr/mlserve__rsyslog.csr].orig\n+++ Cfssl::Csr[/etc/cfssl/csr/mlserve__rsyslog.csr]\n\n+    hosts       => []\n+    key         => {'algo': 'ecdsa', 'size': 256}\n+    common_name => rsyslog\n+    ensure      => present\n+    names       => [{'organisation': 'view'}]\n"}, {"resource": "File[/etc/rsyslog.d/10-kubernetes-node-filters.conf]", "parameters": "--- File[/etc/rsyslog.d/10-kubernetes-node-filters.conf].orig\n+++ File[/etc/rsyslog.d/10-kubernetes-node-filters.conf]\n\n+    group  => root\n+    source => puppet:///modules/profile/kubernetes/node/kubernetes-node-filters.rsyslog.conf\n+    mode   => 0444\n+    ensure => present\n+    notify => Service[rsyslog]\n+    owner  => root\n"}, {"resource": "Systemd::Override[amd-devplugin-after-labeller]", "parameters": "--- Systemd::Override[amd-devplugin-after-labeller].orig\n+++ Systemd::Override[amd-devplugin-after-labeller]\n\n+    ensure  => present\n+    restart => True\n+    unit    => amd-k8s-node-labeller\n"}, {"resource": "Exec[create chained cert /etc/kubernetes/pki/mlserve__istio-cni.chain.pem]", "parameters": "--- Exec[create chained cert /etc/kubernetes/pki/mlserve__istio-cni.chain.pem].orig\n+++ Exec[create chained cert /etc/kubernetes/pki/mlserve__istio-cni.chain.pem]\n\n+    command   => /bin/cat /etc/kubernetes/pki/mlserve__istio-cni.pem /etc/kubernetes/pki/mlserve__istio-cni.chain.pem > /etc/kubernetes/pki/mlserve__istio-cni.chained.pem\n+    unless    => /usr/bin/test \"$(/bin/cat /etc/kubernetes/pki/mlserve__istio-cni.pem /etc/kubernetes/pki/mlserve__istio-cni.chain.pem | sha512sum)\" == \"$(/bin/cat /etc/kubernetes/pki/mlserve__istio-cni.chained.pem | sha512sum)\"\n\n+    subscribe => ['Exec[renew certificate - mlserve__istio-cni]', 'File[/etc/kubernetes/pki/mlserve__istio-cni.chain.pem]', 'File[/etc/kubernetes/pki/mlserve__istio-cni.pem]']\n+    require   => Exec[Generate cert mlserve__istio-cni refresh on intermediate ca change]\n"}, {"resource": "File[/etc/ferm/conf.d/10_kubelet-http]", "content": "--- /etc/ferm/conf.d/10_kubelet-http.orig\n+++ /etc/ferm/conf.d/10_kubelet-http\n@@ -0,0 +1,6 @@\n+# Autogenerated by puppet. DO NOT EDIT BY HAND!\n+#\n+# \n+&R_SERVICE(tcp, 10250, (@resolve((ml-serve-ctrl1001.eqiad.wmnet ml-serve-ctrl1002.eqiad.wmnet)) @resolve((ml-serve-ctrl1001.eqiad.wmnet ml-serve-ctrl1002.eqiad.wmnet), AAAA)));\n+\n+", "parameters": "--- File[/etc/ferm/conf.d/10_kubelet-http].orig\n+++ File[/etc/ferm/conf.d/10_kubelet-http]\n\n+    tag     => ferm\n+    require => File[/etc/ferm/conf.d]\n+    group   => root\n+    mode    => 0400\n+    ensure  => present\n+    notify  => Service[ferm]\n+    owner   => root\n"}, {"resource": "Class[Containerd]", "parameters": "--- Class[Containerd].orig\n+++ Class[Containerd]\n\n+    ensure  => present\n+    require => ['Class[Containerd::Configuration]']\n"}, {"resource": "File[/etc/default/prometheus-node-exporter]", "content": "--- /etc/default/prometheus-node-exporter.orig\n+++ /etc/default/prometheus-node-exporter\n@@ -15,6 +15,7 @@\n  --collector.netdev \\\n  --collector.netstat \\\n  --collector.netstat.fields=^(.*) \\\n+ --collector.processes \\\n  --collector.sockstat \\\n  --collector.stat \\\n  --collector.systemd.enable-restarts-metrics \\"}, {"resource": "Exec[renew certificate - mlserve__rsyslog]", "parameters": "--- Exec[renew certificate - mlserve__rsyslog].orig\n+++ Exec[renew certificate - mlserve__rsyslog]\n\n+    command     => /usr/bin/cfssl sign -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.csr | /usr/bin/cfssljson -bare /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Exec[Generate cert mlserve__rsyslog]\n+    unless      => /usr/bin/openssl x509 -in /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem -checkend 952200\n+    notify      => ['Service[rsyslog]']\n"}, {"resource": "Exec[apt_package_from_component_calico329]", "parameters": "--- Exec[apt_package_from_component_calico329].orig\n+++ Exec[apt_package_from_component_calico329]\n\n+    before      => ['Package[calicoctl]', 'Package[calico-cni]']\n+    command     => /usr/bin/apt-get update\n+    refreshonly => True\n+    subscribe   => Apt::Repository[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n"}, {"resource": "Exec[create chained cert /etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]", "parameters": "--- Exec[create chained cert /etc/kubernetes/pki/mlserve__kubelet_server.chain.pem].orig\n+++ Exec[create chained cert /etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]\n\n+    command   => /bin/cat /etc/kubernetes/pki/mlserve__kubelet_server.pem /etc/kubernetes/pki/mlserve__kubelet_server.chain.pem > /etc/kubernetes/pki/mlserve__kubelet_server.chained.pem\n+    subscribe => ['Exec[renew certificate - mlserve__kubelet_server]', 'File[/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]', 'File[/etc/kubernetes/pki/mlserve__kubelet_server.pem]']\n+    require   => Exec[Generate cert mlserve__kubelet_server refresh on intermediate ca change]\n+    unless    => /usr/bin/test \"$(/bin/cat /etc/kubernetes/pki/mlserve__kubelet_server.pem /etc/kubernetes/pki/mlserve__kubelet_server.chain.pem | sha512sum)\" == \"$(/bin/cat /etc/kubernetes/pki/mlserve__kubelet_server.chained.pem | sha512sum)\"\n\n+    notify    => ['Service[kubelet]']\n"}, {"resource": "Cfssl::Cert[mlserve__kubelet_server]", "parameters": "--- Cfssl::Cert[mlserve__kubelet_server].orig\n+++ Cfssl::Cert[mlserve__kubelet_server]\n\n+    label           => mlserve\n+    notify_services => ['kubelet']\n+    common_name     => kubelet\n+    outdir          => /etc/kubernetes/pki\n+    provide_chain   => True\n+    key             => {'algo': 'ecdsa', 'size': 256}\n+    group           => root\n+    renew_seconds   => 952200\n+    profile         => server\n+    mode            => 0740\n+    owner           => kube\n+    hosts           => ['ml-serve1015', 'ml-serve1015.eqiad.wmnet', '10.64.167.6', '2620:0:861:12f:10:64:167:6']\n+    before_services => []\n+    environment     => ['GODEBUG=x509ignoreCN=0']\n+    ensure          => present\n+    names           => []\n+    auto_renew      => True\n"}, {"resource": "Class[Profile::Rsyslog::Kubernetes]", "parameters": "--- Class[Profile::Rsyslog::Kubernetes].orig\n+++ Class[Profile::Rsyslog::Kubernetes]\n\n+    enable                  => True\n+    kubernetes_cluster_name => ml-serve-eqiad\n+    kafka_brokers           => ['kafka-logging1001.eqiad.wmnet:9093', 'kafka-logging1002.eqiad.wmnet:9093', 'kafka-logging1003.eqiad.wmnet:9093', 'kafka-logging1004.eqiad.wmnet:9093', 'kafka-logging1005.eqiad.wmnet:9093']\n"}, {"resource": "Cfssl::Csr[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "parameters": "--- Cfssl::Csr[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr].orig\n+++ Cfssl::Csr[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]\n\n+    hosts       => []\n+    key         => {'algo': 'ecdsa', 'size': 256}\n+    common_name => system:node:ml-serve1015.eqiad.wmnet\n+    ensure      => present\n+    names       => [{'organisation': 'system:nodes'}]\n"}, {"resource": "K8s::Kubeconfig[/etc/kubernetes/kubelet.conf]", "parameters": "--- K8s::Kubeconfig[/etc/kubernetes/kubelet.conf].orig\n+++ K8s::Kubeconfig[/etc/kubernetes/kubelet.conf]\n\n+    username    => default-auth\n+    group       => kube\n+    mode        => 0400\n+    owner       => kube\n+    require     => ['Class[K8s::Base_dirs]']\n+    auth_cert   => {'cert': '/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem', 'key': '/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet-key.pem', 'chain': '/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem', 'chained': '/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chained.pem'}\n+    ensure      => present\n+    master_host => ml-ctrl.svc.eqiad.wmnet\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calico-cni.chain.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calico-cni.chain.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__calico-cni.chain.pem]\n\n+    group  => root\n+    source => puppet:///modules/profile/pki/intermediates/mlserve-cert.pem\n+    mode   => 0440\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "parameters": "--- Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    target => /etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 10\n"}, {"resource": "Package[istio-cni]", "parameters": "--- Package[istio-cni].orig\n+++ Package[istio-cni]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "Class[Profile::Kubernetes::Node]", "parameters": "--- Class[Profile::Kubernetes::Node].orig\n+++ Class[Profile::Kubernetes::Node]\n\n+    feature_flags                   => {}\n+    kubelet_node_taints             => []\n+    require                         => ['Class[Profile::Rsyslog::Kubernetes]', 'Class[Profile::Netbox::Host]']\n+    kubernetes_cluster_name         => ml-serve-eqiad\n+    docker_kubernetes_user_password => somepassword2\n+    kubelet_node_labels             => []\n"}, {"resource": "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]", "parameters": "--- File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem].orig\n+++ File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]\n\n+    group  => root\n+    source => puppet:///modules/profile/pki/intermediates/mlserve-cert.pem\n+    mode   => 0440\n+    ensure => file\n+    owner  => root\n"}, {"resource": "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem]", "parameters": "--- File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem].orig\n+++ File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Exec[systemd daemon-reload for rsyslog-imfile-remedy.timer (rsyslog-imfile-remedy.timer)]", "parameters": "--- Exec[systemd daemon-reload for rsyslog-imfile-remedy.timer (rsyslog-imfile-remedy.timer)].orig\n+++ Exec[systemd daemon-reload for rsyslog-imfile-remedy.timer (rsyslog-imfile-remedy.timer)]\n\n+    before      => ['Service[rsyslog-imfile-remedy.timer]']\n+    command     => /bin/systemctl daemon-reload\n+    refreshonly => True\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__istio-cni.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__istio-cni.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__istio-cni.pem]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calico-cni.chained.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calico-cni.chained.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__calico-cni.chained.pem]\n\n+    group   => root\n+    ensure  => file\n+    require => Exec[create chained cert /etc/kubernetes/pki/mlserve__calico-cni.chain.pem]\n+    owner   => root\n"}, {"resource": "Exec[Generate cert mlserve__kubelet_server]", "parameters": "--- Exec[Generate cert mlserve__kubelet_server].orig\n+++ Exec[Generate cert mlserve__kubelet_server]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve -profile server /etc/cfssl/csr/mlserve__kubelet_server.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__kubelet_server\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Cfssl::Csr[/etc/cfssl/csr/mlserve__kubelet_server.csr]\n+    unless      => /usr/bin/test \"$(/usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__kubelet_server.pem -noout -pubkey 2>&1)\" == \"$(/usr/bin/openssl pkey -pubout -in /etc/kubernetes/pki/mlserve__kubelet_server-key.pem 2>&1)\"\n\n+    notify      => ['Service[kubelet]']\n"}, {"resource": "Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "parameters": "--- Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header].orig\n+++ Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]\n\n+    source => puppet:///modules/apt/sources-deb822-header.txt\n+    tag    => _etc_apt_sources.list.d_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    target => /etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 01\n"}, {"resource": "File[/etc/systemd/system/kubelet.service.d]", "parameters": "--- File[/etc/systemd/system/kubelet.service.d].orig\n+++ File[/etc/systemd/system/kubelet.service.d]\n\n+    mode   => 0555\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "File[/etc/sysctl.d/75-kube_proxy_icmp.conf]", "content": "--- /etc/sysctl.d/75-kube_proxy_icmp.conf.orig\n+++ /etc/sysctl.d/75-kube_proxy_icmp.conf\n@@ -0,0 +1,4 @@\n+# sysctl parameters managed by Puppet.\n+net.ipv4.conf.all.send_redirects = 0\n+net.ipv4.conf.default.send_redirects = 0\n+net.ipv4.conf.ens11f1np1.send_redirects = 0", "parameters": "--- File[/etc/sysctl.d/75-kube_proxy_icmp.conf].orig\n+++ File[/etc/sysctl.d/75-kube_proxy_icmp.conf]\n\n+    group  => root\n+    ensure => present\n+    notify => Exec[update_sysctl]\n+    owner  => root\n"}, {"resource": "Exec[Generate cert mlserve__istio-cni]", "parameters": "--- Exec[Generate cert mlserve__istio-cni].orig\n+++ Exec[Generate cert mlserve__istio-cni]\n\n+    unless      => /usr/bin/test \"$(/usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__istio-cni.pem -noout -pubkey 2>&1)\" == \"$(/usr/bin/openssl pkey -pubout -in /etc/kubernetes/pki/mlserve__istio-cni-key.pem 2>&1)\"\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__istio-cni.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__istio-cni\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Cfssl::Csr[/etc/cfssl/csr/mlserve__istio-cni.csr]\n"}, {"resource": "Class[Adduser]", "parameters": "--- Class[Adduser].orig\n+++ Class[Adduser]\n\n@@\n-    before => ['Package[puppet]', 'Package[facter]', 'Package[augeas-tools]', 'Package[virt-what]', 'Package[puppet-module-puppetlabs-augeas-core]', 'Package[python3-prometheus-client]', 'Package[python3-yaml]', 'Package[ruby-net-ssh]', 'Package[openssl]', 'Package[ssl-cert]', 'Package[ca-certificates]', 'Package[wmf-certificates]', 'Package[ntp]', 'Package[systemd-timesyncd]', 'Package[exim4-config]', 'Package[exim4-daemon-light]', 'Package[logrotate]', 'Package[prometheus-node-exporter]', 'Package[bsdutils]', 'Package[smartmontools]', 'Package[rsyslog]', 'Package[rsyslog-openssl]', 'Package[cadvisor]', 'Package[acct]', 'Package[byobu]', 'Package[colordiff]', 'Package[curl]', 'Package[debian-goodies]', 'Package[ethtool]', 'Package[gdb]', 'Package[gdisk]', 'Package[git]', 'Package[htop]', 'Package[httpry]', 'Package[iotop]', 'Package[iperf]', 'Package[jq]', 'Package[libtemplate-perl]', 'Package[lldpd]', 'Package[lshw]', 'Package[molly-guard]', 'Package[moreutils]', 'Package[net-tools]', 'Package[numactl]', 'Package[ncdu]', 'Package[ngrep]', 'Package[pigz]', 'Package[psmisc]', 'Package[pv]', 'Package[python3]', 'Package[screen]', 'Package[strace]', 'Package[sysstat]', 'Package[tcpdump]', 'Package[tmux]', 'Package[tree]', 'Package[vim]', 'Package[vim-addon-manager]', 'Package[vim-scripts]', 'Package[wipe]', 'Package[xfsprogs]', 'Package[zsh]', 'Package[icdiff]', 'Package[linux-perf]', 'Package[bsd-mailx]', 'Package[ack]', 'Package[netcat-openbsd]', 'Package[tshark]', 'Package[fzf]', 'Package[ripgrep]', 'Package[fd-find]', 'Package[kitty-terminfo]', 'Package[mtr-tiny]', 'Package[bat]', 'Package[efibootmgr]', 'Package[bind9-dnsutils]', 'Package[tzdata]', 'Package[python3-wmflib]', 'Package[starship]', 'Package[ruby-sorted-set]', 'Package[btop]', 'Package[linux-sysctl-defaults]', 'Package[apport]', 'Package[command-not-found]', 'Package[command-not-found-data]', 'Package[ecryptfs-utils]', 'Package[mlocate]', 'Package[os-prober]', 'Package[python3-apport]', 'Package[wpasupplicant]', 'Package[atop]', 'Package[apt-listchanges]', 'Package[isc-dhcp-client]', 'Package[rasdaemon]', 'Package[openssh-client]', 'Package[openssh-server]', 'Package[debdeploy-client]', 'Package[python3-dateutil]', 'Package[sudo]', 'Package[golang-cfssl]', 'Package[debmonitor-client]', 'Package[nagios-nrpe-server]', 'Package[monitoring-plugins]', 'Package[monitoring-plugins-basic]', 'Package[monitoring-plugins-standard]', 'Package[liburiparser1]', 'Package[python3-attr]', 'Package[freeipmi-tools]', 'Package[freeipmi-ipmiseld]', 'Package[rsyslog-kafka]', 'Package[emacs-nox]', 'Package[prometheus-ipmi-exporter]', 'Package[libnet-dns-perl]', 'Package[iptables]', 'Package[ferm]', 'Package[ulogd2]', 'Package[conntrack]', 'Package[rocm-smi]', 'Package[python3-requests]', 'Package[firmware-amd-graphics]', 'Package[ruby-concurrent]', 'Package[ruby]', 'Package[libruby]', 'Package[puppet-agent]', 'Package[prometheus-rsyslog-exporter]', 'Package[initramfs-tools]', 'Package[python3-click]', 'Package[python3-box]', 'Package[confd]', 'Package[python3-toml]']\n+    before => ['Package[puppet]', 'Package[facter]', 'Package[augeas-tools]', 'Package[virt-what]', 'Package[puppet-module-puppetlabs-augeas-core]', 'Package[python3-prometheus-client]', 'Package[python3-yaml]', 'Package[ruby-net-ssh]', 'Package[openssl]', 'Package[ssl-cert]', 'Package[ca-certificates]', 'Package[wmf-certificates]', 'Package[ntp]', 'Package[systemd-timesyncd]', 'Package[exim4-config]', 'Package[exim4-daemon-light]', 'Package[logrotate]', 'Package[prometheus-node-exporter]', 'Package[bsdutils]', 'Package[smartmontools]', 'Package[rsyslog]', 'Package[rsyslog-openssl]', 'Package[cadvisor]', 'Package[acct]', 'Package[byobu]', 'Package[colordiff]', 'Package[curl]', 'Package[debian-goodies]', 'Package[ethtool]', 'Package[gdb]', 'Package[gdisk]', 'Package[git]', 'Package[htop]', 'Package[httpry]', 'Package[iotop]', 'Package[iperf]', 'Package[jq]', 'Package[libtemplate-perl]', 'Package[lldpd]', 'Package[lshw]', 'Package[molly-guard]', 'Package[moreutils]', 'Package[net-tools]', 'Package[numactl]', 'Package[ncdu]', 'Package[ngrep]', 'Package[pigz]', 'Package[psmisc]', 'Package[pv]', 'Package[python3]', 'Package[screen]', 'Package[strace]', 'Package[sysstat]', 'Package[tcpdump]', 'Package[tmux]', 'Package[tree]', 'Package[vim]', 'Package[vim-addon-manager]', 'Package[vim-scripts]', 'Package[wipe]', 'Package[xfsprogs]', 'Package[zsh]', 'Package[icdiff]', 'Package[linux-perf]', 'Package[bsd-mailx]', 'Package[ack]', 'Package[netcat-openbsd]', 'Package[tshark]', 'Package[fzf]', 'Package[ripgrep]', 'Package[fd-find]', 'Package[kitty-terminfo]', 'Package[mtr-tiny]', 'Package[bat]', 'Package[efibootmgr]', 'Package[bind9-dnsutils]', 'Package[tzdata]', 'Package[python3-wmflib]', 'Package[starship]', 'Package[ruby-sorted-set]', 'Package[btop]', 'Package[linux-sysctl-defaults]', 'Package[apport]', 'Package[command-not-found]', 'Package[command-not-found-data]', 'Package[ecryptfs-utils]', 'Package[mlocate]', 'Package[os-prober]', 'Package[python3-apport]', 'Package[wpasupplicant]', 'Package[atop]', 'Package[apt-listchanges]', 'Package[isc-dhcp-client]', 'Package[rasdaemon]', 'Package[openssh-client]', 'Package[openssh-server]', 'Package[debdeploy-client]', 'Package[python3-dateutil]', 'Package[sudo]', 'Package[golang-cfssl]', 'Package[debmonitor-client]', 'Package[nagios-nrpe-server]', 'Package[monitoring-plugins]', 'Package[monitoring-plugins-basic]', 'Package[monitoring-plugins-standard]', 'Package[liburiparser1]', 'Package[python3-attr]', 'Package[freeipmi-tools]', 'Package[freeipmi-ipmiseld]', 'Package[rsyslog-kafka]', 'Package[emacs-nox]', 'Package[prometheus-ipmi-exporter]', 'Package[libnet-dns-perl]', 'Package[iptables]', 'Package[ferm]', 'Package[ulogd2]', 'Package[conntrack]', 'Package[dragonfly-dfdaemon]', 'Package[dragonfly-dfget]', 'Package[crictl]', 'Package[containerd]', 'Package[nerdctl]', 'Package[rsyslog-kubernetes]', 'Package[linux-cpupower]', 'Package[apparmor]', 'Package[socat]', 'Package[amd-k8s-device-plugin]', 'Package[amd-k8s-node-labeller]', 'Package[rocm-smi]', 'Package[python3-requests]', 'Package[wikimedia-lvs-realserver]', 'Package[ruby-concurrent]', 'Package[ruby]', 'Package[libruby]', 'Package[puppet-agent]', 'Package[linux-image-6.16.3+deb13-amd64]', 'Package[prometheus-rsyslog-exporter]', 'Package[initramfs-tools]', 'Package[python3-click]', 'Package[python3-box]', 'Package[confd]', 'Package[python3-toml]', 'Package[kubernetes-node]', 'Package[calicoctl]', 'Package[calico-cni]', 'Package[istio-cni]', 'Package[firmware-amd-graphics]']\n"}, {"resource": "File[/etc/systemd/system/kube-proxy.service.d/puppet-override.conf]", "content": "--- /etc/systemd/system/kube-proxy.service.d/puppet-override.conf.orig\n+++ /etc/systemd/system/kube-proxy.service.d/puppet-override.conf\n@@ -0,0 +1,2 @@\n+[Unit]\n+After = ferm.service", "parameters": "--- File[/etc/systemd/system/kube-proxy.service.d/puppet-override.conf].orig\n+++ File[/etc/systemd/system/kube-proxy.service.d/puppet-override.conf]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[systemd daemon-reload for kube-proxy.service (kube-proxy)]\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__istio-cni.csr]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__istio-cni.csr].orig\n+++ File[/etc/kubernetes/pki/mlserve__istio-cni.csr]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Package[kubernetes-node]", "parameters": "--- Package[kubernetes-node].orig\n+++ Package[kubernetes-node]\n\n+    ensure   => >=1.31 <1.32\n+    provider => apt\n+    require  => Apt::Package_from_component[kubernetes131]\n"}, {"resource": "Concat[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "parameters": "--- Concat[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources].orig\n+++ Concat[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]\n\n+    show_diff      => True\n+    format         => plain\n+    force          => False\n+    group          => root\n+    mode           => 0444\n+    order          => alpha\n+    owner          => root\n+    backup         => puppet\n+    replace        => True\n+    warn           => False\n+    ensure_newline => False\n+    ensure         => present\n+    notify         => Exec[apt_repository_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n+    path           => /etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n"}, {"resource": "File[/etc/nerdctl/nerdctl.toml]", "content": "--- /etc/nerdctl/nerdctl.toml.orig\n+++ /etc/nerdctl/nerdctl.toml\n@@ -0,0 +1,5 @@\n+# SPDX-License-Identifier: Apache-2.0\n+#\n+# For documentation of the available options, see:\n+# https://github.com/containerd/nerdctl/blob/main/docs/config.md\n+namespace = \"k8s.io\"", "parameters": "--- File[/etc/nerdctl/nerdctl.toml].orig\n+++ File[/etc/nerdctl/nerdctl.toml]\n\n+    mode   => 0644\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calico-cni-key.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calico-cni-key.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__calico-cni-key.pem]\n\n+    show_diff => False\n+    backup    => False\n+    group     => root\n+    mode      => 0440\n+    ensure    => file\n+    owner     => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calicoctl-key.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calicoctl-key.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__calicoctl-key.pem]\n\n+    show_diff => False\n+    backup    => False\n+    group     => root\n+    mode      => 0440\n+    ensure    => file\n+    owner     => root\n"}, {"resource": "Cfssl::Csr[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]", "parameters": "--- Cfssl::Csr[/etc/cfssl/csr/mlserve__system_kube-proxy.csr].orig\n+++ Cfssl::Csr[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]\n\n+    hosts       => []\n+    key         => {'algo': 'ecdsa', 'size': 256}\n+    common_name => system:kube-proxy\n+    ensure      => present\n+    names       => [{'organisation': 'system:node-proxier'}]\n"}, {"resource": "Cfssl::Cert[mlserve__istio-cni]", "parameters": "--- Cfssl::Cert[mlserve__istio-cni].orig\n+++ Cfssl::Cert[mlserve__istio-cni]\n\n+    label           => mlserve\n+    notify_services => []\n+    common_name     => istio-cni\n+    outdir          => /etc/kubernetes/pki\n+    provide_chain   => True\n+    key             => {'algo': 'ecdsa', 'size': 256}\n+    group           => root\n+    renew_seconds   => 952200\n+    mode            => 0740\n+    owner           => root\n+    before_services => []\n+    hosts           => []\n+    environment     => ['GODEBUG=x509ignoreCN=0']\n+    ensure          => present\n+    names           => []\n+    auto_renew      => True\n"}, {"resource": "Package[calico-cni]", "parameters": "--- Package[calico-cni].orig\n+++ Package[calico-cni]\n\n+    ensure   => >=3.29 <3.30\n+    provider => apt\n"}, {"resource": "File[/etc/rsyslog.d/08-input-file-kubernetes-json.conf]", "content": "--- /etc/rsyslog.d/08-input-file-kubernetes-json.conf.orig\n+++ /etc/rsyslog.d/08-input-file-kubernetes-json.conf\n@@ -0,0 +1,8 @@\n+# This file managed by puppet rsyslog::input::file\n+\n+input(type=\"imfile\"\n+      File=\"/var/log/containers/*.log\"\n+      reopenOnTruncate=\"on\"\n+      addMetadata=\"on\"\n+      addCeeTag=\"on\"\n+      Tag=\"input-file-kubernetes\")", "parameters": "--- File[/etc/rsyslog.d/08-input-file-kubernetes-json.conf].orig\n+++ File[/etc/rsyslog.d/08-input-file-kubernetes-json.conf]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Service[rsyslog]\n+    owner  => root\n"}, {"resource": "Package[wikimedia-lvs-realserver]", "parameters": "--- Package[wikimedia-lvs-realserver].orig\n+++ Package[wikimedia-lvs-realserver]\n\n+    ensure   => present\n+    provider => apt\n+    require  => File[/etc/default/wikimedia-lvs-realserver]\n"}, {"resource": "File[/etc/rsyslog.d/09-kubernetes.conf]", "content": "--- /etc/rsyslog.d/09-kubernetes.conf.orig\n+++ /etc/rsyslog.d/09-kubernetes.conf\n@@ -0,0 +1,9 @@\n+module(load=\"mmkubernetes\"\n+        KubernetesURL=\"https://ml-ctrl.svc.eqiad.wmnet:6443\"\n+        tls.mycert=\"/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem\"\n+        tls.myprivkey=\"/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog-key.pem\")\n+action(type=\"mmkubernetes\"\n+       name=\"mmkubernetes\"\n+       action.resumeRetryCount=\"-1\"\n+       action.resumeIntervalMax=\"300\"\n+       action.reportSuspensionContinuation=\"on\")", "parameters": "--- File[/etc/rsyslog.d/09-kubernetes.conf].orig\n+++ File[/etc/rsyslog.d/09-kubernetes.conf]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Service[rsyslog]\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_kube-proxy-key.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_kube-proxy-key.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_kube-proxy-key.pem]\n\n+    show_diff => False\n+    backup    => False\n+    group     => root\n+    mode      => 0440\n+    ensure    => file\n+    owner     => kube\n"}, {"resource": "Exec[renew certificate - mlserve__system_kube-proxy]", "parameters": "--- Exec[renew certificate - mlserve__system_kube-proxy].orig\n+++ Exec[renew certificate - mlserve__system_kube-proxy]\n\n+    command     => /usr/bin/cfssl sign -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/kubernetes/pki/mlserve__system_kube-proxy.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__system_kube-proxy\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Exec[Generate cert mlserve__system_kube-proxy]\n+    unless      => /usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__system_kube-proxy.pem -checkend 952200\n+    notify      => ['Service[kube-proxy]']\n"}, {"resource": "Group[kube]", "parameters": "--- Group[kube].orig\n+++ Group[kube]\n\n+    system => True\n+    ensure => present\n"}, {"resource": "File[/lib/systemd/system/rsyslog-imfile-remedy.service]", "content": "--- /lib/systemd/system/rsyslog-imfile-remedy.service.orig\n+++ /lib/systemd/system/rsyslog-imfile-remedy.service\n@@ -0,0 +1,8 @@\n+[Unit]\n+Description=Restart rsyslog T357616\n+Documentation=https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+\n+[Service]\n+Type=oneshot\n+User=root\n+ExecStart=/usr/bin/systemctl try-restart rsyslog", "parameters": "--- File[/lib/systemd/system/rsyslog-imfile-remedy.service].orig\n+++ File[/lib/systemd/system/rsyslog-imfile-remedy.service]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[systemd daemon-reload for rsyslog-imfile-remedy.service (rsyslog-imfile-remedy.service)]\n+    owner  => root\n"}, {"resource": "Systemd::Timer[rsyslog-release-deleted-inotify-watches]", "parameters": "--- Systemd::Timer[rsyslog-release-deleted-inotify-watches].orig\n+++ Systemd::Timer[rsyslog-release-deleted-inotify-watches]\n\n+    accuracy           => 15sec\n+    unit_name          => rsyslog-release-deleted-inotify-watches.service\n+    ensure             => absent\n+    fixed_random_delay => False\n+    splay              => 0\n+    timer_intervals    => [{'start': 'OnCalendar', 'interval': '*-*-* *:54:00'}]\n"}, {"resource": "Exec[Generate cert mlserve__system_kube-proxy refresh on intermediate ca change]", "parameters": "--- Exec[Generate cert mlserve__system_kube-proxy refresh on intermediate ca change].orig\n+++ Exec[Generate cert mlserve__system_kube-proxy refresh on intermediate ca change]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__system_kube-proxy.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__system_kube-proxy\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]\n+    notify      => ['Service[kube-proxy]']\n"}, {"resource": "File[/etc/apt/preferences.d/apt_pin_firmware_amd_graphics_trixie_bpo_trixie_bpo.pref]", "content": "--- /etc/apt/preferences.d/apt_pin_firmware_amd_graphics_trixie_bpo_trixie_bpo.pref.orig\n+++ /etc/apt/preferences.d/apt_pin_firmware_amd_graphics_trixie_bpo_trixie_bpo.pref\n@@ -0,0 +1,3 @@\n+Package: firmware-amd-graphics\n+Pin: release a=trixie-backports\n+Pin-Priority: 1001", "parameters": "--- File[/etc/apt/preferences.d/apt_pin_firmware_amd_graphics_trixie_bpo_trixie_bpo.pref].orig\n+++ File[/etc/apt/preferences.d/apt_pin_firmware_amd_graphics_trixie_bpo_trixie_bpo.pref]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[exec-apt-get-update-firmware-amd-graphics-trixie-bpo_trixie-bpo]\n+    owner  => root\n"}, {"resource": "File[/var/log/rsyslog-release-deleted-inotify-watches]", "parameters": "--- File[/var/log/rsyslog-release-deleted-inotify-watches].orig\n+++ File[/var/log/rsyslog-release-deleted-inotify-watches]\n\n+    backup => False\n+    group  => root\n+    ensure => absent\n+    mode   => 0755\n+    force  => True\n+    owner  => root\n"}, {"resource": "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh]", "parameters": "--- Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh].orig\n+++ Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label discovery  /etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr | /usr/bin/cfssljson -bare /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]\n+    notify      => ['Service[dragonfly-dfdaemon]']\n"}, {"resource": "File[/etc/ferm/conf.d/10_dragonfly_dfget]", "content": "--- /etc/ferm/conf.d/10_dragonfly_dfget.orig\n+++ /etc/ferm/conf.d/10_dragonfly_dfget\n@@ -0,0 +1,6 @@\n+# Autogenerated by puppet. DO NOT EDIT BY HAND!\n+#\n+# \n+&R_SERVICE(tcp, 15001, $DOMAIN_NETWORKS);\n+\n+", "parameters": "--- File[/etc/ferm/conf.d/10_dragonfly_dfget].orig\n+++ File[/etc/ferm/conf.d/10_dragonfly_dfget]\n\n+    tag     => ferm\n+    require => File[/etc/ferm/conf.d]\n+    group   => root\n+    mode    => 0400\n+    ensure  => present\n+    notify  => Service[ferm]\n+    owner   => root\n"}, {"resource": "Exec[Generate cert mlserve__kubelet_server refresh on intermediate ca change]", "parameters": "--- Exec[Generate cert mlserve__kubelet_server refresh on intermediate ca change].orig\n+++ Exec[Generate cert mlserve__kubelet_server refresh on intermediate ca change]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve -profile server /etc/cfssl/csr/mlserve__kubelet_server.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__kubelet_server\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]\n+    notify      => ['Service[kubelet]']\n"}, {"resource": "Firewall::Service[calico-typha]", "parameters": "--- Firewall::Service[calico-typha].orig\n+++ Firewall::Service[calico-typha]\n\n+    prio                => 10\n+    notrack             => False\n+    src_sets            => ['DOMAIN_NETWORKS']\n+    desc                => \n+    port                => 5473\n+    proto               => tcp\n+    ensure              => present\n+    unrestricted_access => False\n"}, {"resource": "File[/lib/systemd/system/rsyslog-imfile-remedy.timer]", "content": "--- /lib/systemd/system/rsyslog-imfile-remedy.timer.orig\n+++ /lib/systemd/system/rsyslog-imfile-remedy.timer\n@@ -0,0 +1,12 @@\n+[Unit]\n+Description=Periodic execution of rsyslog-imfile-remedy.service\n+\n+[Timer]\n+Unit=rsyslog-imfile-remedy.service\n+# Accuracy sets the maximum time interval around the execution time we want to allow\n+AccuracySec=15sec\n+OnCalendar=*-*-* 00/3:41:00\n+RandomizedDelaySec=30\n+\n+[Install]\n+WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/rsyslog-imfile-remedy.timer].orig\n+++ File[/lib/systemd/system/rsyslog-imfile-remedy.timer]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[systemd daemon-reload for rsyslog-imfile-remedy.timer (rsyslog-imfile-remedy.timer)]\n+    owner  => root\n"}, {"resource": "Exec[Generate cert mlserve__calico-cni refresh on intermediate ca change]", "parameters": "--- Exec[Generate cert mlserve__calico-cni refresh on intermediate ca change].orig\n+++ Exec[Generate cert mlserve__calico-cni refresh on intermediate ca change]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__calico-cni.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__calico-cni\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/kubernetes/pki/mlserve__calico-cni.chain.pem]\n"}, {"resource": "Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "parameters": "--- Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header].orig\n+++ Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]\n\n+    source => puppet:///modules/apt/sources-deb822-header.txt\n+    target => /etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 01\n"}, {"resource": "File[/etc/sysctl.d/75-kube_proxy_conntrack.conf]", "content": "--- /etc/sysctl.d/75-kube_proxy_conntrack.conf.orig\n+++ /etc/sysctl.d/75-kube_proxy_conntrack.conf\n@@ -0,0 +1,2 @@\n+# sysctl parameters managed by Puppet.\n+net.netfilter.nf_conntrack_max = 1048576", "parameters": "--- File[/etc/sysctl.d/75-kube_proxy_conntrack.conf].orig\n+++ File[/etc/sysctl.d/75-kube_proxy_conntrack.conf]\n\n+    group  => root\n+    ensure => present\n+    notify => Exec[update_sysctl]\n+    owner  => root\n"}, {"resource": "Cfssl::Cert[discovery__ml-serve1015_eqiad_wmnet]", "parameters": "--- Cfssl::Cert[discovery__ml-serve1015_eqiad_wmnet].orig\n+++ Cfssl::Cert[discovery__ml-serve1015_eqiad_wmnet]\n\n+    label           => discovery\n+    notify_services => ['dragonfly-dfdaemon']\n+    common_name     => ml-serve1015.eqiad.wmnet\n+    outdir          => /etc/dragonfly\n+    provide_chain   => True\n+    key             => {'algo': 'ecdsa', 'size': 256}\n+    group           => root\n+    renew_seconds   => 952200\n+    mode            => 0740\n+    owner           => dragonfly\n+    hosts           => ['ml-serve1015', 'ml-serve1015.eqiad.wmnet', 'docker-registry.discovery.wmnet', '127.0.0.1', '::1', 'localhost']\n+    before_services => []\n+    environment     => ['GODEBUG=x509ignoreCN=0']\n+    ensure          => present\n+    names           => []\n+    auto_renew      => True\n"}, {"resource": "File[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "parameters": "--- File[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.list].orig\n+++ File[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]\n\n+    owner  => root\n+    ensure => absent\n+    group  => root\n"}, {"resource": "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]", "parameters": "--- File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem].orig\n+++ File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]\n\n+    group  => root\n+    source => puppet:///modules/profile/pki/intermediates/discovery-cert.pem\n+    mode   => 0440\n+    ensure => file\n+    owner  => dragonfly\n"}, {"resource": "Service[apparmor]", "parameters": "--- Service[apparmor].orig\n+++ Service[apparmor]\n\n+    hasstatus  => True\n+    hasrestart => True\n+    ensure     => running\n+    require    => Package[apparmor]\n"}, {"resource": "File[/usr/libexec/cpupower]", "parameters": "--- File[/usr/libexec/cpupower].orig\n+++ File[/usr/libexec/cpupower]\n\n+    group  => root\n+    source => puppet:///modules/cpufrequtils/cpupower.sh\n+    mode   => 0555\n+    ensure => present\n+    owner  => root\n"}, {"resource": "Cfssl::Cert[mlserve__rsyslog]", "parameters": "--- Cfssl::Cert[mlserve__rsyslog].orig\n+++ Cfssl::Cert[mlserve__rsyslog]\n\n+    key             => {'algo': 'ecdsa', 'size': 256}\n+    label           => mlserve\n+    group           => root\n+    renew_seconds   => 952200\n+    provide_chain   => True\n+    mode            => 0740\n+    notify_services => ['rsyslog']\n+    owner           => root\n+    before_services => []\n+    hosts           => []\n+    common_name     => rsyslog\n+    environment     => ['GODEBUG=x509ignoreCN=0']\n+    ensure          => present\n+    names           => [{'organisation': 'view'}]\n+    auto_renew      => True\n"}, {"resource": "File[/etc/default/wikimedia-lvs-realserver]", "content": "--- /etc/default/wikimedia-lvs-realserver.orig\n+++ /etc/default/wikimedia-lvs-realserver\n@@ -0,0 +1,10 @@\n+# This file is managed by puppet!\n+\n+\n+\n+# Location of the sysctl file containing LVS ARP settings\n+SYSCTLFILE=/usr/share/wikimedia-lvs-realserver/sysctl.conf\n+\n+# LVS service IPs to be bound to the loopback interface,\n+# separate using spaces\n+LVS_SERVICE_IPS=\"10.2.2.63 10.2.2.84\"", "parameters": "--- File[/etc/default/wikimedia-lvs-realserver].orig\n+++ File[/etc/default/wikimedia-lvs-realserver]\n\n+    mode   => 0444\n+    group  => root\n+    ensure => present\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem]\n\n+    mode   => 0440\n+    group  => amd-nodelabeller\n+    ensure => file\n+    owner  => amd-nodelabeller\n"}, {"resource": "Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.service (rsyslog-release-deleted-inotify-watches.service)]", "parameters": "--- Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.service (rsyslog-release-deleted-inotify-watches.service)].orig\n+++ Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.service (rsyslog-release-deleted-inotify-watches.service)]\n\n+    command     => /bin/systemctl daemon-reload\n+    refreshonly => True\n"}, {"resource": "Cfssl::Csr[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]", "parameters": "--- Cfssl::Csr[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr].orig\n+++ Cfssl::Csr[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]\n\n+    hosts       => ['ml-serve1015', 'ml-serve1015.eqiad.wmnet', 'docker-registry.discovery.wmnet', '127.0.0.1', '::1', 'localhost']\n+    key         => {'algo': 'ecdsa', 'size': 256}\n+    common_name => ml-serve1015.eqiad.wmnet\n+    ensure      => present\n+    names       => []\n"}, {"resource": "Class[Profile::Amd_gpu]", "parameters": "--- Class[Profile::Amd_gpu].orig\n+++ Class[Profile::Amd_gpu]\n\n+    kubernetes_cluster_name => ml-serve-eqiad\n@@\n-    is_basic_gpu_node       => True\n+    is_basic_gpu_node       => False\n@@\n-    firmwares_from_bpo      => False\n+    firmwares_from_bpo      => True\n@@\n-    is_kubernetes_node      => False\n+    is_kubernetes_node      => True\n"}, {"resource": "File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.timer]", "content": "--- /lib/systemd/system/rsyslog-release-deleted-inotify-watches.timer.orig\n+++ /lib/systemd/system/rsyslog-release-deleted-inotify-watches.timer\n@@ -0,0 +1,12 @@\n+[Unit]\n+Description=Periodic execution of rsyslog-release-deleted-inotify-watches.service\n+\n+[Timer]\n+Unit=rsyslog-release-deleted-inotify-watches.service\n+# Accuracy sets the maximum time interval around the execution time we want to allow\n+AccuracySec=15sec\n+OnCalendar=*-*-* *:54:00\n+RandomizedDelaySec=0\n+\n+[Install]\n+WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.timer].orig\n+++ File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.timer]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    notify => Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.timer (rsyslog-release-deleted-inotify-watches.timer)]\n+    owner  => root\n"}, {"resource": "Class[Dragonfly::Dfdaemon]", "parameters": "--- Class[Dragonfly::Dfdaemon].orig\n+++ Class[Dragonfly::Dfdaemon]\n\n+    supernodes           => ['dragonfly-supernode1001.eqiad.wmnet:8002=1']\n+    ratelimit            => 100M\n+    docker_registry_fqdn => docker-registry.discovery.wmnet\n+    dfdaemon_ssl_cert    => /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem\n+    ensure               => present\n+    proxy_urls_regex     => ['wikimedia/machinelearning-liftwing.*/blobs/sha256.*', 'amd-pytorch.*/blobs/sha256.*']\n+    dfdaemon_ssl_key     => /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem\n"}, {"resource": "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.csr]", "parameters": "--- File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.csr].orig\n+++ File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.csr]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Package[calicoctl]", "parameters": "--- Package[calicoctl].orig\n+++ Package[calicoctl]\n\n+    ensure   => >=3.29 <3.30\n+    provider => apt\n"}, {"resource": "Service[rsyslog-release-deleted-inotify-watches.timer]", "parameters": "--- Service[rsyslog-release-deleted-inotify-watches.timer].orig\n+++ Service[rsyslog-release-deleted-inotify-watches.timer]\n\n+    enable   => False\n+    ensure   => stopped\n+    provider => systemd\n+    before   => ['Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.timer (rsyslog-release-deleted-inotify-watches.timer)]']\n"}, {"resource": "Exec[Generate cert mlserve__amdgpu-node-labeller]", "parameters": "--- Exec[Generate cert mlserve__amdgpu-node-labeller].orig\n+++ Exec[Generate cert mlserve__amdgpu-node-labeller]\n\n+    unless      => /usr/bin/test \"$(/usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem -noout -pubkey 2>&1)\" == \"$(/usr/bin/openssl pkey -pubout -in /etc/kubernetes/pki/mlserve__amdgpu-node-labeller-key.pem 2>&1)\"\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__amdgpu-node-labeller\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Cfssl::Csr[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_kube-proxy.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_kube-proxy.pem]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => kube\n"}, {"resource": "File[/etc/systemd/system/kubelet.service.d/container-runtime.conf]", "content": "--- /etc/systemd/system/kubelet.service.d/container-runtime.conf.orig\n+++ /etc/systemd/system/kubelet.service.d/container-runtime.conf\n@@ -0,0 +1,3 @@\n+[Unit]\n+After=containerd.service\n+Requires=containerd.service", "parameters": "--- File[/etc/systemd/system/kubelet.service.d/container-runtime.conf].orig\n+++ File[/etc/systemd/system/kubelet.service.d/container-runtime.conf]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[systemd daemon-reload for kubelet.service (kubelet-container-runtime)]\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__kubelet_server-key.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__kubelet_server-key.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__kubelet_server-key.pem]\n\n+    show_diff => False\n+    backup    => False\n+    group     => root\n+    mode      => 0440\n+    ensure    => file\n+    owner     => kube\n"}, {"resource": "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chained.pem]", "parameters": "--- File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chained.pem].orig\n+++ File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chained.pem]\n\n+    group   => root\n+    ensure  => file\n+    require => Exec[create chained cert /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]\n+    owner   => root\n"}, {"resource": "Systemd::Unit[kubelet-container-runtime]", "parameters": "--- Systemd::Unit[kubelet-container-runtime].orig\n+++ Systemd::Unit[kubelet-container-runtime]\n\n+    override_filename => container-runtime\n+    override          => True\n+    restart           => True\n+    require           => ['Class[Systemd]']\n+    unit              => kubelet\n+    ensure            => present\n"}, {"resource": "Docker::Credentials[/var/lib/kubelet/config.json]", "parameters": "--- Docker::Credentials[/var/lib/kubelet/config.json].orig\n+++ Docker::Credentials[/var/lib/kubelet/config.json]\n\n+    registry_username => kubernetes\n+    group             => root\n+    allow_group       => True\n+    registry          => docker-registry.discovery.wmnet\n+    registry_password => somepassword2\n+    owner             => root\n"}, {"resource": "Exec[systemd daemon-reload for amd-k8s-node-labeller.service (amd-k8s-node-labeller-amd-devplugin-after-labeller)]", "parameters": "--- Exec[systemd daemon-reload for amd-k8s-node-labeller.service (amd-k8s-node-labeller-amd-devplugin-after-labeller)].orig\n+++ Exec[systemd daemon-reload for amd-k8s-node-labeller.service (amd-k8s-node-labeller-amd-devplugin-after-labeller)]\n\n+    command     => /bin/systemctl daemon-reload\n+    refreshonly => True\n"}, {"resource": "Class[Prometheus::Node_exporter]", "parameters": "--- Class[Prometheus::Node_exporter].orig\n+++ Class[Prometheus::Node_exporter]\n\n@@\n-    collectors_extra => []\n+    collectors_extra => ['processes']\n"}, {"resource": "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet]", "parameters": "--- Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet].orig\n+++ Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Cfssl::Csr[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]\n+    unless      => /usr/bin/test \"$(/usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem -noout -pubkey 2>&1)\" == \"$(/usr/bin/openssl pkey -pubout -in /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet-key.pem 2>&1)\"\n\n+    notify      => ['Service[kubelet]']\n"}, {"resource": "Systemd::Service[rsyslog-imfile-remedy]", "parameters": "--- Systemd::Service[rsyslog-imfile-remedy].orig\n+++ Systemd::Service[rsyslog-imfile-remedy]\n\n+    monitoring_enabled       => False\n+    override                 => False\n+    monitoring_critical      => False\n+    restart                  => False\n+    require                  => Systemd::Unit[rsyslog-imfile-remedy.service]\n+    monitoring_contact_group => admins\n+    service_params           => {}\n+    migration_task           => T407130\n+    ensure                   => present\n+    unit_type                => timer\n"}, {"resource": "Ferm::Service[kubelet-http]", "parameters": "--- Ferm::Service[kubelet-http].orig\n+++ Ferm::Service[kubelet-http]\n\n+    prio                => 10\n+    srange              => (@resolve((ml-serve-ctrl1001.eqiad.wmnet ml-serve-ctrl1002.eqiad.wmnet)) @resolve((ml-serve-ctrl1001.eqiad.wmnet ml-serve-ctrl1002.eqiad.wmnet), AAAA))\n+    notrack             => False\n+    desc                => \n+    port                => 10250\n+    proto               => tcp\n+    ensure              => present\n+    unrestricted_access => False\n"}, {"resource": "Exec[Generate cert mlserve__calico-cni refresh]", "parameters": "--- Exec[Generate cert mlserve__calico-cni refresh].orig\n+++ Exec[Generate cert mlserve__calico-cni refresh]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__calico-cni.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__calico-cni\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/csr/mlserve__calico-cni.csr]\n"}, {"resource": "Class[Base::Sysctl::Inotify]", "parameters": "--- Class[Base::Sysctl::Inotify].orig\n+++ Class[Base::Sysctl::Inotify]\n\n+    max_user_watches   => 32768\n+    max_user_instances => 512\n"}, {"resource": "Systemd::Timer::Job[rsyslog-imfile-remedy]", "parameters": "--- Systemd::Timer::Job[rsyslog-imfile-remedy].orig\n+++ Systemd::Timer::Job[rsyslog-imfile-remedy]\n\n+    monitoring_contact_groups => admins\n+    monitoring_notes_url      => https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+    logging_enabled           => False\n+    logfile_basedir           => /var/log\n+    logfile_name              => syslog.log\n+    description               => Restart rsyslog T357616\n+    syslog_match_startswith   => True\n+    logfile_perms             => all\n+    syslog_force_stop         => True\n+    logfile_group             => root\n+    send_mail_to              => root@ml-serve1015.eqiad.wmnet\n+    monitoring_enabled        => False\n+    user                      => root\n+    success_exit_status       => []\n+    send_mail_only_on_error   => True\n+    send_mail                 => False\n+    private_tmp               => False\n+    ignore_errors             => False\n+    splay                     => 30\n+    command                   => /usr/bin/systemctl try-restart rsyslog\n+    environment               => {}\n+    interval                  => {'start': 'OnCalendar', 'interval': '*-*-* 00/3:41:00'}\n+    ensure                    => present\n+    fixed_random_delay        => False\n"}, {"resource": "File[/var/lib/kubelet/config.json]", "content": "--- /var/lib/kubelet/config.json.orig\n+++ /var/lib/kubelet/config.json\n@@ -0,0 +1,7 @@\n+{\n+  \"auths\": {\n+    \"https://docker-registry.discovery.wmnet\": {\n+      \"auth\": \"a3ViZXJuZXRlczpzb21lcGFzc3dvcmQy\"\n+    }\n+  }\n+}", "parameters": "--- File[/var/lib/kubelet/config.json].orig\n+++ File[/var/lib/kubelet/config.json]\n\n+    show_diff => False\n+    group     => root\n+    mode      => 0440\n+    ensure    => present\n+    owner     => root\n"}, {"resource": "Systemd::Unit[rsyslog-release-deleted-inotify-watches.service]", "parameters": "--- Systemd::Unit[rsyslog-release-deleted-inotify-watches.service].orig\n+++ Systemd::Unit[rsyslog-release-deleted-inotify-watches.service]\n\n+    override_filename => puppet-override.conf\n+    override          => False\n+    restart           => False\n+    require           => ['Class[Systemd]']\n+    unit              => rsyslog-release-deleted-inotify-watches.service\n+    ensure            => absent\n"}, {"resource": "File[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]", "content": "--- /etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr.orig\n+++ /etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr\n@@ -0,0 +1,13 @@\n+{\n+  \"CN\": \"amdgpu-node-labeller\",\n+  \"hosts\": [\n+    \"amdgpu-node-labeller\"\n+  ],\n+  \"key\": {\n+    \"algo\": \"ecdsa\",\n+    \"size\": 256\n+  },\n+  \"names\": [\n+\n+  ]\n+}", "parameters": "--- File[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr].orig\n+++ File[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Cfssl::Cert[mlserve__amdgpu-node-labeller]", "parameters": "--- Cfssl::Cert[mlserve__amdgpu-node-labeller].orig\n+++ Cfssl::Cert[mlserve__amdgpu-node-labeller]\n\n+    label           => mlserve\n+    notify_services => []\n+    common_name     => amdgpu-node-labeller\n+    outdir          => /etc/kubernetes/pki\n+    provide_chain   => True\n+    key             => {'algo': 'ecdsa', 'size': 256}\n+    group           => amd-nodelabeller\n+    renew_seconds   => 952200\n+    mode            => 0740\n+    owner           => amd-nodelabeller\n+    before_services => []\n+    hosts           => []\n+    environment     => ['GODEBUG=x509ignoreCN=0']\n+    ensure          => present\n+    names           => []\n+    auto_renew      => True\n"}, {"resource": "File[/etc/containerd]", "parameters": "--- File[/etc/containerd].orig\n+++ File[/etc/containerd]\n\n+    mode   => 0755\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "content": "--- component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.orig\n+++ component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia\n@@ -0,0 +1,5 @@\n+Types: deb deb-src\n+URIs: http://apt.wikimedia.org/wikimedia\n+Suites: trixie-wikimedia\n+Components: component/istio115\n+Signed-By: /etc/apt/keyrings/wikimedia-archive-keyring.gpg", "parameters": "--- Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    tag    => _etc_apt_sources.list.d_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    target => /etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 10\n"}, {"resource": "Class[Lvs::Realserver]", "parameters": "--- Class[Lvs::Realserver].orig\n+++ Class[Lvs::Realserver]\n\n+    realserver_ips => ['10.2.2.63', '10.2.2.84']\n"}, {"resource": "Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "parameters": "--- Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    target => /etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 10\n"}, {"resource": "Cfssl::Cert[mlserve__calicoctl]", "parameters": "--- Cfssl::Cert[mlserve__calicoctl].orig\n+++ Cfssl::Cert[mlserve__calicoctl]\n\n+    label           => mlserve\n+    notify_services => []\n+    common_name     => calicoctl\n+    outdir          => /etc/kubernetes/pki\n+    provide_chain   => True\n+    key             => {'algo': 'ecdsa', 'size': 256}\n+    group           => root\n+    renew_seconds   => 952200\n+    mode            => 0740\n+    owner           => root\n+    before_services => []\n+    hosts           => []\n+    environment     => ['GODEBUG=x509ignoreCN=0']\n+    ensure          => present\n+    names           => []\n+    auto_renew      => True\n"}, {"resource": "Class[Profile::Containerd]", "parameters": "--- Class[Profile::Containerd].orig\n+++ Class[Profile::Containerd]\n\n+    registry_username       => kubernetes\n+    kubernetes_cluster_name => ml-serve-eqiad\n+    ensure                  => present\n"}, {"resource": "File[/lib/systemd/system/cpupower.service]", "content": "--- /lib/systemd/system/cpupower.service.orig\n+++ /lib/systemd/system/cpupower.service\n@@ -0,0 +1,15 @@\n+# SPDX-License-Identifier: Apache-2.0\n+# Adapted from:\n+# https://sources.debian.org/src/linux/6.16.3-1/tools/power/cpupower/cpupower.service.in\n+\n+[Unit]\n+Description=Apply cpupower configuration\n+\n+[Service]\n+Type=oneshot\n+EnvironmentFile=-/etc/default/cpupower\n+ExecStart=/usr/libexec/cpupower\n+RemainAfterExit=yes\n+\n+[Install]\n+WantedBy=multi-user.target", "parameters": "--- File[/lib/systemd/system/cpupower.service].orig\n+++ File[/lib/systemd/system/cpupower.service]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[systemd daemon-reload for cpupower.service (cpupower)]\n+    owner  => root\n"}, {"resource": "Systemd::Unit[cpupower]", "parameters": "--- Systemd::Unit[cpupower].orig\n+++ Systemd::Unit[cpupower]\n\n+    override_filename => puppet-override.conf\n+    override          => False\n+    restart           => True\n+    require           => ['Class[Systemd]']\n+    unit              => cpupower\n+    ensure            => present\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chained.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chained.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chained.pem]\n\n+    group   => amd-nodelabeller\n+    ensure  => file\n+    require => Exec[create chained cert /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]\n+    owner   => amd-nodelabeller\n"}, {"resource": "Sysctl::Conffile[ipv6-fowarding-accept-ra]", "parameters": "--- Sysctl::Conffile[ipv6-fowarding-accept-ra].orig\n+++ Sysctl::Conffile[ipv6-fowarding-accept-ra]\n\n+    priority => 70\n+    ensure   => present\n"}, {"resource": "Exec[create chained cert /etc/kubernetes/pki/mlserve__calico-cni.chain.pem]", "parameters": "--- Exec[create chained cert /etc/kubernetes/pki/mlserve__calico-cni.chain.pem].orig\n+++ Exec[create chained cert /etc/kubernetes/pki/mlserve__calico-cni.chain.pem]\n\n+    command   => /bin/cat /etc/kubernetes/pki/mlserve__calico-cni.pem /etc/kubernetes/pki/mlserve__calico-cni.chain.pem > /etc/kubernetes/pki/mlserve__calico-cni.chained.pem\n+    unless    => /usr/bin/test \"$(/bin/cat /etc/kubernetes/pki/mlserve__calico-cni.pem /etc/kubernetes/pki/mlserve__calico-cni.chain.pem | sha512sum)\" == \"$(/bin/cat /etc/kubernetes/pki/mlserve__calico-cni.chained.pem | sha512sum)\"\n\n+    subscribe => ['Exec[renew certificate - mlserve__calico-cni]', 'File[/etc/kubernetes/pki/mlserve__calico-cni.chain.pem]', 'File[/etc/kubernetes/pki/mlserve__calico-cni.pem]']\n+    require   => Exec[Generate cert mlserve__calico-cni refresh on intermediate ca change]\n"}, {"resource": "Concat_fragment[main contacts]", "content": "--- main contacts.orig\n+++ main contacts\n@@ -1,3 +1,3 @@\n ---\n-role::ml_k8s::insetup_gpu:\n+role::ml_k8s::worker:\n - Machine Learning"}, {"resource": "Exec[create chained cert /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]", "parameters": "--- Exec[create chained cert /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem].orig\n+++ Exec[create chained cert /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]\n\n+    command   => /bin/cat /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem > /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chained.pem\n+    subscribe => ['Exec[renew certificate - mlserve__system_node_ml-serve1015_eqiad_wmnet]', 'File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]', 'File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem]']\n+    require   => Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh on intermediate ca change]\n+    unless    => /usr/bin/test \"$(/bin/cat /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem | sha512sum)\" == \"$(/bin/cat /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chained.pem | sha512sum)\"\n\n+    notify    => ['Service[kubelet]']\n"}, {"resource": "Systemd::Timer::Job[rsyslog-release-deleted-inotify-watches]", "parameters": "--- Systemd::Timer::Job[rsyslog-release-deleted-inotify-watches].orig\n+++ Systemd::Timer::Job[rsyslog-release-deleted-inotify-watches]\n\n+    monitoring_contact_groups => admins\n+    monitoring_notes_url      => https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+    logging_enabled           => True\n+    logfile_basedir           => /var/log\n+    logfile_name              => syslog.log\n+    description               => Restart rsyslog to release inotify watches of deleted container logs\n+    syslog_match_startswith   => True\n+    logfile_perms             => all\n+    syslog_force_stop         => True\n+    logfile_group             => root\n+    send_mail_to              => root@ml-serve1015.eqiad.wmnet\n+    monitoring_enabled        => False\n+    user                      => root\n+    success_exit_status       => []\n+    send_mail_only_on_error   => True\n+    send_mail                 => False\n+    private_tmp               => False\n+    ignore_errors             => False\n+    command                   => /usr/local/sbin/rsyslog-release-deleted-inotify-watches\n+    environment               => {}\n+    interval                  => {'start': 'OnCalendar', 'interval': '*-*-* *:54:00'}\n+    ensure                    => absent\n+    fixed_random_delay        => False\n"}, {"resource": "Cfssl::Cert[mlserve__system_kube-proxy]", "parameters": "--- Cfssl::Cert[mlserve__system_kube-proxy].orig\n+++ Cfssl::Cert[mlserve__system_kube-proxy]\n\n+    label           => mlserve\n+    notify_services => ['kube-proxy']\n+    common_name     => system:kube-proxy\n+    outdir          => /etc/kubernetes/pki\n+    provide_chain   => True\n+    key             => {'algo': 'ecdsa', 'size': 256}\n+    group           => root\n+    renew_seconds   => 952200\n+    mode            => 0740\n+    owner           => kube\n+    before_services => []\n+    hosts           => []\n+    environment     => ['GODEBUG=x509ignoreCN=0']\n+    ensure          => present\n+    names           => [{'organisation': 'system:node-proxier'}]\n+    auto_renew      => True\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => kube\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.csr]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_kube-proxy.csr].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_kube-proxy.csr]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => kube\n"}, {"resource": "Exec[renew certificate - discovery__ml-serve1015_eqiad_wmnet]", "parameters": "--- Exec[renew certificate - discovery__ml-serve1015_eqiad_wmnet].orig\n+++ Exec[renew certificate - discovery__ml-serve1015_eqiad_wmnet]\n\n+    command     => /usr/bin/cfssl sign -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label discovery  /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.csr | /usr/bin/cfssljson -bare /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet]\n+    unless      => /usr/bin/openssl x509 -in /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem -checkend 952200\n+    notify      => ['Service[dragonfly-dfdaemon]']\n"}, {"resource": "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh]", "parameters": "--- Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh].orig\n+++ Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]\n+    notify      => ['Service[kubelet]']\n"}, {"resource": "Exec[Generate cert mlserve__rsyslog]", "parameters": "--- Exec[Generate cert mlserve__rsyslog].orig\n+++ Exec[Generate cert mlserve__rsyslog]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__rsyslog.csr | /usr/bin/cfssljson -bare /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Cfssl::Csr[/etc/cfssl/csr/mlserve__rsyslog.csr]\n+    unless      => /usr/bin/test \"$(/usr/bin/openssl x509 -in /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem -noout -pubkey 2>&1)\" == \"$(/usr/bin/openssl pkey -pubout -in /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog-key.pem 2>&1)\"\n\n+    notify      => ['Service[rsyslog]']\n"}, {"resource": "File[/etc/cfssl/csr/mlserve__calico-cni.csr]", "content": "--- /etc/cfssl/csr/mlserve__calico-cni.csr.orig\n+++ /etc/cfssl/csr/mlserve__calico-cni.csr\n@@ -0,0 +1,13 @@\n+{\n+  \"CN\": \"calico-cni\",\n+  \"hosts\": [\n+    \"calico-cni\"\n+  ],\n+  \"key\": {\n+    \"algo\": \"ecdsa\",\n+    \"size\": 256\n+  },\n+  \"names\": [\n+\n+  ]\n+}", "parameters": "--- File[/etc/cfssl/csr/mlserve__calico-cni.csr].orig\n+++ File[/etc/cfssl/csr/mlserve__calico-cni.csr]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Firewall::Service[dragonfly_dfget]", "parameters": "--- Firewall::Service[dragonfly_dfget].orig\n+++ Firewall::Service[dragonfly_dfget]\n\n+    prio                => 10\n+    notrack             => False\n+    src_sets            => ['DOMAIN_NETWORKS']\n+    desc                => \n+    port                => 15001\n+    proto               => tcp\n+    ensure              => present\n+    unrestricted_access => False\n"}, {"resource": "Exec[apt_repository_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "parameters": "--- Exec[apt_repository_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Exec[apt_repository_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    command     => /usr/bin/apt-get update \n+    refreshonly => True\n"}, {"resource": "File[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "content": "--- /etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr.orig\n+++ /etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr\n@@ -0,0 +1,19 @@\n+{\n+  \"CN\": \"system:node:ml-serve1015.eqiad.wmnet\",\n+  \"hosts\": [\n+    \"system:node:ml-serve1015.eqiad.wmnet\"\n+  ],\n+  \"key\": {\n+    \"algo\": \"ecdsa\",\n+    \"size\": 256\n+  },\n+  \"names\": [\n+    {\n+      \"C\": null,\n+      \"L\": null,\n+      \"O\": \"system:nodes\",\n+      \"OU\": null,\n+      \"S\": null\n+    }\n+  ]\n+}", "parameters": "--- File[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr].orig\n+++ File[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Concat[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "parameters": "--- Concat[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources].orig\n+++ Concat[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]\n\n+    show_diff      => True\n+    format         => plain\n+    force          => False\n+    group          => root\n+    mode           => 0444\n+    order          => alpha\n+    owner          => root\n+    backup         => puppet\n+    replace        => True\n+    warn           => False\n+    ensure_newline => False\n+    ensure         => present\n+    notify         => Exec[apt_repository_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n+    path           => /etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n"}, {"resource": "Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "parameters": "--- Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header].orig\n+++ Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]\n\n+    source => puppet:///modules/apt/sources-deb822-header.txt\n+    tag    => _etc_apt_sources.list.d_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    target => /etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 01\n"}, {"resource": "File[/etc/cfssl/ssl/mlserve__rsyslog]", "parameters": "--- File[/etc/cfssl/ssl/mlserve__rsyslog].orig\n+++ File[/etc/cfssl/ssl/mlserve__rsyslog]\n\n+    group   => root\n+    recurse => True\n+    mode    => 0740\n+    ensure  => directory\n+    owner   => root\n"}, {"resource": "Rsyslog::Conf[imfile]", "parameters": "--- Rsyslog::Conf[imfile].orig\n+++ Rsyslog::Conf[imfile]\n\n+    priority => 0\n+    mode     => 0444\n+    ensure   => present\n"}, {"resource": "Exec[apt_package_from_component_istio115]", "parameters": "--- Exec[apt_package_from_component_istio115].orig\n+++ Exec[apt_package_from_component_istio115]\n\n+    before      => ['Package[istio-cni]']\n+    command     => /usr/bin/apt-get update\n+    refreshonly => True\n+    subscribe   => Apt::Repository[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n"}, {"resource": "K8s::Kubeconfig[/etc/amd/node-labeller-kubeconfig]", "parameters": "--- K8s::Kubeconfig[/etc/amd/node-labeller-kubeconfig].orig\n+++ K8s::Kubeconfig[/etc/amd/node-labeller-kubeconfig]\n\n+    username    => amdgpu-node-labeller\n+    group       => amd-nodelabeller\n+    mode        => 0400\n+    owner       => amd-nodelabeller\n+    require     => ['File[/etc/amd]', 'Class[K8s::Base_dirs]']\n+    auth_cert   => {'cert': '/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem', 'key': '/etc/kubernetes/pki/mlserve__amdgpu-node-labeller-key.pem', 'chain': '/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem', 'chained': '/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chained.pem'}\n+    ensure      => present\n+    master_host => ml-ctrl.svc.eqiad.wmnet\n"}, {"resource": "Exec[Generate cert mlserve__calicoctl]", "parameters": "--- Exec[Generate cert mlserve__calicoctl].orig\n+++ Exec[Generate cert mlserve__calicoctl]\n\n+    unless      => /usr/bin/test \"$(/usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__calicoctl.pem -noout -pubkey 2>&1)\" == \"$(/usr/bin/openssl pkey -pubout -in /etc/kubernetes/pki/mlserve__calicoctl-key.pem 2>&1)\"\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__calicoctl.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__calicoctl\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Cfssl::Csr[/etc/cfssl/csr/mlserve__calicoctl.csr]\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller-key.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller-key.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller-key.pem]\n\n+    show_diff => False\n+    backup    => False\n+    group     => amd-nodelabeller\n+    mode      => 0440\n+    ensure    => file\n+    owner     => amd-nodelabeller\n"}, {"resource": "File[/etc/kubernetes/kubelet.conf]", "content": "--- /etc/kubernetes/kubelet.conf.orig\n+++ /etc/kubernetes/kubelet.conf\n@@ -0,0 +1,18 @@\n+apiVersion: v1\n+kind: Config\n+preferences: {}\n+current-context: default-system\n+contexts:\n+- name: default-system\n+  context:\n+    cluster: default-cluster\n+    user: default-auth\n+clusters:\n+- name: default-cluster\n+  cluster:\n+    server: https://ml-ctrl.svc.eqiad.wmnet:6443\n+users:\n+- name: default-auth\n+  user:\n+    client-certificate: /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem\n+    client-key: /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet-key.pem", "parameters": "--- File[/etc/kubernetes/kubelet.conf].orig\n+++ File[/etc/kubernetes/kubelet.conf]\n\n+    mode   => 0400\n+    group  => kube\n+    ensure => present\n+    owner  => kube\n"}, {"resource": "File[/etc/modules-load.d/overlay.conf]", "content": "--- /etc/modules-load.d/overlay.conf.orig\n+++ /etc/modules-load.d/overlay.conf\n@@ -0,0 +1 @@\n+overlay", "parameters": "--- File[/etc/modules-load.d/overlay.conf].orig\n+++ File[/etc/modules-load.d/overlay.conf]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[/sbin/modprobe overlay]\n+    owner  => root\n"}, {"resource": "Sysctl::Parameters[ipv6-fowarding-accept-ra]", "parameters": "--- Sysctl::Parameters[ipv6-fowarding-accept-ra].orig\n+++ Sysctl::Parameters[ipv6-fowarding-accept-ra]\n\n+    priority => 70\n+    values   => {'net.ipv6.conf.all.forwarding': 1, 'net.ipv6.conf.ens11f1np1.accept_ra': 2}\n+    ensure   => present\n"}, {"resource": "Sysctl::Parameters[increase_inotify_limits]", "parameters": "--- Sysctl::Parameters[increase_inotify_limits].orig\n+++ Sysctl::Parameters[increase_inotify_limits]\n\n+    priority => 70\n+    values   => {'fs.inotify.max_user_watches': 32768, 'fs.inotify.max_user_instances': 512}\n+    ensure   => present\n"}, {"resource": "Exec[/usr/sbin/dpkg-reconfigure -p critical -f noninteractive wikimedia-lvs-realserver]", "parameters": "--- Exec[/usr/sbin/dpkg-reconfigure -p critical -f noninteractive wikimedia-lvs-realserver].orig\n+++ Exec[/usr/sbin/dpkg-reconfigure -p critical -f noninteractive wikimedia-lvs-realserver]\n\n+    subscribe   => File[/etc/default/wikimedia-lvs-realserver]\n+    refreshonly => True\n+    require     => Package[wikimedia-lvs-realserver]\n+    path        => /bin:/sbin:/usr/bin:/usr/sbin\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calicoctl.chain.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calicoctl.chain.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__calicoctl.chain.pem]\n\n+    group  => root\n+    source => puppet:///modules/profile/pki/intermediates/mlserve-cert.pem\n+    mode   => 0440\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Cfssl::Csr[/etc/cfssl/csr/mlserve__calicoctl.csr]", "parameters": "--- Cfssl::Csr[/etc/cfssl/csr/mlserve__calicoctl.csr].orig\n+++ Cfssl::Csr[/etc/cfssl/csr/mlserve__calicoctl.csr]\n\n+    hosts       => []\n+    key         => {'algo': 'ecdsa', 'size': 256}\n+    common_name => calicoctl\n+    ensure      => present\n+    names       => []\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calicoctl.csr]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calicoctl.csr].orig\n+++ File[/etc/kubernetes/pki/mlserve__calicoctl.csr]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "File[/etc/cfssl/csr/mlserve__rsyslog.csr]", "content": "--- /etc/cfssl/csr/mlserve__rsyslog.csr.orig\n+++ /etc/cfssl/csr/mlserve__rsyslog.csr\n@@ -0,0 +1,19 @@\n+{\n+  \"CN\": \"rsyslog\",\n+  \"hosts\": [\n+    \"rsyslog\"\n+  ],\n+  \"key\": {\n+    \"algo\": \"ecdsa\",\n+    \"size\": 256\n+  },\n+  \"names\": [\n+    {\n+      \"C\": null,\n+      \"L\": null,\n+      \"O\": \"view\",\n+      \"OU\": null,\n+      \"S\": null\n+    }\n+  ]\n+}", "parameters": "--- File[/etc/cfssl/csr/mlserve__rsyslog.csr].orig\n+++ File[/etc/cfssl/csr/mlserve__rsyslog.csr]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Udev::Rule[kube_proxy_conntrack]", "parameters": "--- Udev::Rule[kube_proxy_conntrack].orig\n+++ Udev::Rule[kube_proxy_conntrack]\n\n+    priority => 75\n+    ensure   => present\n"}, {"resource": "File[/etc/sysctl.d/70-increase_inotify_limits.conf]", "content": "--- /etc/sysctl.d/70-increase_inotify_limits.conf.orig\n+++ /etc/sysctl.d/70-increase_inotify_limits.conf\n@@ -0,0 +1,3 @@\n+# sysctl parameters managed by Puppet.\n+fs.inotify.max_user_instances = 512\n+fs.inotify.max_user_watches = 32768", "parameters": "--- File[/etc/sysctl.d/70-increase_inotify_limits.conf].orig\n+++ File[/etc/sysctl.d/70-increase_inotify_limits.conf]\n\n+    group  => root\n+    ensure => present\n+    notify => Exec[update_sysctl]\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/kube-proxy-config.yaml]", "content": "--- /etc/kubernetes/kube-proxy-config.yaml.orig\n+++ /etc/kubernetes/kube-proxy-config.yaml\n@@ -0,0 +1,12 @@\n+---\n+apiVersion: kubeproxy.config.k8s.io/v1alpha1\n+kind: KubeProxyConfiguration\n+hostnameOverride: ml-serve1015.eqiad.wmnet\n+clientConnection:\n+  kubeconfig: \"/etc/kubernetes/proxy.conf\"\n+clusterCIDR: 10.67.16.0/21\n+mode: iptables\n+metricsBindAddress: 0.0.0.0\n+nodePortAddresses:\n+- 0.0.0.0/0\n+- \"::/0\"", "parameters": "--- File[/etc/kubernetes/kube-proxy-config.yaml].orig\n+++ File[/etc/kubernetes/kube-proxy-config.yaml]\n\n+    require => K8s::Package[proxy]\n+    group   => kube\n+    mode    => 0400\n+    ensure  => file\n+    notify  => Service[kube-proxy]\n+    owner   => kube\n"}, {"resource": "Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "parameters": "--- Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header].orig\n+++ Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]\n\n+    source => puppet:///modules/apt/sources-deb822-header.txt\n+    target => /etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 01\n"}, {"resource": "File[/etc/rsyslog.d/40-rsyslog-release-deleted-inotify-watches.conf]", "content": "--- /etc/rsyslog.d/40-rsyslog-release-deleted-inotify-watches.conf.orig\n+++ /etc/rsyslog.d/40-rsyslog-release-deleted-inotify-watches.conf\n@@ -0,0 +1,10 @@\n+# rsyslog.conf(5) configuration file for services.\n+# This file is managed by Puppet.\n+if $programname startswith \"rsyslog-release-deleted-inotify-watches\" then {\n+    action(\n+        type=\"omfile\" file=\"/var/log/rsyslog-release-deleted-inotify-watches/syslog.log\"\n+        fileOwner=\"root\" fileGroup=\"root\"\n+        fileCreateMode=\"0644\"\n+    )\n+    & stop\n+}", "parameters": "--- File[/etc/rsyslog.d/40-rsyslog-release-deleted-inotify-watches.conf].orig\n+++ File[/etc/rsyslog.d/40-rsyslog-release-deleted-inotify-watches.conf]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    notify => Service[rsyslog]\n+    owner  => root\n"}, {"resource": "Motd::Message[ml_k8s::insetup_gpu]", "parameters": "--- Motd::Message[ml_k8s::insetup_gpu].orig\n+++ Motd::Message[ml_k8s::insetup_gpu]\n\n-    priority => 5\n-    message  => ml-serve1015 is a Machine Learning GPU host in setup. (ml_k8s::insetup_gpu)\n-    ensure   => present\n"}, {"resource": "Class[K8s::Kubelet]", "parameters": "--- Class[K8s::Kubelet].orig\n+++ Class[K8s::Kubelet]\n\n+    cluster_dns                     => ['10.67.0.3']\n+    node_taints                     => []\n+    docker_kubernetes_user_password => somepassword2\n+    v_log_level                     => 0\n+    node_labels                     => ['topology.kubernetes.io/region=eqiad', 'topology.kubernetes.io/zone=row-e12', 'node.kubernetes.io/disk-type=ssd']\n+    pod_infra_container_image       => docker-registry.discovery.wmnet/pause:3.6-1\n+    kubelet_cert                    => {'cert': '/etc/kubernetes/pki/mlserve__kubelet_server.pem', 'key': '/etc/kubernetes/pki/mlserve__kubelet_server-key.pem', 'chain': '/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem', 'chained': '/etc/kubernetes/pki/mlserve__kubelet_server.chained.pem'}\n+    cluster_domain                  => cluster.local\n+    cni_bin_dir                     => /opt/cni/bin\n+    cni_conf_dir                    => /etc/cni/net.d\n+    version                         => 1.31\n+    ipv6dualstack                   => False\n+    system_reserved                 => {'cpu': '45.3', 'memory': '50.83Gi'}\n+    kubeconfig                      => /etc/kubernetes/kubelet.conf\n"}, {"resource": "Systemd::Unit[rsyslog-release-deleted-inotify-watches.timer]", "parameters": "--- Systemd::Unit[rsyslog-release-deleted-inotify-watches.timer].orig\n+++ Systemd::Unit[rsyslog-release-deleted-inotify-watches.timer]\n\n+    override_filename => puppet-override.conf\n+    override          => False\n+    restart           => False\n+    require           => ['Class[Systemd]']\n+    unit              => rsyslog-release-deleted-inotify-watches.timer\n+    ensure            => absent\n"}, {"resource": "Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "content": "--- component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.orig\n+++ component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia\n@@ -0,0 +1,5 @@\n+Types: deb deb-src\n+URIs: http://apt.wikimedia.org/wikimedia\n+Suites: trixie-wikimedia\n+Components: component/calico329\n+Signed-By: /etc/apt/keyrings/wikimedia-archive-keyring.gpg", "parameters": "--- Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    tag    => _etc_apt_sources.list.d_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    target => /etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 10\n"}, {"resource": "File[/etc/nerdctl]", "parameters": "--- File[/etc/nerdctl].orig\n+++ File[/etc/nerdctl]\n\n+    mode   => 0755\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "Package[amd-k8s-device-plugin]", "parameters": "--- Package[amd-k8s-device-plugin].orig\n+++ Package[amd-k8s-device-plugin]\n\n+    ensure   => present\n+    provider => apt\n"}, {"resource": "Exec[exec-apt-get-update-linux-6.16-trixie_trixie-bpo]", "parameters": "--- Exec[exec-apt-get-update-linux-6.16-trixie_trixie-bpo].orig\n+++ Exec[exec-apt-get-update-linux-6.16-trixie_trixie-bpo]\n\n+    command     => /usr/bin/apt-get update\n+    refreshonly => True\n"}, {"resource": "Exec[Generate cert mlserve__istio-cni refresh]", "parameters": "--- Exec[Generate cert mlserve__istio-cni refresh].orig\n+++ Exec[Generate cert mlserve__istio-cni refresh]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__istio-cni.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__istio-cni\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/csr/mlserve__istio-cni.csr]\n"}, {"resource": "Exec[systemd daemon-reload for cpupower.service (cpupower)]", "parameters": "--- Exec[systemd daemon-reload for cpupower.service (cpupower)].orig\n+++ Exec[systemd daemon-reload for cpupower.service (cpupower)]\n\n+    command     => /bin/systemctl daemon-reload\n+    notify      => ['Service[cpupower]']\n+    refreshonly => True\n"}, {"resource": "Exec[Generate cert mlserve__system_kube-proxy]", "parameters": "--- Exec[Generate cert mlserve__system_kube-proxy].orig\n+++ Exec[Generate cert mlserve__system_kube-proxy]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__system_kube-proxy.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__system_kube-proxy\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Cfssl::Csr[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]\n+    unless      => /usr/bin/test \"$(/usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__system_kube-proxy.pem -noout -pubkey 2>&1)\" == \"$(/usr/bin/openssl pkey -pubout -in /etc/kubernetes/pki/mlserve__system_kube-proxy-key.pem 2>&1)\"\n\n+    notify      => ['Service[kube-proxy]']\n"}, {"resource": "File[/etc/cni/net.d/10-calico.conflist]", "content": "--- /etc/cni/net.d/10-calico.conflist.orig\n+++ /etc/cni/net.d/10-calico.conflist\n@@ -0,0 +1,39 @@\n+{\n+  \"name\": \"k8s-pod-network\",\n+  \"cniVersion\": \"0.3.1\",\n+  \"plugins\": [\n+    {\n+      \"type\": \"calico\",\n+      \"log_level\": \"info\",\n+      \"datastore_type\": \"kubernetes\",\n+      \"mtu\": 1460,\n+      \"ipam\": {\n+        \"type\": \"calico-ipam\",\n+        \"assign_ipv4\": \"true\",\n+        \"assign_ipv6\": \"true\"\n+      },\n+      \"policy\": {\n+        \"type\": \"k8s\"\n+      },\n+      \"kubernetes\": {\n+        \"kubeconfig\": \"/etc/cni/net.d/calico-kubeconfig\"\n+      }\n+    },\n+    {\n+      \"name\": \"istio-cni\",\n+      \"type\": \"istio-cni\",\n+      \"log_level\": \"info\",\n+      \"kubernetes\": {\n+        \"kubeconfig\": \"/etc/cni/net.d/istio-kubeconfig\",\n+        \"cni_bin_dir\": \"/opt/cni/bin\",\n+        \"exclude_namespaces\": [\n+          \"istio-system\",\n+          \"kube-system\",\n+          \"knative-serving\",\n+          \"cert-manager\",\n+          \"kserve\"\n+        ]\n+      }\n+    }\n+  ]\n+}", "parameters": "--- File[/etc/cni/net.d/10-calico.conflist].orig\n+++ File[/etc/cni/net.d/10-calico.conflist]\n\n+    owner => root\n+    mode  => 0755\n+    group => root\n"}, {"resource": "Sysctl::Parameters[kube_proxy_icmp]", "parameters": "--- Sysctl::Parameters[kube_proxy_icmp].orig\n+++ Sysctl::Parameters[kube_proxy_icmp]\n\n+    priority => 75\n+    values   => {'net.ipv4.conf.all.send_redirects': 0, 'net.ipv4.conf.default.send_redirects': 0, 'net.ipv4.conf.ens11f1np1.send_redirects': 0}\n+    ensure   => present\n"}, {"resource": "Package[socat]", "parameters": "--- Package[socat].orig\n+++ Package[socat]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "Apt::Package_from_bpo[firmware-amd-graphics-trixie-bpo]", "parameters": "--- Apt::Package_from_bpo[firmware-amd-graphics-trixie-bpo].orig\n+++ Apt::Package_from_bpo[firmware-amd-graphics-trixie-bpo]\n\n+    packages        => {'firmware-amd-graphics': '20251021-1~bpo13+1'}\n+    priority        => 1001\n+    distro          => trixie\n+    ensure_packages => True\n"}, {"resource": "Systemd::Override[ferm-service-auto-restart]", "parameters": "--- Systemd::Override[ferm-service-auto-restart].orig\n+++ Systemd::Override[ferm-service-auto-restart]\n\n+    source  => puppet:///modules/profile/kubernetes/node/ferm_systemd_override\n+    ensure  => present\n+    restart => False\n+    unit    => ferm\n"}, {"resource": "File[/etc/default/kubelet]", "content": "--- /etc/default/kubelet.orig\n+++ /etc/default/kubelet\n@@ -0,0 +1,11 @@\n+###\n+# kubernetes kubelet (minion) config\n+\n+DAEMON_ARGS=\"--config=/etc/kubernetes/kubelet-config.yaml \\\n+ --hostname-override=ml-serve1015.eqiad.wmnet \\\n+ --kubeconfig=/etc/kubernetes/kubelet.conf \\\n+ --node-ip=10.64.167.6 \\\n+ --node-labels=node.kubernetes.io/disk-type=ssd,topology.kubernetes.io/region=eqiad,topology.kubernetes.io/zone=row-e12 \\\n+ --register-schedulable=false \\\n+ --system-reserved=cpu=45.3,memory=50.83Gi \\\n+ --v=0\"", "parameters": "--- File[/etc/default/kubelet].orig\n+++ File[/etc/default/kubelet]\n\n+    group  => root\n+    mode   => 0644\n+    ensure => file\n+    notify => Service[kubelet]\n+    owner  => root\n"}, {"resource": "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem]", "parameters": "--- File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem].orig\n+++ File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem]\n\n+    show_diff => False\n+    backup    => False\n+    group     => root\n+    mode      => 0440\n+    ensure    => file\n+    owner     => dragonfly\n"}, {"resource": "File[/etc/cni/net.d]", "parameters": "--- File[/etc/cni/net.d].orig\n+++ File[/etc/cni/net.d]\n\n+    mode   => 0755\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "Service[kube-proxy]", "parameters": "--- Service[kube-proxy].orig\n+++ Service[kube-proxy]\n\n+    enable => True\n+    ensure => running\n"}, {"resource": "Sysctl::Conffile[kube_proxy_icmp]", "parameters": "--- Sysctl::Conffile[kube_proxy_icmp].orig\n+++ Sysctl::Conffile[kube_proxy_icmp]\n\n+    priority => 75\n+    ensure   => present\n"}, {"resource": "Concat::Fragment[main contacts]"}, {"resource": "File[/etc/cni]", "parameters": "--- File[/etc/cni].orig\n+++ File[/etc/cni]\n\n+    mode   => 0755\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "Systemd::Service[cpupower]", "parameters": "--- Systemd::Service[cpupower].orig\n+++ Systemd::Service[cpupower]\n\n+    monitoring_enabled       => False\n+    override                 => False\n+    monitoring_critical      => False\n+    restart                  => True\n+    service_params           => {}\n+    monitoring_contact_group => admins\n+    migration_task           => T407130\n+    ensure                   => present\n+    unit_type                => service\n"}, {"resource": "File[/etc/update-motd.d/05-ml-k8s--insetup-gpu]", "content": "--- /etc/update-motd.d/05-ml-k8s--insetup-gpu.orig\n+++ /etc/update-motd.d/05-ml-k8s--insetup-gpu\n@@ -1,2 +0,0 @@\n-#!/bin/sh\n-printf \"%s\\n\" \"ml-serve1015 is a Machine Learning GPU host in setup. (ml_k8s::insetup_gpu)\"", "parameters": "--- File[/etc/update-motd.d/05-ml-k8s--insetup-gpu].orig\n+++ File[/etc/update-motd.d/05-ml-k8s--insetup-gpu]\n\n-    group  => root\n-    mode   => 0555\n-    ensure => present\n-    owner  => root\n"}, {"resource": "Sysctl::Parameters[kube_proxy_conntrack]", "parameters": "--- Sysctl::Parameters[kube_proxy_conntrack].orig\n+++ Sysctl::Parameters[kube_proxy_conntrack]\n\n+    ensure   => present\n+    priority => 75\n+    values   => {'net.netfilter.nf_conntrack_max': 1048576}\n+    module   => nf_conntrack\n"}, {"resource": "Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "parameters": "--- Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    target => /etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 10\n"}, {"resource": "Motd::Script[ml_k8s::worker]", "parameters": "--- Motd::Script[ml_k8s::worker].orig\n+++ Motd::Script[ml_k8s::worker]\n\n+    priority => 5\n+    ensure   => present\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.csr]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.csr].orig\n+++ File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.csr]\n\n+    mode   => 0440\n+    group  => amd-nodelabeller\n+    ensure => file\n+    owner  => amd-nodelabeller\n"}, {"resource": "Motd::Message[ml_k8s::worker]", "parameters": "--- Motd::Message[ml_k8s::worker].orig\n+++ Motd::Message[ml_k8s::worker]\n\n+    priority => 5\n+    message  => ml-serve1015 is a ML Kubernetes worker node (ml_k8s::worker)\n+    ensure   => present\n"}, {"resource": "Class[Profile::Lvs::Realserver]", "parameters": "--- Class[Profile::Lvs::Realserver].orig\n+++ Class[Profile::Lvs::Realserver]\n\n+    use_conftool         => False\n+    poolcounter_backends => [{'label': 'pc1', 'fqdn': 'poolcounter1006.eqiad.wmnet'}, {'label': 'pc2', 'fqdn': 'poolcounter1007.eqiad.wmnet'}]\n+    pools                => {'inference': {}, 'k8s-ingress-ml-serve': {}}\n"}, {"resource": "Class[Calico]", "parameters": "--- Class[Calico].orig\n+++ Class[Calico]\n\n+    master_fqdn        => ml-ctrl.svc.eqiad.wmnet\n+    version            => 3.29\n+    calicoctl_username => calicoctl\n+    auth_cert          => {'cert': '/etc/kubernetes/pki/mlserve__calicoctl.pem', 'key': '/etc/kubernetes/pki/mlserve__calicoctl-key.pem', 'chain': '/etc/kubernetes/pki/mlserve__calicoctl.chain.pem', 'chained': '/etc/kubernetes/pki/mlserve__calicoctl.chained.pem'}\n"}, {"resource": "File[/etc/systemd/system/amd-k8s-node-labeller.service.d]", "parameters": "--- File[/etc/systemd/system/amd-k8s-node-labeller.service.d].orig\n+++ File[/etc/systemd/system/amd-k8s-node-labeller.service.d]\n\n+    mode   => 0555\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "Exec[Generate cert mlserve__kubelet_server refresh]", "parameters": "--- Exec[Generate cert mlserve__kubelet_server refresh].orig\n+++ Exec[Generate cert mlserve__kubelet_server refresh]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve -profile server /etc/cfssl/csr/mlserve__kubelet_server.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__kubelet_server\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/csr/mlserve__kubelet_server.csr]\n+    notify      => ['Service[kubelet]']\n"}, {"resource": "Exec[renew certificate - mlserve__calico-cni]", "parameters": "--- Exec[renew certificate - mlserve__calico-cni].orig\n+++ Exec[renew certificate - mlserve__calico-cni]\n\n+    unless      => /usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__calico-cni.pem -checkend 952200\n+    command     => /usr/bin/cfssl sign -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/kubernetes/pki/mlserve__calico-cni.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__calico-cni\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Exec[Generate cert mlserve__calico-cni]\n"}, {"resource": "File[/etc/udev/rules.d/70-kfd.rules]", "content": "--- /etc/udev/rules.d/70-kfd.rules.orig\n+++ /etc/udev/rules.d/70-kfd.rules\n@@ -0,0 +1 @@\n+SUBSYSTEM==\"kfd\", KERNEL==\"kfd\", MODE=\"0666\"", "parameters": "--- File[/etc/udev/rules.d/70-kfd.rules].orig\n+++ File[/etc/udev/rules.d/70-kfd.rules]\n\n+    group => root\n+    mode  => 0544\n+    owner => root\n"}, {"resource": "Class[Profile::Calico::Kubernetes]", "parameters": "--- Class[Profile::Calico::Kubernetes].orig\n+++ Class[Profile::Calico::Kubernetes]\n\n+    kubernetes_cluster_name => ml-serve-eqiad\n"}, {"resource": "Concat_file[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "parameters": "--- Concat_file[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources].orig\n+++ Concat_file[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]\n\n+    show_diff      => True\n+    tag            => _etc_apt_sources.list.d_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    format         => plain\n+    group          => root\n+    mode           => 0444\n+    order          => alpha\n+    owner          => root\n+    replace        => True\n+    backup         => puppet\n+    ensure_newline => False\n+    force          => False\n"}, {"resource": "File[/etc/modprobe.d/blacklist-wmf_overlay.conf]", "content": "--- /etc/modprobe.d/blacklist-wmf_overlay.conf.orig\n+++ /etc/modprobe.d/blacklist-wmf_overlay.conf\n@@ -1,7 +1,3 @@\n # wmf_overlay - blacklisted kernel modules\n # This file is managed by Puppet\n #\n-blacklist overlay\n-install overlay /bin/true\n-blacklist overlayfs\n-install overlayfs /bin/true", "parameters": "--- File[/etc/modprobe.d/blacklist-wmf_overlay.conf].orig\n+++ File[/etc/modprobe.d/blacklist-wmf_overlay.conf]\n\n@@\n-    ensure => present\n+    ensure => absent\n"}, {"resource": "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem]", "parameters": "--- File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem].orig\n+++ File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem]\n\n+    group   => root\n+    ensure  => file\n+    require => Exec[create chained cert /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]\n+    owner   => dragonfly\n"}, {"resource": "Concat[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "parameters": "--- Concat[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources].orig\n+++ Concat[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]\n\n+    show_diff      => True\n+    format         => plain\n+    force          => False\n+    group          => root\n+    mode           => 0444\n+    order          => alpha\n+    owner          => root\n+    backup         => puppet\n+    replace        => True\n+    warn           => False\n+    ensure_newline => False\n+    ensure         => present\n+    notify         => Exec[apt_repository_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n+    path           => /etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n"}, {"resource": "File[/etc/cfssl/csr/mlserve__kubelet_server.csr]", "content": "--- /etc/cfssl/csr/mlserve__kubelet_server.csr.orig\n+++ /etc/cfssl/csr/mlserve__kubelet_server.csr\n@@ -0,0 +1,17 @@\n+{\n+  \"CN\": \"kubelet\",\n+  \"hosts\": [\n+    \"ml-serve1015\",\n+    \"ml-serve1015.eqiad.wmnet\",\n+    \"10.64.167.6\",\n+    \"2620:0:861:12f:10:64:167:6\",\n+    \"kubelet\"\n+  ],\n+  \"key\": {\n+    \"algo\": \"ecdsa\",\n+    \"size\": 256\n+  },\n+  \"names\": [\n+\n+  ]\n+}", "parameters": "--- File[/etc/cfssl/csr/mlserve__kubelet_server.csr].orig\n+++ File[/etc/cfssl/csr/mlserve__kubelet_server.csr]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Systemd::Unit[rsyslog-imfile-remedy.service]", "parameters": "--- Systemd::Unit[rsyslog-imfile-remedy.service].orig\n+++ Systemd::Unit[rsyslog-imfile-remedy.service]\n\n+    override_filename => puppet-override.conf\n+    override          => False\n+    restart           => False\n+    require           => ['Class[Systemd]']\n+    unit              => rsyslog-imfile-remedy.service\n+    ensure            => present\n"}, {"resource": "Exec[renew certificate - mlserve__amdgpu-node-labeller]", "parameters": "--- Exec[renew certificate - mlserve__amdgpu-node-labeller].orig\n+++ Exec[renew certificate - mlserve__amdgpu-node-labeller]\n\n+    unless      => /usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem -checkend 952200\n+    command     => /usr/bin/cfssl sign -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__amdgpu-node-labeller\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Exec[Generate cert mlserve__amdgpu-node-labeller]\n"}, {"resource": "File[/etc/systemd/system/kube-proxy.service.d]", "parameters": "--- File[/etc/systemd/system/kube-proxy.service.d].orig\n+++ File[/etc/systemd/system/kube-proxy.service.d]\n\n+    mode   => 0555\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "File[/etc/dragonfly]", "parameters": "--- File[/etc/dragonfly].orig\n+++ File[/etc/dragonfly]\n\n+    group   => root\n+    recurse => True\n+    mode    => 0740\n+    ensure  => directory\n+    owner   => dragonfly\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]\n\n+    group  => root\n+    source => puppet:///modules/profile/pki/intermediates/mlserve-cert.pem\n+    mode   => 0440\n+    ensure => file\n+    owner  => kube\n"}, {"resource": "Exec[create chained cert /etc/kubernetes/pki/mlserve__calicoctl.chain.pem]", "parameters": "--- Exec[create chained cert /etc/kubernetes/pki/mlserve__calicoctl.chain.pem].orig\n+++ Exec[create chained cert /etc/kubernetes/pki/mlserve__calicoctl.chain.pem]\n\n+    command   => /bin/cat /etc/kubernetes/pki/mlserve__calicoctl.pem /etc/kubernetes/pki/mlserve__calicoctl.chain.pem > /etc/kubernetes/pki/mlserve__calicoctl.chained.pem\n+    unless    => /usr/bin/test \"$(/bin/cat /etc/kubernetes/pki/mlserve__calicoctl.pem /etc/kubernetes/pki/mlserve__calicoctl.chain.pem | sha512sum)\" == \"$(/bin/cat /etc/kubernetes/pki/mlserve__calicoctl.chained.pem | sha512sum)\"\n\n+    subscribe => ['Exec[renew certificate - mlserve__calicoctl]', 'File[/etc/kubernetes/pki/mlserve__calicoctl.chain.pem]', 'File[/etc/kubernetes/pki/mlserve__calicoctl.pem]']\n+    require   => Exec[Generate cert mlserve__calicoctl refresh on intermediate ca change]\n"}, {"resource": "File[/etc/amd]", "parameters": "--- File[/etc/amd].orig\n+++ File[/etc/amd]\n\n+    owner  => root\n+    ensure => directory\n+    group  => root\n"}, {"resource": "Class[Containerd::Configuration]", "parameters": "--- Class[Containerd::Configuration].orig\n+++ Class[Containerd::Configuration]\n\n+    registry_username => kubernetes\n+    sandbox_image     => docker-registry.discovery.wmnet/pause:3.6-1\n+    dragonfly_enabled => True\n+    ensure            => present\n"}, {"resource": "Concat_file[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "parameters": "--- Concat_file[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources].orig\n+++ Concat_file[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]\n\n+    show_diff      => True\n+    tag            => _etc_apt_sources.list.d_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    format         => plain\n+    group          => root\n+    mode           => 0444\n+    order          => alpha\n+    owner          => root\n+    replace        => True\n+    backup         => puppet\n+    ensure_newline => False\n+    force          => False\n"}, {"resource": "Exec[Generate cert mlserve__istio-cni refresh on intermediate ca change]", "parameters": "--- Exec[Generate cert mlserve__istio-cni refresh on intermediate ca change].orig\n+++ Exec[Generate cert mlserve__istio-cni refresh on intermediate ca change]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__istio-cni.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__istio-cni\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/kubernetes/pki/mlserve__istio-cni.chain.pem]\n"}, {"resource": "Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "parameters": "--- Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header].orig\n+++ Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]\n\n+    source => puppet:///modules/apt/sources-deb822-header.txt\n+    target => /etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 01\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]\n\n+    group  => amd-nodelabeller\n+    source => puppet:///modules/profile/pki/intermediates/mlserve-cert.pem\n+    mode   => 0440\n+    ensure => file\n+    owner  => amd-nodelabeller\n"}, {"resource": "Systemd::Unit[kube-proxy]", "parameters": "--- Systemd::Unit[kube-proxy].orig\n+++ Systemd::Unit[kube-proxy]\n\n+    override_filename => puppet-override.conf\n+    override          => True\n+    restart           => True\n+    require           => ['Class[Systemd]']\n+    unit              => kube-proxy\n+    ensure            => present\n"}, {"resource": "Exec[Generate cert mlserve__system_kube-proxy refresh]", "parameters": "--- Exec[Generate cert mlserve__system_kube-proxy refresh].orig\n+++ Exec[Generate cert mlserve__system_kube-proxy refresh]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__system_kube-proxy.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__system_kube-proxy\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]\n+    notify      => ['Service[kube-proxy]']\n"}, {"resource": "Apt::Pin[apt_pin_linux-6.16-trixie_trixie-bpo]", "parameters": "--- Apt::Pin[apt_pin_linux-6.16-trixie_trixie-bpo].orig\n+++ Apt::Pin[apt_pin_linux-6.16-trixie_trixie-bpo]\n\n+    priority => 1001\n+    before   => ['Package[linux-image-6.16.3+deb13-amd64]']\n+    pin      => release a=trixie-backports\n+    ensure   => present\n+    notify   => Exec[exec-apt-get-update-linux-6.16-trixie_trixie-bpo]\n+    package  => linux-image-6.16.3+deb13-amd64\n"}, {"resource": "File[/etc/logrotate.d/rsyslog-release-deleted-inotify-watches]", "content": "--- /etc/logrotate.d/rsyslog-release-deleted-inotify-watches.orig\n+++ /etc/logrotate.d/rsyslog-release-deleted-inotify-watches\n@@ -0,0 +1,12 @@\n+# logrotate(8) config for rsyslog-release-deleted-inotify-watches\n+\n+/var/log/rsyslog-release-deleted-inotify-watches/*.log {\n+    daily\n+    copytruncate\n+    missingok\n+    compress\n+    delaycompress\n+    notifempty\n+    rotate 15\n+    size 256M\n+}", "parameters": "--- File[/etc/logrotate.d/rsyslog-release-deleted-inotify-watches].orig\n+++ File[/etc/logrotate.d/rsyslog-release-deleted-inotify-watches]\n\n+    mode   => 0444\n+    group  => root\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "Exec[create chained cert /etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]", "parameters": "--- Exec[create chained cert /etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem].orig\n+++ Exec[create chained cert /etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]\n\n+    command   => /bin/cat /etc/kubernetes/pki/mlserve__system_kube-proxy.pem /etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem > /etc/kubernetes/pki/mlserve__system_kube-proxy.chained.pem\n+    subscribe => ['Exec[renew certificate - mlserve__system_kube-proxy]', 'File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]', 'File[/etc/kubernetes/pki/mlserve__system_kube-proxy.pem]']\n+    require   => Exec[Generate cert mlserve__system_kube-proxy refresh on intermediate ca change]\n+    unless    => /usr/bin/test \"$(/bin/cat /etc/kubernetes/pki/mlserve__system_kube-proxy.pem /etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem | sha512sum)\" == \"$(/bin/cat /etc/kubernetes/pki/mlserve__system_kube-proxy.chained.pem | sha512sum)\"\n\n+    notify    => ['Service[kube-proxy]']\n"}, {"resource": "Systemd::Syslog[rsyslog-release-deleted-inotify-watches]", "parameters": "--- Systemd::Syslog[rsyslog-release-deleted-inotify-watches].orig\n+++ Systemd::Syslog[rsyslog-release-deleted-inotify-watches]\n\n+    group                  => root\n+    base_dir               => /var/log\n+    force_stop             => True\n+    owner                  => root\n+    programname_comparison => startswith\n+    readable_by            => all\n+    log_filename           => syslog.log\n+    ensure                 => absent\n"}, {"resource": "File[/etc/cni/net.d/calico-kubeconfig]", "content": "--- /etc/cni/net.d/calico-kubeconfig.orig\n+++ /etc/cni/net.d/calico-kubeconfig\n@@ -0,0 +1,18 @@\n+apiVersion: v1\n+kind: Config\n+preferences: {}\n+current-context: default-system\n+contexts:\n+- name: default-system\n+  context:\n+    cluster: default-cluster\n+    user: calico-cni\n+clusters:\n+- name: default-cluster\n+  cluster:\n+    server: https://ml-ctrl.svc.eqiad.wmnet:6443\n+users:\n+- name: calico-cni\n+  user:\n+    client-certificate: /etc/kubernetes/pki/mlserve__calico-cni.pem\n+    client-key: /etc/kubernetes/pki/mlserve__calico-cni-key.pem", "parameters": "--- File[/etc/cni/net.d/calico-kubeconfig].orig\n+++ File[/etc/cni/net.d/calico-kubeconfig]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => present\n+    owner  => root\n"}, {"resource": "Cfssl::Csr[/etc/cfssl/csr/mlserve__kubelet_server.csr]", "parameters": "--- Cfssl::Csr[/etc/cfssl/csr/mlserve__kubelet_server.csr].orig\n+++ Cfssl::Csr[/etc/cfssl/csr/mlserve__kubelet_server.csr]\n\n+    hosts       => ['ml-serve1015', 'ml-serve1015.eqiad.wmnet', '10.64.167.6', '2620:0:861:12f:10:64:167:6']\n+    key         => {'algo': 'ecdsa', 'size': 256}\n+    common_name => kubelet\n+    ensure      => present\n+    names       => []\n"}, {"resource": "Cfssl::Csr[/etc/cfssl/csr/mlserve__calico-cni.csr]", "parameters": "--- Cfssl::Csr[/etc/cfssl/csr/mlserve__calico-cni.csr].orig\n+++ Cfssl::Csr[/etc/cfssl/csr/mlserve__calico-cni.csr]\n\n+    hosts       => []\n+    key         => {'algo': 'ecdsa', 'size': 256}\n+    common_name => calico-cni\n+    ensure      => present\n+    names       => []\n"}, {"resource": "File[/etc/cfssl/csr/mlserve__calicoctl.csr]", "content": "--- /etc/cfssl/csr/mlserve__calicoctl.csr.orig\n+++ /etc/cfssl/csr/mlserve__calicoctl.csr\n@@ -0,0 +1,13 @@\n+{\n+  \"CN\": \"calicoctl\",\n+  \"hosts\": [\n+    \"calicoctl\"\n+  ],\n+  \"key\": {\n+    \"algo\": \"ecdsa\",\n+    \"size\": 256\n+  },\n+  \"names\": [\n+\n+  ]\n+}", "parameters": "--- File[/etc/cfssl/csr/mlserve__calicoctl.csr].orig\n+++ File[/etc/cfssl/csr/mlserve__calicoctl.csr]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Sysctl::Conffile[kube_proxy_conntrack]", "parameters": "--- Sysctl::Conffile[kube_proxy_conntrack].orig\n+++ Sysctl::Conffile[kube_proxy_conntrack]\n\n+    priority => 75\n+    ensure   => present\n"}, {"resource": "Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "content": "--- component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.orig\n+++ component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia\n@@ -0,0 +1,5 @@\n+Types: deb deb-src\n+URIs: http://apt.wikimedia.org/wikimedia\n+Suites: trixie-wikimedia\n+Components: component/kubernetes131\n+Signed-By: /etc/apt/keyrings/wikimedia-archive-keyring.gpg", "parameters": "--- Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    tag    => _etc_apt_sources.list.d_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    target => /etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 10\n"}, {"resource": "Exec[renew certificate - mlserve__calicoctl]", "parameters": "--- Exec[renew certificate - mlserve__calicoctl].orig\n+++ Exec[renew certificate - mlserve__calicoctl]\n\n+    unless      => /usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__calicoctl.pem -checkend 952200\n+    command     => /usr/bin/cfssl sign -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/kubernetes/pki/mlserve__calicoctl.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__calicoctl\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Exec[Generate cert mlserve__calicoctl]\n"}, {"resource": "Rsyslog::Conf[output_kafka_k8s]", "parameters": "--- Rsyslog::Conf[output_kafka_k8s].orig\n+++ Rsyslog::Conf[output_kafka_k8s]\n\n+    priority => 35\n+    mode     => 0444\n+    ensure   => present\n"}, {"resource": "Package[dragonfly-dfdaemon]", "parameters": "--- Package[dragonfly-dfdaemon].orig\n+++ Package[dragonfly-dfdaemon]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "File[/etc/kubernetes]", "parameters": "--- File[/etc/kubernetes].orig\n+++ File[/etc/kubernetes]\n\n+    mode   => 0755\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "Exec[systemd daemon-reload for kubelet.service (kubelet-container-runtime)]", "parameters": "--- Exec[systemd daemon-reload for kubelet.service (kubelet-container-runtime)].orig\n+++ Exec[systemd daemon-reload for kubelet.service (kubelet-container-runtime)]\n\n+    command     => /bin/systemctl daemon-reload\n+    notify      => ['Service[kubelet]']\n+    refreshonly => True\n"}, {"resource": "Exec[Generate cert mlserve__rsyslog refresh on intermediate ca change]", "parameters": "--- Exec[Generate cert mlserve__rsyslog refresh on intermediate ca change].orig\n+++ Exec[Generate cert mlserve__rsyslog refresh on intermediate ca change]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__rsyslog.csr | /usr/bin/cfssljson -bare /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]\n+    notify      => ['Service[rsyslog]']\n"}, {"resource": "File[/etc/dragonfly/dfget.yml]", "content": "--- /etc/dragonfly/dfget.yml.orig\n+++ /etc/dragonfly/dfget.yml\n@@ -0,0 +1,5 @@\n+# List of supernodes in the format\n+# host:port(default:8002)=weight(default:1)\n+# FIXME: Figure out how weight is exactly handled, could we use multiple supernodes without split brain?\n+nodes:\n+  - dragonfly-supernode1001.eqiad.wmnet:8002=1", "parameters": "--- File[/etc/dragonfly/dfget.yml].orig\n+++ File[/etc/dragonfly/dfget.yml]\n\n+    group  => root\n+    mode   => 0644\n+    ensure => file\n+    notify => Service[dragonfly-dfdaemon]\n+    owner  => root\n"}, {"resource": "Systemd::Timer[rsyslog-imfile-remedy]", "parameters": "--- Systemd::Timer[rsyslog-imfile-remedy].orig\n+++ Systemd::Timer[rsyslog-imfile-remedy]\n\n+    accuracy           => 15sec\n+    unit_name          => rsyslog-imfile-remedy.service\n+    ensure             => present\n+    fixed_random_delay => False\n+    splay              => 30\n+    timer_intervals    => [{'start': 'OnCalendar', 'interval': '*-*-* 00/3:41:00'}]\n"}, {"resource": "File[/etc/calico/pki]", "parameters": "--- File[/etc/calico/pki].orig\n+++ File[/etc/calico/pki]\n\n+    mode   => 0755\n+    group  => root\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "File[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]", "content": "--- /etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr.orig\n+++ /etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr\n@@ -0,0 +1,18 @@\n+{\n+  \"CN\": \"ml-serve1015.eqiad.wmnet\",\n+  \"hosts\": [\n+    \"ml-serve1015\",\n+    \"ml-serve1015.eqiad.wmnet\",\n+    \"docker-registry.discovery.wmnet\",\n+    \"127.0.0.1\",\n+    \"::1\",\n+    \"localhost\"\n+  ],\n+  \"key\": {\n+    \"algo\": \"ecdsa\",\n+    \"size\": 256\n+  },\n+  \"names\": [\n+\n+  ]\n+}", "parameters": "--- File[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr].orig\n+++ File[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Package[rsyslog-kubernetes]", "parameters": "--- Package[rsyslog-kubernetes].orig\n+++ Package[rsyslog-kubernetes]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "Ferm::Service[dragonfly_dfget]", "parameters": "--- Ferm::Service[dragonfly_dfget].orig\n+++ Ferm::Service[dragonfly_dfget]\n\n+    prio                => 10\n+    notrack             => False\n+    src_sets            => ['DOMAIN_NETWORKS']\n+    desc                => \n+    port                => 15001\n+    proto               => tcp\n+    ensure              => present\n+    unrestricted_access => False\n"}, {"resource": "Rsyslog::Conf[kubernetes-node-filters]", "parameters": "--- Rsyslog::Conf[kubernetes-node-filters].orig\n+++ Rsyslog::Conf[kubernetes-node-filters]\n\n+    source   => puppet:///modules/profile/kubernetes/node/kubernetes-node-filters.rsyslog.conf\n+    priority => 10\n+    mode     => 0444\n+    ensure   => present\n"}, {"resource": "File[/etc/calico/calicoctl-kubeconfig]", "content": "--- /etc/calico/calicoctl-kubeconfig.orig\n+++ /etc/calico/calicoctl-kubeconfig\n@@ -0,0 +1,18 @@\n+apiVersion: v1\n+kind: Config\n+preferences: {}\n+current-context: default-system\n+contexts:\n+- name: default-system\n+  context:\n+    cluster: default-cluster\n+    user: calicoctl\n+clusters:\n+- name: default-cluster\n+  cluster:\n+    server: https://ml-ctrl.svc.eqiad.wmnet:6443\n+users:\n+- name: calicoctl\n+  user:\n+    client-certificate: /etc/kubernetes/pki/mlserve__calicoctl.pem\n+    client-key: /etc/kubernetes/pki/mlserve__calicoctl-key.pem", "parameters": "--- File[/etc/calico/calicoctl-kubeconfig].orig\n+++ File[/etc/calico/calicoctl-kubeconfig]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => present\n+    owner  => root\n"}, {"resource": "File[/var/lib/prometheus/node.d/role_owner.prom]", "content": "--- /var/lib/prometheus/node.d/role_owner.prom.orig\n+++ /var/lib/prometheus/node.d/role_owner.prom\n@@ -1,3 +1,3 @@\n # HELP role_owner The team owner of the server role\n # TYPE role_owner gauge\n-role_owner{team=\"machine-learning\",role=\"ml_k8s::insetup_gpu\",cluster=\"ml_serve\"} 1.0\n+role_owner{team=\"machine-learning\",role=\"ml_k8s::worker\",cluster=\"ml_serve\"} 1.0"}, {"resource": "Exec[renew certificate - mlserve__system_node_ml-serve1015_eqiad_wmnet]", "parameters": "--- Exec[renew certificate - mlserve__system_node_ml-serve1015_eqiad_wmnet].orig\n+++ Exec[renew certificate - mlserve__system_node_ml-serve1015_eqiad_wmnet]\n\n+    command     => /usr/bin/cfssl sign -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet]\n+    unless      => /usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem -checkend 952200\n+    notify      => ['Service[kubelet]']\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calico-cni.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calico-cni.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__calico-cni.pem]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calicoctl.chained.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calicoctl.chained.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__calicoctl.chained.pem]\n\n+    group   => root\n+    ensure  => file\n+    require => Exec[create chained cert /etc/kubernetes/pki/mlserve__calicoctl.chain.pem]\n+    owner   => root\n"}, {"resource": "Cfssl::Csr[/etc/cfssl/csr/mlserve__istio-cni.csr]", "parameters": "--- Cfssl::Csr[/etc/cfssl/csr/mlserve__istio-cni.csr].orig\n+++ Cfssl::Csr[/etc/cfssl/csr/mlserve__istio-cni.csr]\n\n+    hosts       => []\n+    key         => {'algo': 'ecdsa', 'size': 256}\n+    common_name => istio-cni\n+    ensure      => present\n+    names       => []\n"}, {"resource": "Rsyslog::Conf[rsyslog-release-deleted-inotify-watches]", "parameters": "--- Rsyslog::Conf[rsyslog-release-deleted-inotify-watches].orig\n+++ Rsyslog::Conf[rsyslog-release-deleted-inotify-watches]\n\n+    priority => 40\n+    mode     => 0444\n+    ensure   => absent\n+    require  => File[/var/log/rsyslog-release-deleted-inotify-watches]\n"}, {"resource": "Exec[apt_repository_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "parameters": "--- Exec[apt_repository_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Exec[apt_repository_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    command     => /usr/bin/apt-get update \n+    refreshonly => True\n"}, {"resource": "K8s::Kubeconfig[/etc/cni/net.d/calico-kubeconfig]", "parameters": "--- K8s::Kubeconfig[/etc/cni/net.d/calico-kubeconfig].orig\n+++ K8s::Kubeconfig[/etc/cni/net.d/calico-kubeconfig]\n\n+    username    => calico-cni\n+    group       => root\n+    mode        => 0400\n+    owner       => root\n+    require     => ['File[/etc/cni/net.d]', 'Class[K8s::Base_dirs]']\n+    auth_cert   => {'cert': '/etc/kubernetes/pki/mlserve__calico-cni.pem', 'key': '/etc/kubernetes/pki/mlserve__calico-cni-key.pem', 'chain': '/etc/kubernetes/pki/mlserve__calico-cni.chain.pem', 'chained': '/etc/kubernetes/pki/mlserve__calico-cni.chained.pem'}\n+    ensure      => present\n+    master_host => ml-ctrl.svc.eqiad.wmnet\n"}, {"resource": "Apt::Package_from_bpo[linux-6.16-trixie]", "parameters": "--- Apt::Package_from_bpo[linux-6.16-trixie].orig\n+++ Apt::Package_from_bpo[linux-6.16-trixie]\n\n+    packages        => {'linux-image-6.16.3+deb13-amd64': 'present'}\n+    priority        => 1001\n+    distro          => trixie\n+    ensure_packages => True\n"}, {"resource": "Kmod::Module[overlay]", "parameters": "--- Kmod::Module[overlay].orig\n+++ Kmod::Module[overlay]\n\n+    ensure => present\n"}, {"resource": "Exec[Generate cert mlserve__calicoctl refresh on intermediate ca change]", "parameters": "--- Exec[Generate cert mlserve__calicoctl refresh on intermediate ca change].orig\n+++ Exec[Generate cert mlserve__calicoctl refresh on intermediate ca change]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__calicoctl.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__calicoctl\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/kubernetes/pki/mlserve__calicoctl.chain.pem]\n"}, {"resource": "Nrpe::Monitor_service[disk_space]", "parameters": "--- Nrpe::Monitor_service[disk_space].orig\n+++ Nrpe::Monitor_service[disk_space]\n\n@@\n-    nrpe_command => /usr/lib/nagios/plugins/check_disk -w 6% -c 3% -W 6% -K 3% -l -e -A -i \"/srv/sd[a-b][1-3]\" -i \"/srv/nvme[0-9]n[0-9]p[0-9]\" --exclude-type=fuse --exclude-type=fuse.fuse_dfs --exclude-type=tracefs\n+    nrpe_command => /usr/lib/nagios/plugins/check_disk -w 10% -c 5% -W 6% -K 3% -l -e -A -i '/(var/lib|run)/(containerd|kubelet)/*' --exclude-type=tracefs\n"}, {"resource": "Systemd::Unit[rsyslog-imfile-remedy.timer]", "parameters": "--- Systemd::Unit[rsyslog-imfile-remedy.timer].orig\n+++ Systemd::Unit[rsyslog-imfile-remedy.timer]\n\n+    override_filename => puppet-override.conf\n+    override          => False\n+    restart           => False\n+    require           => ['Class[Systemd]']\n+    unit              => rsyslog-imfile-remedy.timer\n+    ensure            => present\n"}, {"resource": "Systemd::Override[container-runtime]", "parameters": "--- Systemd::Override[container-runtime].orig\n+++ Systemd::Override[container-runtime]\n\n+    ensure  => present\n+    restart => True\n+    unit    => kubelet\n"}, {"resource": "File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.service]", "content": "--- /lib/systemd/system/rsyslog-release-deleted-inotify-watches.service.orig\n+++ /lib/systemd/system/rsyslog-release-deleted-inotify-watches.service\n@@ -0,0 +1,8 @@\n+[Unit]\n+Description=Restart rsyslog to release inotify watches of deleted container logs\n+Documentation=https://wikitech.wikimedia.org/wiki/Monitoring/systemd_unit_state\n+\n+[Service]\n+Type=oneshot\n+User=root\n+ExecStart=/usr/local/sbin/rsyslog-release-deleted-inotify-watches", "parameters": "--- File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.service].orig\n+++ File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.service]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => absent\n+    notify => Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.service (rsyslog-release-deleted-inotify-watches.service)]\n+    owner  => root\n"}, {"resource": "File[/etc/rsyslog.d/20-shellbox.conf]", "parameters": "--- File[/etc/rsyslog.d/20-shellbox.conf].orig\n+++ File[/etc/rsyslog.d/20-shellbox.conf]\n\n+    group  => root\n+    source => puppet:///modules/profile/rsyslog/shellbox.rsyslog.conf\n+    mode   => 0444\n+    ensure => present\n+    notify => Service[rsyslog]\n+    owner  => root\n"}, {"resource": "Package[linux-cpupower]", "parameters": "--- Package[linux-cpupower].orig\n+++ Package[linux-cpupower]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "File[/etc/amd/node-labeller-kubeconfig]", "content": "--- /etc/amd/node-labeller-kubeconfig.orig\n+++ /etc/amd/node-labeller-kubeconfig\n@@ -0,0 +1,18 @@\n+apiVersion: v1\n+kind: Config\n+preferences: {}\n+current-context: default-system\n+contexts:\n+- name: default-system\n+  context:\n+    cluster: default-cluster\n+    user: amdgpu-node-labeller\n+clusters:\n+- name: default-cluster\n+  cluster:\n+    server: https://ml-ctrl.svc.eqiad.wmnet:6443\n+users:\n+- name: amdgpu-node-labeller\n+  user:\n+    client-certificate: /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem\n+    client-key: /etc/kubernetes/pki/mlserve__amdgpu-node-labeller-key.pem", "parameters": "--- File[/etc/amd/node-labeller-kubeconfig].orig\n+++ File[/etc/amd/node-labeller-kubeconfig]\n\n+    mode   => 0400\n+    group  => amd-nodelabeller\n+    ensure => present\n+    owner  => amd-nodelabeller\n"}, {"resource": "Class[Profile::Base::Production]", "parameters": "--- Class[Profile::Base::Production].orig\n+++ Class[Profile::Base::Production]\n\n@@\n-    role_description => Machine Learning GPU host in setup.\n+    role_description => ML Kubernetes worker node\n"}, {"resource": "Rsyslog::Conf[input-file-kubernetes-json]", "parameters": "--- Rsyslog::Conf[input-file-kubernetes-json].orig\n+++ Rsyslog::Conf[input-file-kubernetes-json]\n\n+    priority => 8\n+    mode     => 0444\n+    ensure   => present\n+    require  => Rsyslog::Conf[imfile]\n"}, {"resource": "Systemd::Service[rsyslog-release-deleted-inotify-watches]", "parameters": "--- Systemd::Service[rsyslog-release-deleted-inotify-watches].orig\n+++ Systemd::Service[rsyslog-release-deleted-inotify-watches]\n\n+    monitoring_enabled       => False\n+    override                 => False\n+    monitoring_critical      => False\n+    restart                  => False\n+    require                  => Systemd::Unit[rsyslog-release-deleted-inotify-watches.service]\n+    monitoring_contact_group => admins\n+    service_params           => {}\n+    migration_task           => T407130\n+    ensure                   => absent\n+    unit_type                => timer\n"}, {"resource": "Class[Base::Kernel]", "parameters": "--- Class[Base::Kernel].orig\n+++ Class[Base::Kernel]\n\n@@\n-    overlayfs => False\n+    overlayfs => True\n"}, {"resource": "File[/usr/local/sbin/rsyslog-release-deleted-inotify-watches]", "parameters": "--- File[/usr/local/sbin/rsyslog-release-deleted-inotify-watches].orig\n+++ File[/usr/local/sbin/rsyslog-release-deleted-inotify-watches]\n\n+    group  => root\n+    mode   => 0544\n+    ensure => absent\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/pki]", "parameters": "--- File[/etc/kubernetes/pki].orig\n+++ File[/etc/kubernetes/pki]\n\n+    mode   => 0755\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "K8s::Kubelet::Cni[calico]", "parameters": "--- K8s::Kubelet::Cni[calico].orig\n+++ K8s::Kubelet::Cni[calico]\n\n+    priority => 10\n+    config   => {'name': 'k8s-pod-network', 'cniVersion': '0.3.1', 'plugins': [{'type': 'calico', 'log_level': 'info', 'datastore_type': 'kubernetes', 'mtu': 1460, 'ipam': {'type': 'calico-ipam', 'assign_ipv4': 'true', 'assign_ipv6': 'true'}, 'policy': {'type': 'k8s'}, 'kubernetes': {'kubeconfig': '/etc/cni/net.d/calico-kubeconfig'}}, {'name': 'istio-cni', 'type': 'istio-cni', 'log_level': 'info', 'kubernetes': {'kubeconfig': '/etc/cni/net.d/istio-kubeconfig', 'cni_bin_dir': '/opt/cni/bin', 'exclude_namespaces': ['istio-system', 'kube-system', 'knative-serving', 'cert-manager', 'kserve']}}]}\n+    require  => ['Class[K8s::Kubelet::Cni::Base]']\n"}, {"resource": "Exec[apt_pin_apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo]", "parameters": "--- Exec[apt_pin_apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo].orig\n+++ Exec[apt_pin_apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo]\n\n+    command     => /usr/bin/apt-get update\n+    refreshonly => True\n"}, {"resource": "File[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]", "content": "--- /etc/cfssl/csr/mlserve__system_kube-proxy.csr.orig\n+++ /etc/cfssl/csr/mlserve__system_kube-proxy.csr\n@@ -0,0 +1,19 @@\n+{\n+  \"CN\": \"system:kube-proxy\",\n+  \"hosts\": [\n+    \"system:kube-proxy\"\n+  ],\n+  \"key\": {\n+    \"algo\": \"ecdsa\",\n+    \"size\": 256\n+  },\n+  \"names\": [\n+    {\n+      \"C\": null,\n+      \"L\": null,\n+      \"O\": \"system:node-proxier\",\n+      \"OU\": null,\n+      \"S\": null\n+    }\n+  ]\n+}", "parameters": "--- File[/etc/cfssl/csr/mlserve__system_kube-proxy.csr].orig\n+++ File[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "File[/etc/udev/rules.d/75-kube_proxy_conntrack.rules]", "content": "--- /etc/udev/rules.d/75-kube_proxy_conntrack.rules.orig\n+++ /etc/udev/rules.d/75-kube_proxy_conntrack.rules\n@@ -0,0 +1,2 @@\n+ACTION==\"add\", SUBSYSTEM==\"module\", KERNEL==\"nf_conntrack\", \\\n+    RUN+=\"/usr/lib/systemd/systemd-sysctl --prefix net.netfilter.nf_conntrack_max\"", "parameters": "--- File[/etc/udev/rules.d/75-kube_proxy_conntrack.rules].orig\n+++ File[/etc/udev/rules.d/75-kube_proxy_conntrack.rules]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[udev_reload]\n+    owner  => root\n"}, {"resource": "Class[Profile::Base]", "parameters": "--- Class[Profile::Base].orig\n+++ Class[Profile::Base]\n\n@@\n-    use_linux_from_bpo_on_trixie => False\n+    use_linux_from_bpo_on_trixie => True\n@@\n-    overlayfs                    => False\n+    overlayfs                    => True\n"}, {"resource": "File[/etc/kubernetes/kubelet-config.yaml]", "content": "--- /etc/kubernetes/kubelet-config.yaml.orig\n+++ /etc/kubernetes/kubelet-config.yaml\n@@ -0,0 +1,25 @@\n+---\n+apiVersion: kubelet.config.k8s.io/v1beta1\n+kind: KubeletConfiguration\n+tlsPrivateKeyFile: \"/etc/kubernetes/pki/mlserve__kubelet_server-key.pem\"\n+tlsCertFile: \"/etc/kubernetes/pki/mlserve__kubelet_server.chained.pem\"\n+clusterDomain: cluster.local\n+clusterDNS:\n+- 10.67.0.3\n+authentication:\n+  anonymous:\n+    enabled: false\n+  webhook:\n+    enabled: true\n+  x509:\n+    clientCAFile: \"/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem\"\n+authorization:\n+  mode: Webhook\n+cgroupDriver: systemd\n+evictionHard:\n+  imagefs.available: 15%\n+  memory.available: 300M\n+  nodefs.available: 10%\n+  nodefs.inodesFree: 5%\n+containerRuntimeEndpoint: unix:///run/containerd/containerd.sock\n+seccompDefault: true", "parameters": "--- File[/etc/kubernetes/kubelet-config.yaml].orig\n+++ File[/etc/kubernetes/kubelet-config.yaml]\n\n+    require => K8s::Package[kubelet]\n+    group   => kube\n+    mode    => 0400\n+    ensure  => file\n+    notify  => Service[kubelet]\n+    owner   => kube\n"}, {"resource": "Exec[create chained cert /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]", "parameters": "--- Exec[create chained cert /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem].orig\n+++ Exec[create chained cert /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]\n\n+    command   => /bin/cat /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem > /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem\n+    subscribe => ['Exec[renew certificate - discovery__ml-serve1015_eqiad_wmnet]', 'File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]', 'File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem]']\n+    require   => Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh on intermediate ca change]\n+    unless    => /usr/bin/test \"$(/bin/cat /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem | sha512sum)\" == \"$(/bin/cat /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem | sha512sum)\"\n\n+    notify    => ['Service[dragonfly-dfdaemon]']\n"}, {"resource": "Exec[systemd daemon-reload for ferm.service (ferm-ferm-service-auto-restart)]", "parameters": "--- Exec[systemd daemon-reload for ferm.service (ferm-ferm-service-auto-restart)].orig\n+++ Exec[systemd daemon-reload for ferm.service (ferm-ferm-service-auto-restart)]\n\n+    before      => ['Service[ferm]']\n+    command     => /bin/systemctl daemon-reload\n+    refreshonly => True\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__istio-cni.chain.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__istio-cni.chain.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__istio-cni.chain.pem]\n\n+    group  => root\n+    source => puppet:///modules/profile/pki/intermediates/mlserve-cert.pem\n+    mode   => 0440\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Exec[apt_repository_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "parameters": "--- Exec[apt_repository_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Exec[apt_repository_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    command     => /usr/bin/apt-get update \n+    refreshonly => True\n"}, {"resource": "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.csr]", "parameters": "--- File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.csr].orig\n+++ File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.csr]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => dragonfly\n"}, {"resource": "File[/etc/apt/preferences.d/apt_pin_linux_6_16_trixie_trixie_bpo.pref]", "content": "--- /etc/apt/preferences.d/apt_pin_linux_6_16_trixie_trixie_bpo.pref.orig\n+++ /etc/apt/preferences.d/apt_pin_linux_6_16_trixie_trixie_bpo.pref\n@@ -0,0 +1,3 @@\n+Package: linux-image-6.16.3+deb13-amd64\n+Pin: release a=trixie-backports\n+Pin-Priority: 1001", "parameters": "--- File[/etc/apt/preferences.d/apt_pin_linux_6_16_trixie_trixie_bpo.pref].orig\n+++ File[/etc/apt/preferences.d/apt_pin_linux_6_16_trixie_trixie_bpo.pref]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[exec-apt-get-update-linux-6.16-trixie_trixie-bpo]\n+    owner  => root\n"}, {"resource": "Sysctl::Conffile[increase_inotify_limits]", "parameters": "--- Sysctl::Conffile[increase_inotify_limits].orig\n+++ Sysctl::Conffile[increase_inotify_limits]\n\n+    priority => 70\n+    ensure   => present\n"}, {"resource": "K8s::Package[proxy]", "parameters": "--- K8s::Package[proxy].orig\n+++ K8s::Package[proxy]\n\n+    distro          => trixie-wikimedia\n+    require         => ['Class[K8s::Base_dirs]']\n+    ensure_packages => True\n+    priority        => 1001\n+    uri             => http://apt.wikimedia.org/wikimedia\n+    version         => 1.31\n+    package         => node\n"}, {"resource": "Exec[apt_package_from_component_kubernetes131]", "parameters": "--- Exec[apt_package_from_component_kubernetes131].orig\n+++ Exec[apt_package_from_component_kubernetes131]\n\n+    before      => []\n+    command     => /usr/bin/apt-get update\n+    refreshonly => True\n+    subscribe   => Apt::Repository[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n"}, {"resource": "Exec[apt_pin_apt_pin_linux-6.16-trixie_trixie-bpo]", "parameters": "--- Exec[apt_pin_apt_pin_linux-6.16-trixie_trixie-bpo].orig\n+++ Exec[apt_pin_apt_pin_linux-6.16-trixie_trixie-bpo]\n\n+    command     => /usr/bin/apt-get update\n+    refreshonly => True\n"}, {"resource": "Package[dragonfly-dfget]", "parameters": "--- Package[dragonfly-dfget].orig\n+++ Package[dragonfly-dfget]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "parameters": "--- Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header].orig\n+++ Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]\n\n+    source => puppet:///modules/apt/sources-deb822-header.txt\n+    tag    => _etc_apt_sources.list.d_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    target => /etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources\n+    order  => 01\n"}, {"resource": "Class[Profile::Dragonfly::Dfdaemon]", "parameters": "--- Class[Profile::Dragonfly::Dfdaemon].orig\n+++ Class[Profile::Dragonfly::Dfdaemon]\n\n+    supernodes           => ['dragonfly-supernode1001.eqiad.wmnet:8002=1']\n+    ratelimit            => 100M\n+    docker_registry_fqdn => docker-registry.discovery.wmnet\n+    ensure               => present\n+    proxy_urls_regex     => ['wikimedia/machinelearning-liftwing.*/blobs/sha256.*', 'amd-pytorch.*/blobs/sha256.*']\n"}, {"resource": "File[/etc/update-motd.d/05-ml-k8s--worker]", "content": "--- /etc/update-motd.d/05-ml-k8s--worker.orig\n+++ /etc/update-motd.d/05-ml-k8s--worker\n@@ -0,0 +1,2 @@\n+#!/bin/sh\n+printf \"%s\\n\" \"ml-serve1015 is a ML Kubernetes worker node (ml_k8s::worker)\"", "parameters": "--- File[/etc/update-motd.d/05-ml-k8s--worker].orig\n+++ File[/etc/update-motd.d/05-ml-k8s--worker]\n\n+    group  => root\n+    mode   => 0555\n+    ensure => present\n+    owner  => root\n"}, {"resource": "Package[apparmor]", "parameters": "--- Package[apparmor].orig\n+++ Package[apparmor]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "Apt::Package_from_component[istio115]", "parameters": "--- Apt::Package_from_component[istio115].orig\n+++ Apt::Package_from_component[istio115]\n\n+    distro          => trixie-wikimedia\n+    component       => component/istio115\n+    ensure_packages => True\n+    packages        => {'istio-cni': 'present'}\n+    priority        => 1001\n+    uri             => http://apt.wikimedia.org/wikimedia\n+    ensure          => present\n"}, {"resource": "Service[kubelet]", "parameters": "--- Service[kubelet].orig\n+++ Service[kubelet]\n\n+    enable    => True\n+    ensure    => running\n+    subscribe => ['File[/etc/kubernetes/kubelet.conf]']\n"}, {"resource": "Exec[Generate cert mlserve__amdgpu-node-labeller refresh]", "parameters": "--- Exec[Generate cert mlserve__amdgpu-node-labeller refresh].orig\n+++ Exec[Generate cert mlserve__amdgpu-node-labeller refresh]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__amdgpu-node-labeller\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]\n"}, {"resource": "Exec[create chained cert /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]", "parameters": "--- Exec[create chained cert /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem].orig\n+++ Exec[create chained cert /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]\n\n+    command   => /bin/cat /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem > /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chained.pem\n+    subscribe => ['Exec[renew certificate - mlserve__rsyslog]', 'File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]', 'File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem]']\n+    require   => Exec[Generate cert mlserve__rsyslog refresh on intermediate ca change]\n+    unless    => /usr/bin/test \"$(/bin/cat /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem | sha512sum)\" == \"$(/bin/cat /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chained.pem | sha512sum)\"\n\n+    notify    => ['Service[rsyslog]']\n"}, {"resource": "Ferm::Service[calico_typha]", "parameters": "--- Ferm::Service[calico_typha].orig\n+++ Ferm::Service[calico_typha]\n\n+    prio                => 10\n+    notrack             => False\n+    src_sets            => ['DOMAIN_NETWORKS']\n+    desc                => \n+    port                => 5473\n+    proto               => tcp\n+    ensure              => present\n+    unrestricted_access => False\n"}, {"resource": "File[/etc/systemd/system/amd-k8s-node-labeller.service.d/amd-devplugin-after-labeller.conf]", "content": "--- /etc/systemd/system/amd-k8s-node-labeller.service.d/amd-devplugin-after-labeller.conf.orig\n+++ /etc/systemd/system/amd-k8s-node-labeller.service.d/amd-devplugin-after-labeller.conf\n@@ -0,0 +1,3 @@\n+[Unit]\n+After=amd-k8s-device-plugin.service\n+Requires=amd-k8s-device-plugin.service", "parameters": "--- File[/etc/systemd/system/amd-k8s-node-labeller.service.d/amd-devplugin-after-labeller.conf].orig\n+++ File[/etc/systemd/system/amd-k8s-node-labeller.service.d/amd-devplugin-after-labeller.conf]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[systemd daemon-reload for amd-k8s-node-labeller.service (amd-k8s-node-labeller-amd-devplugin-after-labeller)]\n+    owner  => root\n"}, {"resource": "Package[crictl]", "parameters": "--- Package[crictl].orig\n+++ Package[crictl]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__istio-cni.chained.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__istio-cni.chained.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__istio-cni.chained.pem]\n\n+    group   => root\n+    ensure  => file\n+    require => Exec[create chained cert /etc/kubernetes/pki/mlserve__istio-cni.chain.pem]\n+    owner   => root\n"}, {"resource": "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh on intermediate ca change]", "parameters": "--- Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh on intermediate ca change].orig\n+++ Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh on intermediate ca change]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]\n+    notify      => ['Service[kubelet]']\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]\n\n+    group  => root\n+    source => puppet:///modules/profile/pki/intermediates/mlserve-cert.pem\n+    mode   => 0440\n+    ensure => file\n+    owner  => kube\n"}, {"resource": "Package[firmware-amd-graphics]", "parameters": "--- Package[firmware-amd-graphics].orig\n+++ Package[firmware-amd-graphics]\n\n@@\n-    ensure => installed\n+    ensure => 20251021-1~bpo13+1\n"}, {"resource": "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet]", "parameters": "--- Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet].orig\n+++ Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label discovery  /etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr | /usr/bin/cfssljson -bare /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Cfssl::Csr[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]\n+    unless      => /usr/bin/test \"$(/usr/bin/openssl x509 -in /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem -noout -pubkey 2>&1)\" == \"$(/usr/bin/openssl pkey -pubout -in /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem 2>&1)\"\n\n+    notify      => ['Service[dragonfly-dfdaemon]']\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__istio-cni-key.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__istio-cni-key.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__istio-cni-key.pem]\n\n+    show_diff => False\n+    backup    => False\n+    group     => root\n+    mode      => 0440\n+    ensure    => file\n+    owner     => root\n"}, {"resource": "K8s::Kubeconfig[/etc/cni/net.d/istio-kubeconfig]", "parameters": "--- K8s::Kubeconfig[/etc/cni/net.d/istio-kubeconfig].orig\n+++ K8s::Kubeconfig[/etc/cni/net.d/istio-kubeconfig]\n\n+    username    => istio-cni\n+    group       => root\n+    mode        => 0400\n+    owner       => root\n+    require     => ['File[/etc/cni/net.d]', 'Class[K8s::Base_dirs]']\n+    auth_cert   => {'cert': '/etc/kubernetes/pki/mlserve__istio-cni.pem', 'key': '/etc/kubernetes/pki/mlserve__istio-cni-key.pem', 'chain': '/etc/kubernetes/pki/mlserve__istio-cni.chain.pem', 'chained': '/etc/kubernetes/pki/mlserve__istio-cni.chained.pem'}\n+    ensure      => present\n+    master_host => ml-ctrl.svc.eqiad.wmnet\n"}, {"resource": "File[/etc/rsyslog.d/35-output-kafka-k8s.conf]", "content": "--- /etc/rsyslog.d/35-output-kafka-k8s.conf.orig\n+++ /etc/rsyslog.d/35-output-kafka-k8s.conf\n@@ -0,0 +1,24 @@\n+\n+\n+if ( $.log_outputs contains \"k8s\" ) then {\n+    action(type=\"mmjsonparse\" name=\"mmjsonparse_kafka_k8s\")\n+\n+    action(type=\"omkafka\"\n+           name=\"omkafka_k8s\"\n+           broker=[\"kafka-logging1001.eqiad.wmnet:9093\",\"kafka-logging1002.eqiad.wmnet:9093\",\"kafka-logging1003.eqiad.wmnet:9093\",\"kafka-logging1004.eqiad.wmnet:9093\",\"kafka-logging1005.eqiad.wmnet:9093\"]\n+           topic=\"k8s-ml-serve-eqiad\"\n+           partitions.auto=\"on\"\n+           template=\"syslog_cee\"\n+           queue.type=\"LinkedList\" queue.size=\"10000\" queue.filename=\"output_kafka_k8s\"\n+           queue.highWatermark=\"7000\" queue.lowWatermark=\"6000\"\n+           queue.checkpointInterval=\"5\"\n+           queue.maxDiskSpace=\"40960000\"\n+           confParam=[ \"security.protocol=ssl\",\n+                       \"ssl.ca.location=/etc/ssl/certs/wmf-ca-certificates.crt\",\n+                       \"compression.codec=snappy\",\n+                       \"socket.timeout.ms=10000\",\n+                       \"socket.keepalive.enable=true\",\n+                       \"queue.buffering.max.ms=50\",\n+                       \"batch.num.messages=1000\" ]\n+    )\n+}", "parameters": "--- File[/etc/rsyslog.d/35-output-kafka-k8s.conf].orig\n+++ File[/etc/rsyslog.d/35-output-kafka-k8s.conf]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Service[rsyslog]\n+    owner  => root\n"}, {"resource": "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh on intermediate ca change]", "parameters": "--- Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh on intermediate ca change].orig\n+++ Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh on intermediate ca change]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label discovery  /etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr | /usr/bin/cfssljson -bare /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]\n+    notify      => ['Service[dragonfly-dfdaemon]']\n"}, {"resource": "Exec[renew certificate - mlserve__istio-cni]", "parameters": "--- Exec[renew certificate - mlserve__istio-cni].orig\n+++ Exec[renew certificate - mlserve__istio-cni]\n\n+    unless      => /usr/bin/openssl x509 -in /etc/kubernetes/pki/mlserve__istio-cni.pem -checkend 952200\n+    command     => /usr/bin/cfssl sign -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/kubernetes/pki/mlserve__istio-cni.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__istio-cni\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    require     => Exec[Generate cert mlserve__istio-cni]\n"}, {"resource": "Class[Cpufrequtils]", "parameters": "--- Class[Cpufrequtils].orig\n+++ Class[Cpufrequtils]\n\n+    governor => performance\n+    ensure   => present\n"}, {"resource": "Class[Profile::Apt]", "parameters": "--- Class[Profile::Apt].orig\n+++ Class[Profile::Apt]\n\n@@\n-    before => ['Package[puppet]', 'Package[facter]', 'Package[augeas-tools]', 'Package[virt-what]', 'Package[puppet-module-puppetlabs-augeas-core]', 'Package[python3-prometheus-client]', 'Package[python3-yaml]', 'Package[ruby-net-ssh]', 'Package[openssl]', 'Package[ssl-cert]', 'Package[ca-certificates]', 'Package[wmf-certificates]', 'Package[ntp]', 'Package[systemd-timesyncd]', 'Package[exim4-config]', 'Package[exim4-daemon-light]', 'Package[logrotate]', 'Package[prometheus-node-exporter]', 'Package[bsdutils]', 'Package[smartmontools]', 'Package[rsyslog]', 'Package[rsyslog-openssl]', 'Package[cadvisor]', 'Package[acct]', 'Package[byobu]', 'Package[colordiff]', 'Package[curl]', 'Package[debian-goodies]', 'Package[ethtool]', 'Package[gdb]', 'Package[gdisk]', 'Package[git]', 'Package[htop]', 'Package[httpry]', 'Package[iotop]', 'Package[iperf]', 'Package[jq]', 'Package[libtemplate-perl]', 'Package[lldpd]', 'Package[lshw]', 'Package[molly-guard]', 'Package[moreutils]', 'Package[net-tools]', 'Package[numactl]', 'Package[ncdu]', 'Package[ngrep]', 'Package[pigz]', 'Package[psmisc]', 'Package[pv]', 'Package[python3]', 'Package[screen]', 'Package[strace]', 'Package[sysstat]', 'Package[tcpdump]', 'Package[tmux]', 'Package[tree]', 'Package[vim]', 'Package[vim-addon-manager]', 'Package[vim-scripts]', 'Package[wipe]', 'Package[xfsprogs]', 'Package[zsh]', 'Package[icdiff]', 'Package[linux-perf]', 'Package[bsd-mailx]', 'Package[ack]', 'Package[netcat-openbsd]', 'Package[tshark]', 'Package[fzf]', 'Package[ripgrep]', 'Package[fd-find]', 'Package[kitty-terminfo]', 'Package[mtr-tiny]', 'Package[bat]', 'Package[efibootmgr]', 'Package[bind9-dnsutils]', 'Package[tzdata]', 'Package[python3-wmflib]', 'Package[starship]', 'Package[ruby-sorted-set]', 'Package[btop]', 'Package[linux-sysctl-defaults]', 'Package[apport]', 'Package[command-not-found]', 'Package[command-not-found-data]', 'Package[ecryptfs-utils]', 'Package[mlocate]', 'Package[os-prober]', 'Package[python3-apport]', 'Package[wpasupplicant]', 'Package[atop]', 'Package[apt-listchanges]', 'Package[isc-dhcp-client]', 'Package[rasdaemon]', 'Package[openssh-client]', 'Package[openssh-server]', 'Package[debdeploy-client]', 'Package[python3-dateutil]', 'Package[sudo]', 'Package[golang-cfssl]', 'Package[debmonitor-client]', 'Package[nagios-nrpe-server]', 'Package[monitoring-plugins]', 'Package[monitoring-plugins-basic]', 'Package[monitoring-plugins-standard]', 'Package[liburiparser1]', 'Package[python3-attr]', 'Package[freeipmi-tools]', 'Package[freeipmi-ipmiseld]', 'Package[rsyslog-kafka]', 'Package[emacs-nox]', 'Package[prometheus-ipmi-exporter]', 'Package[libnet-dns-perl]', 'Package[iptables]', 'Package[ferm]', 'Package[ulogd2]', 'Package[conntrack]', 'Package[rocm-smi]', 'Package[python3-requests]', 'Package[firmware-amd-graphics]', 'Package[ruby-concurrent]', 'Package[ruby]', 'Package[libruby]', 'Package[puppet-agent]', 'Package[prometheus-rsyslog-exporter]', 'Package[initramfs-tools]', 'Package[python3-click]', 'Package[python3-box]', 'Package[confd]', 'Package[python3-toml]']\n+    before => ['Package[puppet]', 'Package[facter]', 'Package[augeas-tools]', 'Package[virt-what]', 'Package[puppet-module-puppetlabs-augeas-core]', 'Package[python3-prometheus-client]', 'Package[python3-yaml]', 'Package[ruby-net-ssh]', 'Package[openssl]', 'Package[ssl-cert]', 'Package[ca-certificates]', 'Package[wmf-certificates]', 'Package[ntp]', 'Package[systemd-timesyncd]', 'Package[exim4-config]', 'Package[exim4-daemon-light]', 'Package[logrotate]', 'Package[prometheus-node-exporter]', 'Package[bsdutils]', 'Package[smartmontools]', 'Package[rsyslog]', 'Package[rsyslog-openssl]', 'Package[cadvisor]', 'Package[acct]', 'Package[byobu]', 'Package[colordiff]', 'Package[curl]', 'Package[debian-goodies]', 'Package[ethtool]', 'Package[gdb]', 'Package[gdisk]', 'Package[git]', 'Package[htop]', 'Package[httpry]', 'Package[iotop]', 'Package[iperf]', 'Package[jq]', 'Package[libtemplate-perl]', 'Package[lldpd]', 'Package[lshw]', 'Package[molly-guard]', 'Package[moreutils]', 'Package[net-tools]', 'Package[numactl]', 'Package[ncdu]', 'Package[ngrep]', 'Package[pigz]', 'Package[psmisc]', 'Package[pv]', 'Package[python3]', 'Package[screen]', 'Package[strace]', 'Package[sysstat]', 'Package[tcpdump]', 'Package[tmux]', 'Package[tree]', 'Package[vim]', 'Package[vim-addon-manager]', 'Package[vim-scripts]', 'Package[wipe]', 'Package[xfsprogs]', 'Package[zsh]', 'Package[icdiff]', 'Package[linux-perf]', 'Package[bsd-mailx]', 'Package[ack]', 'Package[netcat-openbsd]', 'Package[tshark]', 'Package[fzf]', 'Package[ripgrep]', 'Package[fd-find]', 'Package[kitty-terminfo]', 'Package[mtr-tiny]', 'Package[bat]', 'Package[efibootmgr]', 'Package[bind9-dnsutils]', 'Package[tzdata]', 'Package[python3-wmflib]', 'Package[starship]', 'Package[ruby-sorted-set]', 'Package[btop]', 'Package[linux-sysctl-defaults]', 'Package[apport]', 'Package[command-not-found]', 'Package[command-not-found-data]', 'Package[ecryptfs-utils]', 'Package[mlocate]', 'Package[os-prober]', 'Package[python3-apport]', 'Package[wpasupplicant]', 'Package[atop]', 'Package[apt-listchanges]', 'Package[isc-dhcp-client]', 'Package[rasdaemon]', 'Package[openssh-client]', 'Package[openssh-server]', 'Package[debdeploy-client]', 'Package[python3-dateutil]', 'Package[sudo]', 'Package[golang-cfssl]', 'Package[debmonitor-client]', 'Package[nagios-nrpe-server]', 'Package[monitoring-plugins]', 'Package[monitoring-plugins-basic]', 'Package[monitoring-plugins-standard]', 'Package[liburiparser1]', 'Package[python3-attr]', 'Package[freeipmi-tools]', 'Package[freeipmi-ipmiseld]', 'Package[rsyslog-kafka]', 'Package[emacs-nox]', 'Package[prometheus-ipmi-exporter]', 'Package[libnet-dns-perl]', 'Package[iptables]', 'Package[ferm]', 'Package[ulogd2]', 'Package[conntrack]', 'Package[dragonfly-dfdaemon]', 'Package[dragonfly-dfget]', 'Package[crictl]', 'Package[containerd]', 'Package[nerdctl]', 'Package[rsyslog-kubernetes]', 'Package[linux-cpupower]', 'Package[apparmor]', 'Package[socat]', 'Package[amd-k8s-device-plugin]', 'Package[amd-k8s-node-labeller]', 'Package[rocm-smi]', 'Package[python3-requests]', 'Package[wikimedia-lvs-realserver]', 'Package[ruby-concurrent]', 'Package[ruby]', 'Package[libruby]', 'Package[puppet-agent]', 'Package[linux-image-6.16.3+deb13-amd64]', 'Package[prometheus-rsyslog-exporter]', 'Package[initramfs-tools]', 'Package[python3-click]', 'Package[python3-box]', 'Package[confd]', 'Package[python3-toml]', 'Package[kubernetes-node]', 'Package[calicoctl]', 'Package[calico-cni]', 'Package[istio-cni]', 'Package[firmware-amd-graphics]']\n"}, {"resource": "Kmod::Blacklist[wmf_overlay]", "parameters": "--- Kmod::Blacklist[wmf_overlay].orig\n+++ Kmod::Blacklist[wmf_overlay]\n\n@@\n-    ensure  => present\n+    ensure  => absent\n@@\n-    modules => ['overlayfs', 'overlay']\n+    modules => []\n"}, {"resource": "Nrpe::Check[check_disk_space]", "parameters": "--- Nrpe::Check[check_disk_space].orig\n+++ Nrpe::Check[check_disk_space]\n\n@@\n-    command => /usr/lib/nagios/plugins/check_disk -w 6% -c 3% -W 6% -K 3% -l -e -A -i \"/srv/sd[a-b][1-3]\" -i \"/srv/nvme[0-9]n[0-9]p[0-9]\" --exclude-type=fuse --exclude-type=fuse.fuse_dfs --exclude-type=tracefs\n+    command => /usr/lib/nagios/plugins/check_disk -w 10% -c 5% -W 6% -K 3% -l -e -A -i '/(var/lib|run)/(containerd|kubelet)/*' --exclude-type=tracefs\n"}, {"resource": "File[/etc/cfssl/csr/mlserve__istio-cni.csr]", "content": "--- /etc/cfssl/csr/mlserve__istio-cni.csr.orig\n+++ /etc/cfssl/csr/mlserve__istio-cni.csr\n@@ -0,0 +1,13 @@\n+{\n+  \"CN\": \"istio-cni\",\n+  \"hosts\": [\n+    \"istio-cni\"\n+  ],\n+  \"key\": {\n+    \"algo\": \"ecdsa\",\n+    \"size\": 256\n+  },\n+  \"names\": [\n+\n+  ]\n+}", "parameters": "--- File[/etc/cfssl/csr/mlserve__istio-cni.csr].orig\n+++ File[/etc/cfssl/csr/mlserve__istio-cni.csr]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "User[kube]", "parameters": "--- User[kube].orig\n+++ User[kube]\n\n+    system => True\n+    shell  => /usr/sbin/nologin\n+    gid    => kube\n+    home   => /nonexistent\n+    ensure => present\n"}, {"resource": "File[/etc/systemd/system/ferm.service.d/ferm-service-auto-restart.conf]", "parameters": "--- File[/etc/systemd/system/ferm.service.d/ferm-service-auto-restart.conf].orig\n+++ File[/etc/systemd/system/ferm.service.d/ferm-service-auto-restart.conf]\n\n+    group  => root\n+    source => puppet:///modules/profile/kubernetes/node/ferm_systemd_override\n+    mode   => 0444\n+    ensure => present\n+    notify => Exec[systemd daemon-reload for ferm.service (ferm-ferm-service-auto-restart)]\n+    owner  => root\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__calico-cni.csr]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__calico-cni.csr].orig\n+++ File[/etc/kubernetes/pki/mlserve__calico-cni.csr]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "Cfssl::Csr[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]", "parameters": "--- Cfssl::Csr[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr].orig\n+++ Cfssl::Csr[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]\n\n+    hosts       => []\n+    key         => {'algo': 'ecdsa', 'size': 256}\n+    common_name => amdgpu-node-labeller\n+    ensure      => present\n+    names       => []\n"}, {"resource": "Service[dragonfly-dfdaemon]", "parameters": "--- Service[dragonfly-dfdaemon].orig\n+++ Service[dragonfly-dfdaemon]\n\n+    ensure => running\n"}, {"resource": "Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.timer (rsyslog-release-deleted-inotify-watches.timer)]", "parameters": "--- Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.timer (rsyslog-release-deleted-inotify-watches.timer)].orig\n+++ Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.timer (rsyslog-release-deleted-inotify-watches.timer)]\n\n+    command     => /bin/systemctl daemon-reload\n+    refreshonly => True\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]\n\n+    group  => root\n+    source => puppet:///modules/profile/pki/intermediates/mlserve-cert.pem\n+    mode   => 0440\n+    ensure => file\n+    owner  => kube\n"}, {"resource": "K8s::Kubeconfig[/etc/kubernetes/proxy.conf]", "parameters": "--- K8s::Kubeconfig[/etc/kubernetes/proxy.conf].orig\n+++ K8s::Kubeconfig[/etc/kubernetes/proxy.conf]\n\n+    username    => default-proxy\n+    group       => kube\n+    mode        => 0400\n+    owner       => kube\n+    require     => ['Class[K8s::Base_dirs]']\n+    auth_cert   => {'cert': '/etc/kubernetes/pki/mlserve__system_kube-proxy.pem', 'key': '/etc/kubernetes/pki/mlserve__system_kube-proxy-key.pem', 'chain': '/etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem', 'chained': '/etc/kubernetes/pki/mlserve__system_kube-proxy.chained.pem'}\n+    ensure      => present\n+    master_host => ml-ctrl.svc.eqiad.wmnet\n"}, {"resource": "Apt::Repository[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "parameters": "--- Apt::Repository[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Apt::Repository[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    trust_repo               => False\n+    components               => component/istio115\n+    dist                     => trixie-wikimedia\n+    keyfile                  => puppet:///modules/install_server/autoinstall/keyring/wikimedia-archive-keyring.gpg\n+    uri                      => http://apt.wikimedia.org/wikimedia\n+    allow_releaseinfo_change => False\n+    bin                      => True\n+    source                   => True\n+    ensure                   => present\n"}, {"resource": "Package[nerdctl]", "parameters": "--- Package[nerdctl].orig\n+++ Package[nerdctl]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "Motd::Script[ml_k8s::insetup_gpu]", "parameters": "--- Motd::Script[ml_k8s::insetup_gpu].orig\n+++ Motd::Script[ml_k8s::insetup_gpu]\n\n-    priority => 5\n-    ensure   => present\n"}, {"resource": "Service[containerd]", "parameters": "--- Service[containerd].orig\n+++ Service[containerd]\n\n+    ensure => running\n"}, {"resource": "Apt::Repository[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "parameters": "--- Apt::Repository[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia].orig\n+++ Apt::Repository[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]\n\n+    trust_repo               => False\n+    components               => component/kubernetes131\n+    dist                     => trixie-wikimedia\n+    keyfile                  => puppet:///modules/install_server/autoinstall/keyring/wikimedia-archive-keyring.gpg\n+    uri                      => http://apt.wikimedia.org/wikimedia\n+    allow_releaseinfo_change => False\n+    bin                      => True\n+    source                   => True\n+    ensure                   => present\n"}, {"resource": "K8s::Kubeconfig[/etc/calico/calicoctl-kubeconfig]", "parameters": "--- K8s::Kubeconfig[/etc/calico/calicoctl-kubeconfig].orig\n+++ K8s::Kubeconfig[/etc/calico/calicoctl-kubeconfig]\n\n+    username    => calicoctl\n+    group       => root\n+    mode        => 0400\n+    owner       => root\n+    require     => ['Class[K8s::Base_dirs]']\n+    auth_cert   => {'cert': '/etc/kubernetes/pki/mlserve__calicoctl.pem', 'key': '/etc/kubernetes/pki/mlserve__calicoctl-key.pem', 'chain': '/etc/kubernetes/pki/mlserve__calicoctl.chain.pem', 'chained': '/etc/kubernetes/pki/mlserve__calicoctl.chained.pem'}\n+    ensure      => present\n+    master_host => ml-ctrl.svc.eqiad.wmnet\n"}, {"resource": "Class[Toil::Rsyslog_imfile_remedy]", "parameters": "--- Class[Toil::Rsyslog_imfile_remedy].orig\n+++ Class[Toil::Rsyslog_imfile_remedy]\n\n+    period_hours => 3\n+    ensure       => present\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem]\n\n+    mode   => 0440\n+    group  => root\n+    ensure => file\n+    owner  => kube\n"}, {"resource": "File[/etc/apparmor.d/abstractions]", "parameters": "--- File[/etc/apparmor.d/abstractions].orig\n+++ File[/etc/apparmor.d/abstractions]\n\n+    require => Package[apparmor]\n+    group   => root\n+    mode    => 0755\n+    ensure  => directory\n+    owner   => root\n"}, {"resource": "File[/etc/cni/net.d/istio-kubeconfig]", "content": "--- /etc/cni/net.d/istio-kubeconfig.orig\n+++ /etc/cni/net.d/istio-kubeconfig\n@@ -0,0 +1,18 @@\n+apiVersion: v1\n+kind: Config\n+preferences: {}\n+current-context: default-system\n+contexts:\n+- name: default-system\n+  context:\n+    cluster: default-cluster\n+    user: istio-cni\n+clusters:\n+- name: default-cluster\n+  cluster:\n+    server: https://ml-ctrl.svc.eqiad.wmnet:6443\n+users:\n+- name: istio-cni\n+  user:\n+    client-certificate: /etc/kubernetes/pki/mlserve__istio-cni.pem\n+    client-key: /etc/kubernetes/pki/mlserve__istio-cni-key.pem", "parameters": "--- File[/etc/cni/net.d/istio-kubeconfig].orig\n+++ File[/etc/cni/net.d/istio-kubeconfig]\n\n+    mode   => 0400\n+    group  => root\n+    ensure => present\n+    owner  => root\n"}, {"resource": "Apt::Pin[apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo]", "parameters": "--- Apt::Pin[apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo].orig\n+++ Apt::Pin[apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo]\n\n+    priority => 1001\n+    before   => ['Package[firmware-amd-graphics]']\n+    pin      => release a=trixie-backports\n+    ensure   => present\n+    notify   => Exec[exec-apt-get-update-firmware-amd-graphics-trixie-bpo_trixie-bpo]\n+    package  => firmware-amd-graphics\n"}, {"resource": "File[/etc/calico/calicoctl.cfg]", "content": "--- /etc/calico/calicoctl.cfg.orig\n+++ /etc/calico/calicoctl.cfg\n@@ -0,0 +1,10 @@\n+# This configures calicoctl to use the kubernetes datastore.\n+# The user referenced in the kubeconfig file probably needs broad permissions, see:\n+# https://docs.projectcalico.org/getting-started/clis/calicoctl/configure/overview\n+# https://docs.projectcalico.org/getting-started/kubernetes/hardway/end-user-rbac\n+apiVersion: projectcalico.org/v3\n+kind: CalicoAPIConfig\n+metadata:\n+spec:\n+  datastoreType: \"kubernetes\"\n+  kubeconfig: \"/etc/calico/calicoctl-kubeconfig\"", "parameters": "--- File[/etc/calico/calicoctl.cfg].orig\n+++ File[/etc/calico/calicoctl.cfg]\n\n+    mode   => 0444\n+    group  => root\n+    ensure => file\n+    owner  => root\n"}, {"resource": "File[/etc/ferm/conf.d/10_calico-bird]", "content": "--- /etc/ferm/conf.d/10_calico-bird.orig\n+++ /etc/ferm/conf.d/10_calico-bird\n@@ -0,0 +1,6 @@\n+# Autogenerated by puppet. DO NOT EDIT BY HAND!\n+#\n+# \n+&R_SERVICE(tcp, 179, ($NETWORK_INFRA 10.64.167.1));\n+\n+", "parameters": "--- File[/etc/ferm/conf.d/10_calico-bird].orig\n+++ File[/etc/ferm/conf.d/10_calico-bird]\n\n+    tag     => ferm\n+    require => File[/etc/ferm/conf.d]\n+    group   => root\n+    mode    => 0400\n+    ensure  => present\n+    notify  => Service[ferm]\n+    owner   => root\n"}, {"resource": "Service[rsyslog-imfile-remedy.timer]", "parameters": "--- Service[rsyslog-imfile-remedy.timer].orig\n+++ Service[rsyslog-imfile-remedy.timer]\n\n+    enable   => True\n+    ensure   => running\n+    provider => systemd\n"}, {"resource": "File[/etc/calico]", "parameters": "--- File[/etc/calico].orig\n+++ File[/etc/calico]\n\n+    mode   => 0755\n+    group  => root\n+    ensure => directory\n+    owner  => root\n"}, {"resource": "Exec[cpupower_reload]", "parameters": "--- Exec[cpupower_reload].orig\n+++ Exec[cpupower_reload]\n\n+    command => /usr/bin/systemctl restart cpupower\n+    unless  => /usr/bin/cpupower frequency-info -p | /bin/grep -wq performance\n+    require => File[/etc/default/cpupower]\n"}, {"resource": "Exec[Generate cert mlserve__amdgpu-node-labeller refresh on intermediate ca change]", "parameters": "--- Exec[Generate cert mlserve__amdgpu-node-labeller refresh on intermediate ca change].orig\n+++ Exec[Generate cert mlserve__amdgpu-node-labeller refresh on intermediate ca change]\n\n+    command     => /usr/bin/cfssl gencert -config /etc/cfssl/client-cfssl.conf -tls-remote-ca /etc/ssl/certs/wmf-ca-certificates.crt -mutual-tls-client-cert /etc/cfssl/mutual_tls_client_cert.pem -mutual-tls-client-key /var/lib/puppet/ssl/private_keys/ml-serve1015.eqiad.wmnet.pem -label mlserve  /etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr | /usr/bin/cfssljson -bare /etc/kubernetes/pki/mlserve__amdgpu-node-labeller\n\n+    environment => ['GODEBUG=x509ignoreCN=0']\n+    refreshonly => True\n+    subscribe   => File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chained.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chained.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chained.pem]\n\n+    group   => root\n+    ensure  => file\n+    require => Exec[create chained cert /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]\n+    owner   => kube\n"}, {"resource": "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog-key.pem]", "parameters": "--- File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog-key.pem].orig\n+++ File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog-key.pem]\n\n+    show_diff => False\n+    backup    => False\n+    group     => root\n+    mode      => 0440\n+    ensure    => file\n+    owner     => root\n"}, {"resource": "Apt::Package_from_component[kubernetes131]", "parameters": "--- Apt::Package_from_component[kubernetes131].orig\n+++ Apt::Package_from_component[kubernetes131]\n\n+    distro          => trixie-wikimedia\n+    component       => component/kubernetes131\n+    ensure_packages => True\n+    packages        => []\n+    priority        => 1001\n+    uri             => http://apt.wikimedia.org/wikimedia\n+    ensure          => present\n"}, {"resource": "Package[containerd]", "parameters": "--- Package[containerd].orig\n+++ Package[containerd]\n\n+    ensure   => installed\n+    provider => apt\n"}, {"resource": "File[/etc/rsyslog.d/00-imfile.conf]", "content": "--- /etc/rsyslog.d/00-imfile.conf.orig\n+++ /etc/rsyslog.d/00-imfile.conf\n@@ -0,0 +1 @@\n+module(load=\"imfile\")", "parameters": "--- File[/etc/rsyslog.d/00-imfile.conf].orig\n+++ File[/etc/rsyslog.d/00-imfile.conf]\n\n+    group  => root\n+    mode   => 0444\n+    ensure => present\n+    notify => Service[rsyslog]\n+    owner  => root\n"}, {"resource": "K8s::Package[kubelet]", "parameters": "--- K8s::Package[kubelet].orig\n+++ K8s::Package[kubelet]\n\n+    distro          => trixie-wikimedia\n+    require         => ['Class[K8s::Base_dirs]']\n+    ensure_packages => True\n+    priority        => 1001\n+    uri             => http://apt.wikimedia.org/wikimedia\n+    version         => 1.31\n+    package         => node\n"}, {"resource": "File[/etc/default/kube-proxy]", "content": "--- /etc/default/kube-proxy.orig\n+++ /etc/default/kube-proxy\n@@ -0,0 +1,7 @@\n+###\n+# Kubernetes proxy config.\n+\n+# default config should be adequate\n+\n+DAEMON_ARGS=\"--config=/etc/kubernetes/kube-proxy-config.yaml \\\n+ --v=0\"", "parameters": "--- File[/etc/default/kube-proxy].orig\n+++ File[/etc/default/kube-proxy]\n\n+    group  => root\n+    mode   => 0644\n+    ensure => file\n+    notify => Service[kube-proxy]\n+    owner  => root\n"}, {"resource": "Exec[exec-apt-get-update-firmware-amd-graphics-trixie-bpo_trixie-bpo]", "parameters": "--- Exec[exec-apt-get-update-firmware-amd-graphics-trixie-bpo_trixie-bpo].orig\n+++ Exec[exec-apt-get-update-firmware-amd-graphics-trixie-bpo_trixie-bpo]\n\n+    command     => /usr/bin/apt-get update\n+    refreshonly => True\n"}, {"resource": "Class[K8s::Proxy]", "parameters": "--- Class[K8s::Proxy].orig\n+++ Class[K8s::Proxy]\n\n+    cluster_cidr  => {'v4': '10.67.16.0/21', 'v6': '2620:0:861:300::/64'}\n+    proxy_mode    => iptables\n+    v_log_level   => 0\n+    version       => 1.31\n+    ipv6dualstack => False\n+    kubeconfig    => /etc/kubernetes/proxy.conf\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__kubelet_server.chained.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__kubelet_server.chained.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__kubelet_server.chained.pem]\n\n+    group   => root\n+    ensure  => file\n+    require => Exec[create chained cert /etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]\n+    owner   => kube\n"}, {"resource": "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet-key.pem]", "parameters": "--- File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet-key.pem].orig\n+++ File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet-key.pem]\n\n+    show_diff => False\n+    backup    => False\n+    group     => root\n+    mode      => 0440\n+    ensure    => file\n+    owner     => kube\n"}], "perc_changed": "25.20%"}, "core": {"total": 2865, "only_in_self": ["File[/etc/update-motd.d/05-ml-k8s--insetup-gpu]", "Node[__node_regexp__ml-serve10145.eqiad.]"], "only_in_other": ["Concat[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_file[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_file[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_file[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[/sbin/modprobe overlay]", "Exec[/usr/sbin/dpkg-reconfigure -p critical -f noninteractive wikimedia-lvs-realserver]", "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh on intermediate ca change]", "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh]", "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet]", "Exec[Generate cert mlserve__amdgpu-node-labeller refresh on intermediate ca change]", "Exec[Generate cert mlserve__amdgpu-node-labeller refresh]", "Exec[Generate cert mlserve__amdgpu-node-labeller]", "Exec[Generate cert mlserve__calico-cni refresh on intermediate ca change]", "Exec[Generate cert mlserve__calico-cni refresh]", "Exec[Generate cert mlserve__calico-cni]", "Exec[Generate cert mlserve__calicoctl refresh on intermediate ca change]", "Exec[Generate cert mlserve__calicoctl refresh]", "Exec[Generate cert mlserve__calicoctl]", "Exec[Generate cert mlserve__istio-cni refresh on intermediate ca change]", "Exec[Generate cert mlserve__istio-cni refresh]", "Exec[Generate cert mlserve__istio-cni]", "Exec[Generate cert mlserve__kubelet_server refresh on intermediate ca change]", "Exec[Generate cert mlserve__kubelet_server refresh]", "Exec[Generate cert mlserve__kubelet_server]", "Exec[Generate cert mlserve__rsyslog refresh on intermediate ca change]", "Exec[Generate cert mlserve__rsyslog refresh]", "Exec[Generate cert mlserve__rsyslog]", "Exec[Generate cert mlserve__system_kube-proxy refresh on intermediate ca change]", "Exec[Generate cert mlserve__system_kube-proxy refresh]", "Exec[Generate cert mlserve__system_kube-proxy]", "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh on intermediate ca change]", "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh]", "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet]", "Exec[apt_package_from_component_calico329]", "Exec[apt_package_from_component_istio115]", "Exec[apt_package_from_component_kubernetes131]", "Exec[apt_pin_apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo]", "Exec[apt_pin_apt_pin_linux-6.16-trixie_trixie-bpo]", "Exec[apt_repository_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[apt_repository_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[apt_repository_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[cpupower_reload]", "Exec[create chained cert /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]", "Exec[create chained cert /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__calico-cni.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__calicoctl.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__istio-cni.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]", "Exec[exec-apt-get-update-firmware-amd-graphics-trixie-bpo_trixie-bpo]", "Exec[exec-apt-get-update-linux-6.16-trixie_trixie-bpo]", "Exec[renew certificate - discovery__ml-serve1015_eqiad_wmnet]", "Exec[renew certificate - mlserve__amdgpu-node-labeller]", "Exec[renew certificate - mlserve__calico-cni]", "Exec[renew certificate - mlserve__calicoctl]", "Exec[renew certificate - mlserve__istio-cni]", "Exec[renew certificate - mlserve__kubelet_server]", "Exec[renew certificate - mlserve__rsyslog]", "Exec[renew certificate - mlserve__system_kube-proxy]", "Exec[renew certificate - mlserve__system_node_ml-serve1015_eqiad_wmnet]", "Exec[systemd daemon-reload for amd-k8s-node-labeller.service (amd-k8s-node-labeller-amd-devplugin-after-labeller)]", "Exec[systemd daemon-reload for cpupower.service (cpupower)]", "Exec[systemd daemon-reload for ferm.service (ferm-ferm-service-auto-restart)]", "Exec[systemd daemon-reload for kube-proxy.service (kube-proxy)]", "Exec[systemd daemon-reload for kubelet.service (kubelet-container-runtime)]", "Exec[systemd daemon-reload for rsyslog-imfile-remedy.service (rsyslog-imfile-remedy.service)]", "Exec[systemd daemon-reload for rsyslog-imfile-remedy.timer (rsyslog-imfile-remedy.timer)]", "Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.service (rsyslog-release-deleted-inotify-watches.service)]", "Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.timer (rsyslog-release-deleted-inotify-watches.timer)]", "File[/etc/amd/node-labeller-kubeconfig]", "File[/etc/amd]", "File[/etc/apparmor.d/abstractions]", "File[/etc/apt/preferences.d/apt_pin_firmware_amd_graphics_trixie_bpo_trixie_bpo.pref]", "File[/etc/apt/preferences.d/apt_pin_linux_6_16_trixie_trixie_bpo.pref]", "File[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "File[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "File[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "File[/etc/calico/calicoctl-kubeconfig]", "File[/etc/calico/calicoctl.cfg]", "File[/etc/calico/pki]", "File[/etc/calico]", "File[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]", "File[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]", "File[/etc/cfssl/csr/mlserve__calico-cni.csr]", "File[/etc/cfssl/csr/mlserve__calicoctl.csr]", "File[/etc/cfssl/csr/mlserve__istio-cni.csr]", "File[/etc/cfssl/csr/mlserve__kubelet_server.csr]", "File[/etc/cfssl/csr/mlserve__rsyslog.csr]", "File[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]", "File[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog-key.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chained.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.csr]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog]", "File[/etc/cni/net.d/10-calico.conflist]", "File[/etc/cni/net.d/calico-kubeconfig]", "File[/etc/cni/net.d/istio-kubeconfig]", "File[/etc/cni/net.d]", "File[/etc/cni]", "File[/etc/containerd/config.toml]", "File[/etc/containerd]", "File[/etc/default/cpupower]", "File[/etc/default/kube-proxy]", "File[/etc/default/kubelet]", "File[/etc/default/wikimedia-lvs-realserver]", "File[/etc/dragonfly/dfdaemon.yml]", "File[/etc/dragonfly/dfget.yml]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.csr]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem]", "File[/etc/dragonfly]", "File[/etc/ferm/conf.d/10_calico-bird]", "File[/etc/ferm/conf.d/10_calico_typha]", "File[/etc/ferm/conf.d/10_dragonfly_dfget]", "File[/etc/ferm/conf.d/10_kubelet-http]", "File[/etc/kubernetes/kube-proxy-config.yaml]", "File[/etc/kubernetes/kubelet-config.yaml]", "File[/etc/kubernetes/kubelet.conf]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller-key.pem]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chained.pem]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.csr]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni-key.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni.chain.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni.chained.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni.csr]", "File[/etc/kubernetes/pki/mlserve__calico-cni.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl-key.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl.chain.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl.chained.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl.csr]", "File[/etc/kubernetes/pki/mlserve__calicoctl.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni-key.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni.chain.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni.chained.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni.csr]", "File[/etc/kubernetes/pki/mlserve__istio-cni.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server-key.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.chained.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.csr]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy-key.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chained.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.csr]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet-key.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chained.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem]", "File[/etc/kubernetes/pki]", "File[/etc/kubernetes/proxy.conf]", "File[/etc/kubernetes]", "File[/etc/logrotate.d/rsyslog-release-deleted-inotify-watches]", "File[/etc/modules-load.d/overlay.conf]", "File[/etc/nerdctl/nerdctl.toml]", "File[/etc/nerdctl]", "File[/etc/rsyslog.d/00-imfile.conf]", "File[/etc/rsyslog.d/08-input-file-kubernetes-json.conf]", "File[/etc/rsyslog.d/09-kubernetes.conf]", "File[/etc/rsyslog.d/10-kubernetes-node-filters.conf]", "File[/etc/rsyslog.d/20-shellbox.conf]", "File[/etc/rsyslog.d/35-output-kafka-k8s.conf]", "File[/etc/rsyslog.d/40-rsyslog-release-deleted-inotify-watches.conf]", "File[/etc/sysctl.d/70-increase_inotify_limits.conf]", "File[/etc/sysctl.d/70-ipv6-fowarding-accept-ra.conf]", "File[/etc/sysctl.d/75-kube_proxy_conntrack.conf]", "File[/etc/sysctl.d/75-kube_proxy_icmp.conf]", "File[/etc/systemd/system/amd-k8s-node-labeller.service.d/amd-devplugin-after-labeller.conf]", "File[/etc/systemd/system/amd-k8s-node-labeller.service.d]", "File[/etc/systemd/system/ferm.service.d/ferm-service-auto-restart.conf]", "File[/etc/systemd/system/kube-proxy.service.d/puppet-override.conf]", "File[/etc/systemd/system/kube-proxy.service.d]", "File[/etc/systemd/system/kubelet.service.d/container-runtime.conf]", "File[/etc/systemd/system/kubelet.service.d]", "File[/etc/udev/rules.d/70-kfd.rules]", "File[/etc/udev/rules.d/70-render.rules]", "File[/etc/udev/rules.d/75-kube_proxy_conntrack.rules]", "File[/etc/update-motd.d/05-ml-k8s--worker]", "File[/lib/systemd/system/cpupower.service]", "File[/lib/systemd/system/rsyslog-imfile-remedy.service]", "File[/lib/systemd/system/rsyslog-imfile-remedy.timer]", "File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.service]", "File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.timer]", "File[/usr/libexec/cpupower]", "File[/usr/local/sbin/rsyslog-release-deleted-inotify-watches]", "File[/var/lib/kubelet/config.json]", "File[/var/lib/kubelet]", "File[/var/log/rsyslog-release-deleted-inotify-watches]", "File[/var/run/kubernetes]", "Group[kube]", "Node[__node_regexp__ml-serve1001-91012345.eqiad.]", "Package[amd-k8s-device-plugin]", "Package[amd-k8s-node-labeller]", "Package[apparmor]", "Package[calico-cni]", "Package[calicoctl]", "Package[containerd]", "Package[crictl]", "Package[dragonfly-dfdaemon]", "Package[dragonfly-dfget]", "Package[istio-cni]", "Package[kubernetes-node]", "Package[linux-cpupower]", "Package[linux-image-6.16.3+deb13-amd64]", "Package[nerdctl]", "Package[rsyslog-kubernetes]", "Package[socat]", "Package[wikimedia-lvs-realserver]", "Service[apparmor]", "Service[containerd]", "Service[cpupower]", "Service[dragonfly-dfdaemon]", "Service[kube-proxy]", "Service[kubelet]", "Service[rsyslog-imfile-remedy.timer]", "Service[rsyslog-release-deleted-inotify-watches.timer]", "User[kube]"], "resource_diffs": [{"resource": "File[/etc/nagios/nrpe.d/check_disk_space.cfg]", "content": "--- /etc/nagios/nrpe.d/check_disk_space.cfg.orig\n+++ /etc/nagios/nrpe.d/check_disk_space.cfg\n@@ -1,2 +1,2 @@\n # File generated by puppet. DO NOT edit by hand\n-command[check_disk_space]=/usr/lib/nagios/plugins/check_disk -w 6% -c 3% -W 6% -K 3% -l -e -A -i \"/srv/sd[a-b][1-3]\" -i \"/srv/nvme[0-9]n[0-9]p[0-9]\" --exclude-type=fuse --exclude-type=fuse.fuse_dfs --exclude-type=tracefs\n+command[check_disk_space]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -W 6% -K 3% -l -e -A -i '/(var/lib|run)/(containerd|kubelet)/*' --exclude-type=tracefs"}, {"resource": "File[/etc/modprobe.d/blacklist-wmf_overlay.conf]", "content": "--- /etc/modprobe.d/blacklist-wmf_overlay.conf.orig\n+++ /etc/modprobe.d/blacklist-wmf_overlay.conf\n@@ -1,7 +1,3 @@\n # wmf_overlay - blacklisted kernel modules\n # This file is managed by Puppet\n #\n-blacklist overlay\n-install overlay /bin/true\n-blacklist overlayfs\n-install overlayfs /bin/true", "parameters": "--- File[/etc/modprobe.d/blacklist-wmf_overlay.conf].orig\n+++ File[/etc/modprobe.d/blacklist-wmf_overlay.conf]\n\n@@\n-    ensure => present\n+    ensure => absent\n"}, {"resource": "File[/etc/default/prometheus-node-exporter]", "content": "--- /etc/default/prometheus-node-exporter.orig\n+++ /etc/default/prometheus-node-exporter\n@@ -15,6 +15,7 @@\n  --collector.netdev \\\n  --collector.netstat \\\n  --collector.netstat.fields=^(.*) \\\n+ --collector.processes \\\n  --collector.sockstat \\\n  --collector.stat \\\n  --collector.systemd.enable-restarts-metrics \\"}, {"resource": "File[/var/lib/prometheus/node.d/role_owner.prom]", "content": "--- /var/lib/prometheus/node.d/role_owner.prom.orig\n+++ /var/lib/prometheus/node.d/role_owner.prom\n@@ -1,3 +1,3 @@\n # HELP role_owner The team owner of the server role\n # TYPE role_owner gauge\n-role_owner{team=\"machine-learning\",role=\"ml_k8s::insetup_gpu\",cluster=\"ml_serve\"} 1.0\n+role_owner{team=\"machine-learning\",role=\"ml_k8s::worker\",cluster=\"ml_serve\"} 1.0"}, {"resource": "Package[firmware-amd-graphics]", "parameters": "--- Package[firmware-amd-graphics].orig\n+++ Package[firmware-amd-graphics]\n\n@@\n-    ensure => installed\n+    ensure => 20251021-1~bpo13+1\n"}, {"resource": "Concat_fragment[main contacts]", "content": "--- main contacts.orig\n+++ main contacts\n@@ -1,3 +1,3 @@\n ---\n-role::ml_k8s::insetup_gpu:\n+role::ml_k8s::worker:\n - Machine Learning"}], "perc_changed": "8.48%"}, "main": {"total": 2865, "only_in_self": ["Class[Role::Ml_k8s::Insetup_gpu]", "File[/etc/update-motd.d/05-ml-k8s--insetup-gpu]", "Motd::Message[ml_k8s::insetup_gpu]", "Motd::Script[ml_k8s::insetup_gpu]", "Node[__node_regexp__ml-serve10145.eqiad.]"], "only_in_other": ["Apt::Package_from_bpo[firmware-amd-graphics-trixie-bpo]", "Apt::Package_from_bpo[linux-6.16-trixie]", "Apt::Package_from_component[calico329]", "Apt::Package_from_component[istio115]", "Apt::Package_from_component[kubernetes131]", "Apt::Pin[apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo]", "Apt::Pin[apt_pin_linux-6.16-trixie_trixie-bpo]", "Apt::Repository[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Apt::Repository[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Apt::Repository[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Cfssl::Cert[discovery__ml-serve1015_eqiad_wmnet]", "Cfssl::Cert[mlserve__amdgpu-node-labeller]", "Cfssl::Cert[mlserve__calico-cni]", "Cfssl::Cert[mlserve__calicoctl]", "Cfssl::Cert[mlserve__istio-cni]", "Cfssl::Cert[mlserve__kubelet_server]", "Cfssl::Cert[mlserve__rsyslog]", "Cfssl::Cert[mlserve__system_kube-proxy]", "Cfssl::Cert[mlserve__system_node_ml-serve1015_eqiad_wmnet]", "Cfssl::Csr[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__calico-cni.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__calicoctl.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__istio-cni.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__kubelet_server.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__rsyslog.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]", "Cfssl::Csr[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "Class[Apparmor]", "Class[Base::Sysctl::Inotify]", "Class[Calico]", "Class[Containerd::Configuration]", "Class[Containerd::Nerdctl]", "Class[Containerd]", "Class[Cpufrequtils]", "Class[Dragonfly::Dfdaemon]", "Class[K8s::Base_dirs]", "Class[K8s::Clusters]", "Class[K8s::Kubelet::Cni::Base]", "Class[K8s::Kubelet]", "Class[K8s::Proxy]", "Class[Lvs::Realserver]", "Class[Profile::Calico::Kubernetes]", "Class[Profile::Containerd]", "Class[Profile::Dragonfly::Dfdaemon]", "Class[Profile::Kubernetes::Container_runtime]", "Class[Profile::Kubernetes::Node]", "Class[Profile::Lvs::Configuration]", "Class[Profile::Lvs::Realserver]", "Class[Profile::Rsyslog::Kubernetes]", "Class[Profile::Rsyslog::Shellbox]", "Class[Role::Ml_k8s::Worker]", "Class[Toil::Rsyslog_imfile_remedy]", "Class[Wmflib::Service::Catalog]", "Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat::Fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat::Fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat::Fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_file[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_file[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_file[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.sources]", "Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat_fragment[component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat_fragment[component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia-header]", "Concat_fragment[component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Docker::Credentials[/var/lib/kubelet/config.json]", "Exec[/sbin/modprobe overlay]", "Exec[/usr/sbin/dpkg-reconfigure -p critical -f noninteractive wikimedia-lvs-realserver]", "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh on intermediate ca change]", "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet refresh]", "Exec[Generate cert discovery__ml-serve1015_eqiad_wmnet]", "Exec[Generate cert mlserve__amdgpu-node-labeller refresh on intermediate ca change]", "Exec[Generate cert mlserve__amdgpu-node-labeller refresh]", "Exec[Generate cert mlserve__amdgpu-node-labeller]", "Exec[Generate cert mlserve__calico-cni refresh on intermediate ca change]", "Exec[Generate cert mlserve__calico-cni refresh]", "Exec[Generate cert mlserve__calico-cni]", "Exec[Generate cert mlserve__calicoctl refresh on intermediate ca change]", "Exec[Generate cert mlserve__calicoctl refresh]", "Exec[Generate cert mlserve__calicoctl]", "Exec[Generate cert mlserve__istio-cni refresh on intermediate ca change]", "Exec[Generate cert mlserve__istio-cni refresh]", "Exec[Generate cert mlserve__istio-cni]", "Exec[Generate cert mlserve__kubelet_server refresh on intermediate ca change]", "Exec[Generate cert mlserve__kubelet_server refresh]", "Exec[Generate cert mlserve__kubelet_server]", "Exec[Generate cert mlserve__rsyslog refresh on intermediate ca change]", "Exec[Generate cert mlserve__rsyslog refresh]", "Exec[Generate cert mlserve__rsyslog]", "Exec[Generate cert mlserve__system_kube-proxy refresh on intermediate ca change]", "Exec[Generate cert mlserve__system_kube-proxy refresh]", "Exec[Generate cert mlserve__system_kube-proxy]", "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh on intermediate ca change]", "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet refresh]", "Exec[Generate cert mlserve__system_node_ml-serve1015_eqiad_wmnet]", "Exec[apt_package_from_component_calico329]", "Exec[apt_package_from_component_istio115]", "Exec[apt_package_from_component_kubernetes131]", "Exec[apt_pin_apt_pin_firmware-amd-graphics-trixie-bpo_trixie-bpo]", "Exec[apt_pin_apt_pin_linux-6.16-trixie_trixie-bpo]", "Exec[apt_repository_component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[apt_repository_component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[apt_repository_component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia]", "Exec[cpupower_reload]", "Exec[create chained cert /etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]", "Exec[create chained cert /etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__calico-cni.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__calicoctl.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__istio-cni.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]", "Exec[create chained cert /etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]", "Exec[exec-apt-get-update-firmware-amd-graphics-trixie-bpo_trixie-bpo]", "Exec[exec-apt-get-update-linux-6.16-trixie_trixie-bpo]", "Exec[renew certificate - discovery__ml-serve1015_eqiad_wmnet]", "Exec[renew certificate - mlserve__amdgpu-node-labeller]", "Exec[renew certificate - mlserve__calico-cni]", "Exec[renew certificate - mlserve__calicoctl]", "Exec[renew certificate - mlserve__istio-cni]", "Exec[renew certificate - mlserve__kubelet_server]", "Exec[renew certificate - mlserve__rsyslog]", "Exec[renew certificate - mlserve__system_kube-proxy]", "Exec[renew certificate - mlserve__system_node_ml-serve1015_eqiad_wmnet]", "Exec[systemd daemon-reload for amd-k8s-node-labeller.service (amd-k8s-node-labeller-amd-devplugin-after-labeller)]", "Exec[systemd daemon-reload for cpupower.service (cpupower)]", "Exec[systemd daemon-reload for ferm.service (ferm-ferm-service-auto-restart)]", "Exec[systemd daemon-reload for kube-proxy.service (kube-proxy)]", "Exec[systemd daemon-reload for kubelet.service (kubelet-container-runtime)]", "Exec[systemd daemon-reload for rsyslog-imfile-remedy.service (rsyslog-imfile-remedy.service)]", "Exec[systemd daemon-reload for rsyslog-imfile-remedy.timer (rsyslog-imfile-remedy.timer)]", "Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.service (rsyslog-release-deleted-inotify-watches.service)]", "Exec[systemd daemon-reload for rsyslog-release-deleted-inotify-watches.timer (rsyslog-release-deleted-inotify-watches.timer)]", "Ferm::Service[calico-bird]", "Ferm::Service[calico_typha]", "Ferm::Service[dragonfly_dfget]", "Ferm::Service[kubelet-http]", "File[/etc/amd/node-labeller-kubeconfig]", "File[/etc/amd]", "File[/etc/apparmor.d/abstractions]", "File[/etc/apt/preferences.d/apt_pin_firmware_amd_graphics_trixie_bpo_trixie_bpo.pref]", "File[/etc/apt/preferences.d/apt_pin_linux_6_16_trixie_trixie_bpo.pref]", "File[/etc/apt/sources.list.d/component-calico329-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "File[/etc/apt/sources.list.d/component-istio115-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "File[/etc/apt/sources.list.d/component-kubernetes131-apt.wikimedia.org-wikimedia-trixie-wikimedia.list]", "File[/etc/calico/calicoctl-kubeconfig]", "File[/etc/calico/calicoctl.cfg]", "File[/etc/calico/pki]", "File[/etc/calico]", "File[/etc/cfssl/csr/discovery__ml-serve1015_eqiad_wmnet.csr]", "File[/etc/cfssl/csr/mlserve__amdgpu-node-labeller.csr]", "File[/etc/cfssl/csr/mlserve__calico-cni.csr]", "File[/etc/cfssl/csr/mlserve__calicoctl.csr]", "File[/etc/cfssl/csr/mlserve__istio-cni.csr]", "File[/etc/cfssl/csr/mlserve__kubelet_server.csr]", "File[/etc/cfssl/csr/mlserve__rsyslog.csr]", "File[/etc/cfssl/csr/mlserve__system_kube-proxy.csr]", "File[/etc/cfssl/csr/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog-key.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chain.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.chained.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.csr]", "File[/etc/cfssl/ssl/mlserve__rsyslog/mlserve__rsyslog.pem]", "File[/etc/cfssl/ssl/mlserve__rsyslog]", "File[/etc/cni/net.d/10-calico.conflist]", "File[/etc/cni/net.d/calico-kubeconfig]", "File[/etc/cni/net.d/istio-kubeconfig]", "File[/etc/cni/net.d]", "File[/etc/cni]", "File[/etc/containerd/config.toml]", "File[/etc/containerd]", "File[/etc/default/cpupower]", "File[/etc/default/kube-proxy]", "File[/etc/default/kubelet]", "File[/etc/default/wikimedia-lvs-realserver]", "File[/etc/dragonfly/dfdaemon.yml]", "File[/etc/dragonfly/dfget.yml]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet-key.pem]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chain.pem]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.chained.pem]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.csr]", "File[/etc/dragonfly/discovery__ml-serve1015_eqiad_wmnet.pem]", "File[/etc/dragonfly]", "File[/etc/ferm/conf.d/10_calico-bird]", "File[/etc/ferm/conf.d/10_calico_typha]", "File[/etc/ferm/conf.d/10_dragonfly_dfget]", "File[/etc/ferm/conf.d/10_kubelet-http]", "File[/etc/kubernetes/kube-proxy-config.yaml]", "File[/etc/kubernetes/kubelet-config.yaml]", "File[/etc/kubernetes/kubelet.conf]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller-key.pem]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chain.pem]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.chained.pem]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.csr]", "File[/etc/kubernetes/pki/mlserve__amdgpu-node-labeller.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni-key.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni.chain.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni.chained.pem]", "File[/etc/kubernetes/pki/mlserve__calico-cni.csr]", "File[/etc/kubernetes/pki/mlserve__calico-cni.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl-key.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl.chain.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl.chained.pem]", "File[/etc/kubernetes/pki/mlserve__calicoctl.csr]", "File[/etc/kubernetes/pki/mlserve__calicoctl.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni-key.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni.chain.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni.chained.pem]", "File[/etc/kubernetes/pki/mlserve__istio-cni.csr]", "File[/etc/kubernetes/pki/mlserve__istio-cni.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server-key.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.chain.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.chained.pem]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.csr]", "File[/etc/kubernetes/pki/mlserve__kubelet_server.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy-key.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chain.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.chained.pem]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.csr]", "File[/etc/kubernetes/pki/mlserve__system_kube-proxy.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet-key.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chain.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.chained.pem]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.csr]", "File[/etc/kubernetes/pki/mlserve__system_node_ml-serve1015_eqiad_wmnet.pem]", "File[/etc/kubernetes/pki]", "File[/etc/kubernetes/proxy.conf]", "File[/etc/kubernetes]", "File[/etc/logrotate.d/rsyslog-release-deleted-inotify-watches]", "File[/etc/modules-load.d/overlay.conf]", "File[/etc/nerdctl/nerdctl.toml]", "File[/etc/nerdctl]", "File[/etc/rsyslog.d/00-imfile.conf]", "File[/etc/rsyslog.d/08-input-file-kubernetes-json.conf]", "File[/etc/rsyslog.d/09-kubernetes.conf]", "File[/etc/rsyslog.d/10-kubernetes-node-filters.conf]", "File[/etc/rsyslog.d/20-shellbox.conf]", "File[/etc/rsyslog.d/35-output-kafka-k8s.conf]", "File[/etc/rsyslog.d/40-rsyslog-release-deleted-inotify-watches.conf]", "File[/etc/sysctl.d/70-increase_inotify_limits.conf]", "File[/etc/sysctl.d/70-ipv6-fowarding-accept-ra.conf]", "File[/etc/sysctl.d/75-kube_proxy_conntrack.conf]", "File[/etc/sysctl.d/75-kube_proxy_icmp.conf]", "File[/etc/systemd/system/amd-k8s-node-labeller.service.d/amd-devplugin-after-labeller.conf]", "File[/etc/systemd/system/amd-k8s-node-labeller.service.d]", "File[/etc/systemd/system/ferm.service.d/ferm-service-auto-restart.conf]", "File[/etc/systemd/system/kube-proxy.service.d/puppet-override.conf]", "File[/etc/systemd/system/kube-proxy.service.d]", "File[/etc/systemd/system/kubelet.service.d/container-runtime.conf]", "File[/etc/systemd/system/kubelet.service.d]", "File[/etc/udev/rules.d/70-kfd.rules]", "File[/etc/udev/rules.d/70-render.rules]", "File[/etc/udev/rules.d/75-kube_proxy_conntrack.rules]", "File[/etc/update-motd.d/05-ml-k8s--worker]", "File[/lib/systemd/system/cpupower.service]", "File[/lib/systemd/system/rsyslog-imfile-remedy.service]", "File[/lib/systemd/system/rsyslog-imfile-remedy.timer]", "File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.service]", "File[/lib/systemd/system/rsyslog-release-deleted-inotify-watches.timer]", "File[/usr/libexec/cpupower]", "File[/usr/local/sbin/rsyslog-release-deleted-inotify-watches]", "File[/var/lib/kubelet/config.json]", "File[/var/lib/kubelet]", "File[/var/log/rsyslog-release-deleted-inotify-watches]", "File[/var/run/kubernetes]", "Firewall::Service[calico-typha]", "Firewall::Service[dragonfly_dfget]", "Group[kube]", "K8s::Kubeconfig[/etc/amd/node-labeller-kubeconfig]", "K8s::Kubeconfig[/etc/calico/calicoctl-kubeconfig]", "K8s::Kubeconfig[/etc/cni/net.d/calico-kubeconfig]", "K8s::Kubeconfig[/etc/cni/net.d/istio-kubeconfig]", "K8s::Kubeconfig[/etc/kubernetes/kubelet.conf]", "K8s::Kubeconfig[/etc/kubernetes/proxy.conf]", "K8s::Kubelet::Cni[calico]", "K8s::Package[kubelet]", "K8s::Package[proxy]", "Kmod::Module[overlay]", "Logrotate::Conf[rsyslog-release-deleted-inotify-watches]", "Motd::Message[ml_k8s::worker]", "Motd::Script[ml_k8s::worker]", "Node[__node_regexp__ml-serve1001-91012345.eqiad.]", "Package[amd-k8s-device-plugin]", "Package[amd-k8s-node-labeller]", "Package[apparmor]", "Package[calico-cni]", "Package[calicoctl]", "Package[containerd]", "Package[crictl]", "Package[dragonfly-dfdaemon]", "Package[dragonfly-dfget]", "Package[istio-cni]", "Package[kubernetes-node]", "Package[linux-cpupower]", "Package[linux-image-6.16.3+deb13-amd64]", "Package[nerdctl]", "Package[rsyslog-kubernetes]", "Package[socat]", "Package[wikimedia-lvs-realserver]", "Rsyslog::Conf[imfile]", "Rsyslog::Conf[input-file-kubernetes-json]", "Rsyslog::Conf[kubernetes-node-filters]", "Rsyslog::Conf[kubernetes]", "Rsyslog::Conf[output_kafka_k8s]", "Rsyslog::Conf[rsyslog-release-deleted-inotify-watches]", "Rsyslog::Conf[shellbox]", "Rsyslog::Input::File[kubernetes-json]", "Service[apparmor]", "Service[containerd]", "Service[cpupower]", "Service[dragonfly-dfdaemon]", "Service[kube-proxy]", "Service[kubelet]", "Service[rsyslog-imfile-remedy.timer]", "Service[rsyslog-release-deleted-inotify-watches.timer]", "Sysctl::Conffile[increase_inotify_limits]", "Sysctl::Conffile[ipv6-fowarding-accept-ra]", "Sysctl::Conffile[kube_proxy_conntrack]", "Sysctl::Conffile[kube_proxy_icmp]", "Sysctl::Parameters[increase_inotify_limits]", "Sysctl::Parameters[ipv6-fowarding-accept-ra]", "Sysctl::Parameters[kube_proxy_conntrack]", "Sysctl::Parameters[kube_proxy_icmp]", "Systemd::Override[amd-devplugin-after-labeller]", "Systemd::Override[container-runtime]", "Systemd::Override[ferm-service-auto-restart]", "Systemd::Service[cpupower]", "Systemd::Service[kube-proxy]", "Systemd::Service[rsyslog-imfile-remedy]", "Systemd::Service[rsyslog-release-deleted-inotify-watches]", "Systemd::Syslog[rsyslog-release-deleted-inotify-watches]", "Systemd::Timer::Job[rsyslog-imfile-remedy]", "Systemd::Timer::Job[rsyslog-release-deleted-inotify-watches]", "Systemd::Timer[rsyslog-imfile-remedy]", "Systemd::Timer[rsyslog-release-deleted-inotify-watches]", "Systemd::Unit[amd-k8s-node-labeller-amd-devplugin-after-labeller]", "Systemd::Unit[cpupower]", "Systemd::Unit[ferm-ferm-service-auto-restart]", "Systemd::Unit[kube-proxy]", "Systemd::Unit[kubelet-container-runtime]", "Systemd::Unit[rsyslog-imfile-remedy.service]", "Systemd::Unit[rsyslog-imfile-remedy.timer]", "Systemd::Unit[rsyslog-release-deleted-inotify-watches.service]", "Systemd::Unit[rsyslog-release-deleted-inotify-watches.timer]", "Udev::Rule[kube_proxy_conntrack]", "User[kube]"], "resource_diffs": [{"resource": "Class[Profile::Monitoring]", "parameters": "--- Class[Profile::Monitoring].orig\n+++ Class[Profile::Monitoring]\n\n@@\n-    nrpe_check_disk_options => -w 6% -c 3% -W 6% -K 3% -l -e -A -i \"/srv/sd[a-b][1-3]\" -i \"/srv/nvme[0-9]n[0-9]p[0-9]\" --exclude-type=fuse --exclude-type=fuse.fuse_dfs --exclude-type=tracefs\n+    nrpe_check_disk_options => -w 10% -c 5% -W 6% -K 3% -l -e -A -i '/(var/lib|run)/(containerd|kubelet)/*' --exclude-type=tracefs\n"}, {"resource": "File[/etc/nagios/nrpe.d/check_disk_space.cfg]", "content": "--- /etc/nagios/nrpe.d/check_disk_space.cfg.orig\n+++ /etc/nagios/nrpe.d/check_disk_space.cfg\n@@ -1,2 +1,2 @@\n # File generated by puppet. DO NOT edit by hand\n-command[check_disk_space]=/usr/lib/nagios/plugins/check_disk -w 6% -c 3% -W 6% -K 3% -l -e -A -i \"/srv/sd[a-b][1-3]\" -i \"/srv/nvme[0-9]n[0-9]p[0-9]\" --exclude-type=fuse --exclude-type=fuse.fuse_dfs --exclude-type=tracefs\n+command[check_disk_space]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -W 6% -K 3% -l -e -A -i '/(var/lib|run)/(containerd|kubelet)/*' --exclude-type=tracefs"}, {"resource": "Concat::Fragment[main contacts]"}, {"resource": "File[/etc/modprobe.d/blacklist-wmf_overlay.conf]", "content": "--- /etc/modprobe.d/blacklist-wmf_overlay.conf.orig\n+++ /etc/modprobe.d/blacklist-wmf_overlay.conf\n@@ -1,7 +1,3 @@\n # wmf_overlay - blacklisted kernel modules\n # This file is managed by Puppet\n #\n-blacklist overlay\n-install overlay /bin/true\n-blacklist overlayfs\n-install overlayfs /bin/true", "parameters": "--- File[/etc/modprobe.d/blacklist-wmf_overlay.conf].orig\n+++ File[/etc/modprobe.d/blacklist-wmf_overlay.conf]\n\n@@\n-    ensure => present\n+    ensure => absent\n"}, {"resource": "File[/etc/default/prometheus-node-exporter]", "content": "--- /etc/default/prometheus-node-exporter.orig\n+++ /etc/default/prometheus-node-exporter\n@@ -15,6 +15,7 @@\n  --collector.netdev \\\n  --collector.netstat \\\n  --collector.netstat.fields=^(.*) \\\n+ --collector.processes \\\n  --collector.sockstat \\\n  --collector.stat \\\n  --collector.systemd.enable-restarts-metrics \\"}, {"resource": "Class[Adduser]", "parameters": "--- Class[Adduser].orig\n+++ Class[Adduser]\n\n@@\n-    before => ['Package[puppet]', 'Package[facter]', 'Package[augeas-tools]', 'Package[virt-what]', 'Package[puppet-module-puppetlabs-augeas-core]', 'Package[python3-prometheus-client]', 'Package[python3-yaml]', 'Package[ruby-net-ssh]', 'Package[openssl]', 'Package[ssl-cert]', 'Package[ca-certificates]', 'Package[wmf-certificates]', 'Package[ntp]', 'Package[systemd-timesyncd]', 'Package[exim4-config]', 'Package[exim4-daemon-light]', 'Package[logrotate]', 'Package[prometheus-node-exporter]', 'Package[bsdutils]', 'Package[smartmontools]', 'Package[rsyslog]', 'Package[rsyslog-openssl]', 'Package[cadvisor]', 'Package[acct]', 'Package[byobu]', 'Package[colordiff]', 'Package[curl]', 'Package[debian-goodies]', 'Package[ethtool]', 'Package[gdb]', 'Package[gdisk]', 'Package[git]', 'Package[htop]', 'Package[httpry]', 'Package[iotop]', 'Package[iperf]', 'Package[jq]', 'Package[libtemplate-perl]', 'Package[lldpd]', 'Package[lshw]', 'Package[molly-guard]', 'Package[moreutils]', 'Package[net-tools]', 'Package[numactl]', 'Package[ncdu]', 'Package[ngrep]', 'Package[pigz]', 'Package[psmisc]', 'Package[pv]', 'Package[python3]', 'Package[screen]', 'Package[strace]', 'Package[sysstat]', 'Package[tcpdump]', 'Package[tmux]', 'Package[tree]', 'Package[vim]', 'Package[vim-addon-manager]', 'Package[vim-scripts]', 'Package[wipe]', 'Package[xfsprogs]', 'Package[zsh]', 'Package[icdiff]', 'Package[linux-perf]', 'Package[bsd-mailx]', 'Package[ack]', 'Package[netcat-openbsd]', 'Package[tshark]', 'Package[fzf]', 'Package[ripgrep]', 'Package[fd-find]', 'Package[kitty-terminfo]', 'Package[mtr-tiny]', 'Package[bat]', 'Package[efibootmgr]', 'Package[bind9-dnsutils]', 'Package[tzdata]', 'Package[python3-wmflib]', 'Package[starship]', 'Package[ruby-sorted-set]', 'Package[btop]', 'Package[linux-sysctl-defaults]', 'Package[apport]', 'Package[command-not-found]', 'Package[command-not-found-data]', 'Package[ecryptfs-utils]', 'Package[mlocate]', 'Package[os-prober]', 'Package[python3-apport]', 'Package[wpasupplicant]', 'Package[atop]', 'Package[apt-listchanges]', 'Package[isc-dhcp-client]', 'Package[rasdaemon]', 'Package[openssh-client]', 'Package[openssh-server]', 'Package[debdeploy-client]', 'Package[python3-dateutil]', 'Package[sudo]', 'Package[golang-cfssl]', 'Package[debmonitor-client]', 'Package[nagios-nrpe-server]', 'Package[monitoring-plugins]', 'Package[monitoring-plugins-basic]', 'Package[monitoring-plugins-standard]', 'Package[liburiparser1]', 'Package[python3-attr]', 'Package[freeipmi-tools]', 'Package[freeipmi-ipmiseld]', 'Package[rsyslog-kafka]', 'Package[emacs-nox]', 'Package[prometheus-ipmi-exporter]', 'Package[libnet-dns-perl]', 'Package[iptables]', 'Package[ferm]', 'Package[ulogd2]', 'Package[conntrack]', 'Package[rocm-smi]', 'Package[python3-requests]', 'Package[firmware-amd-graphics]', 'Package[ruby-concurrent]', 'Package[ruby]', 'Package[libruby]', 'Package[puppet-agent]', 'Package[prometheus-rsyslog-exporter]', 'Package[initramfs-tools]', 'Package[python3-click]', 'Package[python3-box]', 'Package[confd]', 'Package[python3-toml]']\n+    before => ['Package[puppet]', 'Package[facter]', 'Package[augeas-tools]', 'Package[virt-what]', 'Package[puppet-module-puppetlabs-augeas-core]', 'Package[python3-prometheus-client]', 'Package[python3-yaml]', 'Package[ruby-net-ssh]', 'Package[openssl]', 'Package[ssl-cert]', 'Package[ca-certificates]', 'Package[wmf-certificates]', 'Package[ntp]', 'Package[systemd-timesyncd]', 'Package[exim4-config]', 'Package[exim4-daemon-light]', 'Package[logrotate]', 'Package[prometheus-node-exporter]', 'Package[bsdutils]', 'Package[smartmontools]', 'Package[rsyslog]', 'Package[rsyslog-openssl]', 'Package[cadvisor]', 'Package[acct]', 'Package[byobu]', 'Package[colordiff]', 'Package[curl]', 'Package[debian-goodies]', 'Package[ethtool]', 'Package[gdb]', 'Package[gdisk]', 'Package[git]', 'Package[htop]', 'Package[httpry]', 'Package[iotop]', 'Package[iperf]', 'Package[jq]', 'Package[libtemplate-perl]', 'Package[lldpd]', 'Package[lshw]', 'Package[molly-guard]', 'Package[moreutils]', 'Package[net-tools]', 'Package[numactl]', 'Package[ncdu]', 'Package[ngrep]', 'Package[pigz]', 'Package[psmisc]', 'Package[pv]', 'Package[python3]', 'Package[screen]', 'Package[strace]', 'Package[sysstat]', 'Package[tcpdump]', 'Package[tmux]', 'Package[tree]', 'Package[vim]', 'Package[vim-addon-manager]', 'Package[vim-scripts]', 'Package[wipe]', 'Package[xfsprogs]', 'Package[zsh]', 'Package[icdiff]', 'Package[linux-perf]', 'Package[bsd-mailx]', 'Package[ack]', 'Package[netcat-openbsd]', 'Package[tshark]', 'Package[fzf]', 'Package[ripgrep]', 'Package[fd-find]', 'Package[kitty-terminfo]', 'Package[mtr-tiny]', 'Package[bat]', 'Package[efibootmgr]', 'Package[bind9-dnsutils]', 'Package[tzdata]', 'Package[python3-wmflib]', 'Package[starship]', 'Package[ruby-sorted-set]', 'Package[btop]', 'Package[linux-sysctl-defaults]', 'Package[apport]', 'Package[command-not-found]', 'Package[command-not-found-data]', 'Package[ecryptfs-utils]', 'Package[mlocate]', 'Package[os-prober]', 'Package[python3-apport]', 'Package[wpasupplicant]', 'Package[atop]', 'Package[apt-listchanges]', 'Package[isc-dhcp-client]', 'Package[rasdaemon]', 'Package[openssh-client]', 'Package[openssh-server]', 'Package[debdeploy-client]', 'Package[python3-dateutil]', 'Package[sudo]', 'Package[golang-cfssl]', 'Package[debmonitor-client]', 'Package[nagios-nrpe-server]', 'Package[monitoring-plugins]', 'Package[monitoring-plugins-basic]', 'Package[monitoring-plugins-standard]', 'Package[liburiparser1]', 'Package[python3-attr]', 'Package[freeipmi-tools]', 'Package[freeipmi-ipmiseld]', 'Package[rsyslog-kafka]', 'Package[emacs-nox]', 'Package[prometheus-ipmi-exporter]', 'Package[libnet-dns-perl]', 'Package[iptables]', 'Package[ferm]', 'Package[ulogd2]', 'Package[conntrack]', 'Package[dragonfly-dfdaemon]', 'Package[dragonfly-dfget]', 'Package[crictl]', 'Package[containerd]', 'Package[nerdctl]', 'Package[rsyslog-kubernetes]', 'Package[linux-cpupower]', 'Package[apparmor]', 'Package[socat]', 'Package[amd-k8s-device-plugin]', 'Package[amd-k8s-node-labeller]', 'Package[rocm-smi]', 'Package[python3-requests]', 'Package[wikimedia-lvs-realserver]', 'Package[ruby-concurrent]', 'Package[ruby]', 'Package[libruby]', 'Package[puppet-agent]', 'Package[linux-image-6.16.3+deb13-amd64]', 'Package[prometheus-rsyslog-exporter]', 'Package[initramfs-tools]', 'Package[python3-click]', 'Package[python3-box]', 'Package[confd]', 'Package[python3-toml]', 'Package[kubernetes-node]', 'Package[calicoctl]', 'Package[calico-cni]', 'Package[istio-cni]', 'Package[firmware-amd-graphics]']\n"}, {"resource": "File[/var/lib/prometheus/node.d/role_owner.prom]", "content": "--- /var/lib/prometheus/node.d/role_owner.prom.orig\n+++ /var/lib/prometheus/node.d/role_owner.prom\n@@ -1,3 +1,3 @@\n # HELP role_owner The team owner of the server role\n # TYPE role_owner gauge\n-role_owner{team=\"machine-learning\",role=\"ml_k8s::insetup_gpu\",cluster=\"ml_serve\"} 1.0\n+role_owner{team=\"machine-learning\",role=\"ml_k8s::worker\",cluster=\"ml_serve\"} 1.0"}, {"resource": "Nrpe::Monitor_service[disk_space]", "parameters": "--- Nrpe::Monitor_service[disk_space].orig\n+++ Nrpe::Monitor_service[disk_space]\n\n@@\n-    nrpe_command => /usr/lib/nagios/plugins/check_disk -w 6% -c 3% -W 6% -K 3% -l -e -A -i \"/srv/sd[a-b][1-3]\" -i \"/srv/nvme[0-9]n[0-9]p[0-9]\" --exclude-type=fuse --exclude-type=fuse.fuse_dfs --exclude-type=tracefs\n+    nrpe_command => /usr/lib/nagios/plugins/check_disk -w 10% -c 5% -W 6% -K 3% -l -e -A -i '/(var/lib|run)/(containerd|kubelet)/*' --exclude-type=tracefs\n"}, {"resource": "Class[Profile::Base::Production]", "parameters": "--- Class[Profile::Base::Production].orig\n+++ Class[Profile::Base::Production]\n\n@@\n-    role_description => Machine Learning GPU host in setup.\n+    role_description => ML Kubernetes worker node\n"}, {"resource": "Class[Base::Kernel]", "parameters": "--- Class[Base::Kernel].orig\n+++ Class[Base::Kernel]\n\n@@\n-    overlayfs => False\n+    overlayfs => True\n"}, {"resource": "Class[Profile::Base]", "parameters": "--- Class[Profile::Base].orig\n+++ Class[Profile::Base]\n\n@@\n-    use_linux_from_bpo_on_trixie => False\n+    use_linux_from_bpo_on_trixie => True\n@@\n-    overlayfs                    => False\n+    overlayfs                    => True\n"}, {"resource": "Class[Profile::Amd_gpu]", "parameters": "--- Class[Profile::Amd_gpu].orig\n+++ Class[Profile::Amd_gpu]\n\n+    kubernetes_cluster_name => ml-serve-eqiad\n@@\n-    is_basic_gpu_node       => True\n+    is_basic_gpu_node       => False\n@@\n-    firmwares_from_bpo      => False\n+    firmwares_from_bpo      => True\n@@\n-    is_kubernetes_node      => False\n+    is_kubernetes_node      => True\n"}, {"resource": "Class[Prometheus::Node_exporter]", "parameters": "--- Class[Prometheus::Node_exporter].orig\n+++ Class[Prometheus::Node_exporter]\n\n@@\n-    collectors_extra => []\n+    collectors_extra => ['processes']\n"}, {"resource": "Package[firmware-amd-graphics]", "parameters": "--- Package[firmware-amd-graphics].orig\n+++ Package[firmware-amd-graphics]\n\n@@\n-    ensure => installed\n+    ensure => 20251021-1~bpo13+1\n"}, {"resource": "Class[Profile::Apt]", "parameters": "--- Class[Profile::Apt].orig\n+++ Class[Profile::Apt]\n\n@@\n-    before => ['Package[puppet]', 'Package[facter]', 'Package[augeas-tools]', 'Package[virt-what]', 'Package[puppet-module-puppetlabs-augeas-core]', 'Package[python3-prometheus-client]', 'Package[python3-yaml]', 'Package[ruby-net-ssh]', 'Package[openssl]', 'Package[ssl-cert]', 'Package[ca-certificates]', 'Package[wmf-certificates]', 'Package[ntp]', 'Package[systemd-timesyncd]', 'Package[exim4-config]', 'Package[exim4-daemon-light]', 'Package[logrotate]', 'Package[prometheus-node-exporter]', 'Package[bsdutils]', 'Package[smartmontools]', 'Package[rsyslog]', 'Package[rsyslog-openssl]', 'Package[cadvisor]', 'Package[acct]', 'Package[byobu]', 'Package[colordiff]', 'Package[curl]', 'Package[debian-goodies]', 'Package[ethtool]', 'Package[gdb]', 'Package[gdisk]', 'Package[git]', 'Package[htop]', 'Package[httpry]', 'Package[iotop]', 'Package[iperf]', 'Package[jq]', 'Package[libtemplate-perl]', 'Package[lldpd]', 'Package[lshw]', 'Package[molly-guard]', 'Package[moreutils]', 'Package[net-tools]', 'Package[numactl]', 'Package[ncdu]', 'Package[ngrep]', 'Package[pigz]', 'Package[psmisc]', 'Package[pv]', 'Package[python3]', 'Package[screen]', 'Package[strace]', 'Package[sysstat]', 'Package[tcpdump]', 'Package[tmux]', 'Package[tree]', 'Package[vim]', 'Package[vim-addon-manager]', 'Package[vim-scripts]', 'Package[wipe]', 'Package[xfsprogs]', 'Package[zsh]', 'Package[icdiff]', 'Package[linux-perf]', 'Package[bsd-mailx]', 'Package[ack]', 'Package[netcat-openbsd]', 'Package[tshark]', 'Package[fzf]', 'Package[ripgrep]', 'Package[fd-find]', 'Package[kitty-terminfo]', 'Package[mtr-tiny]', 'Package[bat]', 'Package[efibootmgr]', 'Package[bind9-dnsutils]', 'Package[tzdata]', 'Package[python3-wmflib]', 'Package[starship]', 'Package[ruby-sorted-set]', 'Package[btop]', 'Package[linux-sysctl-defaults]', 'Package[apport]', 'Package[command-not-found]', 'Package[command-not-found-data]', 'Package[ecryptfs-utils]', 'Package[mlocate]', 'Package[os-prober]', 'Package[python3-apport]', 'Package[wpasupplicant]', 'Package[atop]', 'Package[apt-listchanges]', 'Package[isc-dhcp-client]', 'Package[rasdaemon]', 'Package[openssh-client]', 'Package[openssh-server]', 'Package[debdeploy-client]', 'Package[python3-dateutil]', 'Package[sudo]', 'Package[golang-cfssl]', 'Package[debmonitor-client]', 'Package[nagios-nrpe-server]', 'Package[monitoring-plugins]', 'Package[monitoring-plugins-basic]', 'Package[monitoring-plugins-standard]', 'Package[liburiparser1]', 'Package[python3-attr]', 'Package[freeipmi-tools]', 'Package[freeipmi-ipmiseld]', 'Package[rsyslog-kafka]', 'Package[emacs-nox]', 'Package[prometheus-ipmi-exporter]', 'Package[libnet-dns-perl]', 'Package[iptables]', 'Package[ferm]', 'Package[ulogd2]', 'Package[conntrack]', 'Package[rocm-smi]', 'Package[python3-requests]', 'Package[firmware-amd-graphics]', 'Package[ruby-concurrent]', 'Package[ruby]', 'Package[libruby]', 'Package[puppet-agent]', 'Package[prometheus-rsyslog-exporter]', 'Package[initramfs-tools]', 'Package[python3-click]', 'Package[python3-box]', 'Package[confd]', 'Package[python3-toml]']\n+    before => ['Package[puppet]', 'Package[facter]', 'Package[augeas-tools]', 'Package[virt-what]', 'Package[puppet-module-puppetlabs-augeas-core]', 'Package[python3-prometheus-client]', 'Package[python3-yaml]', 'Package[ruby-net-ssh]', 'Package[openssl]', 'Package[ssl-cert]', 'Package[ca-certificates]', 'Package[wmf-certificates]', 'Package[ntp]', 'Package[systemd-timesyncd]', 'Package[exim4-config]', 'Package[exim4-daemon-light]', 'Package[logrotate]', 'Package[prometheus-node-exporter]', 'Package[bsdutils]', 'Package[smartmontools]', 'Package[rsyslog]', 'Package[rsyslog-openssl]', 'Package[cadvisor]', 'Package[acct]', 'Package[byobu]', 'Package[colordiff]', 'Package[curl]', 'Package[debian-goodies]', 'Package[ethtool]', 'Package[gdb]', 'Package[gdisk]', 'Package[git]', 'Package[htop]', 'Package[httpry]', 'Package[iotop]', 'Package[iperf]', 'Package[jq]', 'Package[libtemplate-perl]', 'Package[lldpd]', 'Package[lshw]', 'Package[molly-guard]', 'Package[moreutils]', 'Package[net-tools]', 'Package[numactl]', 'Package[ncdu]', 'Package[ngrep]', 'Package[pigz]', 'Package[psmisc]', 'Package[pv]', 'Package[python3]', 'Package[screen]', 'Package[strace]', 'Package[sysstat]', 'Package[tcpdump]', 'Package[tmux]', 'Package[tree]', 'Package[vim]', 'Package[vim-addon-manager]', 'Package[vim-scripts]', 'Package[wipe]', 'Package[xfsprogs]', 'Package[zsh]', 'Package[icdiff]', 'Package[linux-perf]', 'Package[bsd-mailx]', 'Package[ack]', 'Package[netcat-openbsd]', 'Package[tshark]', 'Package[fzf]', 'Package[ripgrep]', 'Package[fd-find]', 'Package[kitty-terminfo]', 'Package[mtr-tiny]', 'Package[bat]', 'Package[efibootmgr]', 'Package[bind9-dnsutils]', 'Package[tzdata]', 'Package[python3-wmflib]', 'Package[starship]', 'Package[ruby-sorted-set]', 'Package[btop]', 'Package[linux-sysctl-defaults]', 'Package[apport]', 'Package[command-not-found]', 'Package[command-not-found-data]', 'Package[ecryptfs-utils]', 'Package[mlocate]', 'Package[os-prober]', 'Package[python3-apport]', 'Package[wpasupplicant]', 'Package[atop]', 'Package[apt-listchanges]', 'Package[isc-dhcp-client]', 'Package[rasdaemon]', 'Package[openssh-client]', 'Package[openssh-server]', 'Package[debdeploy-client]', 'Package[python3-dateutil]', 'Package[sudo]', 'Package[golang-cfssl]', 'Package[debmonitor-client]', 'Package[nagios-nrpe-server]', 'Package[monitoring-plugins]', 'Package[monitoring-plugins-basic]', 'Package[monitoring-plugins-standard]', 'Package[liburiparser1]', 'Package[python3-attr]', 'Package[freeipmi-tools]', 'Package[freeipmi-ipmiseld]', 'Package[rsyslog-kafka]', 'Package[emacs-nox]', 'Package[prometheus-ipmi-exporter]', 'Package[libnet-dns-perl]', 'Package[iptables]', 'Package[ferm]', 'Package[ulogd2]', 'Package[conntrack]', 'Package[dragonfly-dfdaemon]', 'Package[dragonfly-dfget]', 'Package[crictl]', 'Package[containerd]', 'Package[nerdctl]', 'Package[rsyslog-kubernetes]', 'Package[linux-cpupower]', 'Package[apparmor]', 'Package[socat]', 'Package[amd-k8s-device-plugin]', 'Package[amd-k8s-node-labeller]', 'Package[rocm-smi]', 'Package[python3-requests]', 'Package[wikimedia-lvs-realserver]', 'Package[ruby-concurrent]', 'Package[ruby]', 'Package[libruby]', 'Package[puppet-agent]', 'Package[linux-image-6.16.3+deb13-amd64]', 'Package[prometheus-rsyslog-exporter]', 'Package[initramfs-tools]', 'Package[python3-click]', 'Package[python3-box]', 'Package[confd]', 'Package[python3-toml]', 'Package[kubernetes-node]', 'Package[calicoctl]', 'Package[calico-cni]', 'Package[istio-cni]', 'Package[firmware-amd-graphics]']\n"}, {"resource": "Kmod::Blacklist[wmf_overlay]", "parameters": "--- Kmod::Blacklist[wmf_overlay].orig\n+++ Kmod::Blacklist[wmf_overlay]\n\n@@\n-    ensure  => present\n+    ensure  => absent\n@@\n-    modules => ['overlayfs', 'overlay']\n+    modules => []\n"}, {"resource": "Concat_fragment[main contacts]", "content": "--- main contacts.orig\n+++ main contacts\n@@ -1,3 +1,3 @@\n ---\n-role::ml_k8s::insetup_gpu:\n+role::ml_k8s::worker:\n - Machine Learning"}, {"resource": "Nrpe::Check[check_disk_space]", "parameters": "--- Nrpe::Check[check_disk_space].orig\n+++ Nrpe::Check[check_disk_space]\n\n@@\n-    command => /usr/lib/nagios/plugins/check_disk -w 6% -c 3% -W 6% -K 3% -l -e -A -i \"/srv/sd[a-b][1-3]\" -i \"/srv/nvme[0-9]n[0-9]p[0-9]\" --exclude-type=fuse --exclude-type=fuse.fuse_dfs --exclude-type=tracefs\n+    command => /usr/lib/nagios/plugins/check_disk -w 10% -c 5% -W 6% -K 3% -l -e -A -i '/(var/lib|run)/(containerd|kubelet)/*' --exclude-type=tracefs\n"}], "perc_changed": "13.12%"}}}