{"host": "cloudcontrol1011.eqiad.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 4223, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/nova/vendor_data.json]", "content": "--- /etc/nova/vendor_data.json.orig\n+++ /etc/nova/vendor_data.json\n@@ -1,4 +1,4 @@\n {\n   \"domain\": \"eqiad1.wikimedia.cloud\",\n-  \"cloud-init\": \"MIME-Version: 1.0\\nContent-Type: multipart/mixed; boundary=\\\"XXXXboundary text\\\"\\n\\nThis is a multipart config in MIME format.\\nIt contains a cloud-init config followed by\\na first boot script.\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/jinja2; charset=\\\"us-ascii\\\"\\n\\n## template: jinja\\n#cloud-config\\n\\ndatasource:\\n  OpenStack:\\n    timeout: 10\\n    max_wait: 120\\n    retries: 10\\n\\nhostname: {{ds.meta_data.name}}\\nfqdn: {{ds.meta_data.name}}.{{ds.meta_data.project_id}}.eqiad1.wikimedia.cloud\\n\\n# Prevent puppet from doing things before we're ready.\\n# APT fails to acquire GPG keys if packages dirmngr or gpg are missing\\nbootcmd:\\n  - [ cloud-init-per, once, disablepuppet, puppet, agent, --disable, \\\"prevent puppet runs during cloud-init\\\" ]\\n  - [ cloud-init-per, once, purgeresolved, /usr/bin/dpkg, --purge, systemd-resolved, libnss-resolve ]\\n  - [ cloud-init-per, once, dirmngr-aptupdate, apt-get, update ]\\n  - [ cloud-init-per, once, dirmngr-aptinstall, apt-get, install, \\\"-y\\\", dirmngr ]\\n  - [ cloud-init-per, once, gnupginstall, apt-get, install, \\\"-y\\\", gnupg ]\\n\\n\\n# /etc/block-ldap-key-lookup:\\n#   Prevent non-root logins while the VM is being setup\\n#   The ssh-key-ldap-lookup script rejects non-root user logins if this file\\n#   is present.\\n#\\n# /etc/rsyslog.d/60-puppet.conf:\\n#   Enable console logging for puppet\\n#\\n# /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf:\\n#   Enable root console on serial0\\n#   (cloud-init will create any needed parent dirs)\\n#\\n# /etc/apt/preferences.d/wikimedia.pref\\n#   Prefer packages from wikimedia repo\\nwrite_files:\\n    - content: \\\"VM is work in progress\\\"\\n      path: /etc/block-ldap-key-lookup\\n    - content: \\\"daemon.* |/dev/console\\\"\\n      path: /etc/rsyslog.d/60-puppet.conf\\n    - content: {{ds.meta_data.project_id}}\\n      path: /etc/openstack/project_id\\n    - content: {{ds.ec2_metadata.local_ipv4}}\\n      path: /etc/openstack/host_ip\\n    - content: UNKNOWN\\n      path: /etc/openstack/project_name\\n    - content: |\\n        [Service]\\n        ExecStart=\\n        ExecStart=-/sbin/agetty --autologin root --noclear %I $TERM\\n      path: /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf\\n    - content: |\\n        Package: *\\n        Pin: release o=Wikimedia\\n        Pin-Priority: 1001\\n      path: /etc/apt/preferences.d/wikimedia.pref\\n    - content: |\\n        clouds:\\n          novaobserver:\\n            auth:\\n              auth_url: https://openstack.eqiad1.wikimediacloud.org:25000/v3\\n              username: novaobserver\\n              password: dummy_observer_password\\n              project_id: observer\\n              user_domain_id: default\\n              project_domain_id: default\\n            region_name: eqiad1-r\\n            identity_api_version: 3\\n      path: /etc/openstack/clouds.yaml\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.image_build is defined %}\\n    - content: building\\n      path: /.base_image_is_building\\n{% endif %}\\n{% endif %}\\n\\n# resetting ttys0 so root is logged in\\nruncmd:\\n    - [systemctl, enable, serial-getty@ttyS0.service]\\n    - [systemctl, restart, serial-getty@ttyS0.service]\\n\\nssh_pwauth: False\\n\\nmanage_etc_hosts: false\\n\\n\\n# gpg: needed for apt to function properly\\n# pciutils: used by puppet to determine is_virtual\\n# python3-keystoneclient and openstackclient:  used to look up project name\\npackages:\\n    - gpg\\n    - pciutils\\n    - python3-keystoneclient\\n    - python3-openstackclient\\n\\n# Mark out ephemeral0 so that cloud-init leaves it alone;\\n#  we'll let puppet mount it later.\\nmounts:\\n    - [ephemeral0]\\n\\n# You'll see that we're setting apt_preserve_sources_list twice here.  That's\\n#  because there's a bug in cloud-init where it tries to reconcile the\\n#  two settings and if they're different the stage fails. That means that\\n#  if one of them is set differently from the default (True) then nothing\\n#  works.\\n#\\n#  Setting preserve_sources_list avoids clobbering the pre-set repos\\n#  (T340814) but still lets us inject the mediawiki repos.\\napt_preserve_sources_list: True\\napt:\\n    preserve_sources_list: True\\n    sources:\\n        wikimedia.list:\\n            source: |\\n                deb http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n                deb-src http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n            filename: wikimedia.list\\n            keyid: 9D392D3FFADF18FB\\n\\npackage_update: true\\npackage_upgrade: true\\n\\nfinal_message: cloud-init has finished\\n\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.install_puppet is defined %}\\n# This configures puppet and starts the puppet agent (which we don't want).\\n#  When the boot script starts we'll kill the agent and run puppet synchronously.\\npuppet:\\n    start_service: false\\n    exec: false\\n    package_name: puppet-agent\\n    conf:\\n        agent:\\n            server: 'puppet'\\n            certname: '%f'\\n{% endif %}\\n{% endif %}\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/text/x-shellscript; charset=\\\"us-ascii\\\"\\n#!/bin/bash\\n\\nset -x\\n\\n# Set fqdn with project name.\\n# We have to do this here since project_name isn't available via metadata.\\nhostname=`hostname`\\nip=`cat /etc/openstack/host_ip`\\nproject_id=`cat /etc/openstack/project_id`\\nproject_name=`openstack project show --os-cloud novaobserver -c name $project_id -f value`\\necho $project_name > /etc/openstack/project_name\\nfqdn=$hostname.$project_name.eqiad1.wikimedia.cloud\\ncat > /etc/hosts<< EOF\\n$ip $fqdn $hostname\\n127.0.0.1 localhost\\n\\n# The following lines are desirable for IPv6 capable hosts\\n::1 ip6-localhost ip6-loopback\\nfe00::0 ip6-localnet\\nff00::0 ip6-mcastprefix\\nff02::1 ip6-allnodes\\nff02::2 ip6-allrouters\\nff02::3 ip6-allhosts\\nEOF\\n\\nif ! command -v apt-get >/dev/null; then\\n    # If we don't have apt then none of our init script will\\n    #  work so let's just get out of here\\n\\n    rm /etc/block-ldap-key-lookup\\n    touch /.cloud-init-finished\\n    exit 0\\nfi\\n\\n# Function to test if puppet is running or not\\npuppet_is_running() {\\n    PUPPETLOCK=`puppet agent --configprint agent_catalog_run_lockfile`\\n    echo \\\"checking if puppet is running by looking at ${PUPPETLOCK}\\\"\\n    # If no lockfile is defined, we can't really tell what's happening.\\n    # Assume puppet is not running in this case\\n    test -n \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # If the lockfile is not present, puppet is not running\\n    test -e \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # Now let's see if the PID at $PUPPETLOCK is indeed present and running\\n    local puppetpid=$(cat \\\"$PUPPETLOCK\\\")\\n    local cmdline_file=\\\"/proc/${puppetpid}/cmdline\\\"\\n    if [ -f \\\"$cmdline_file\\\" ]; then\\n        # Puppet is indeed running\\n        grep -q puppet \\\"$cmdline_file\\\" && return 0\\n    fi\\n\\n    # The lock file is stale, delete it so that we're all on the same page\\n    echo \\\"Clearing stale puppet agent lock for pid ${puppetpid}\\\"\\n    rm $PUPPETLOCK\\n    return 1\\n}\\n\\n# loop function to wait for puppet to finish its execution\\nwait_for_puppet() {\\n    local attempts=${1:-30}\\n    for _ in $(seq \\\"$attempts\\\"); do\\n        if ! puppet_is_running; then\\n            return 0\\n        fi\\n        echo \\\"Waiting for puppet run to complete\\\"\\n        sleep 10\\n    done\\n    # If puppet is still running at this point, report an error\\n    echo \\\"Tired of waiting for puppet run to complete; giving up\\\"\\n    return 1\\n}\\n\\n# Upstream debian cloud base images come with chrony installed,\\n# as well as a service user with an invalid (>499) ID.  We can't\\n# just uninstall chrony because systemd depends on it; installing\\n# systemd-timesyncd uninstalls chrony but then runs afoul of the\\n# user ID that was created by chrony. To work around all that,\\n# we detect if chrony is installed and if it is, remove the\\n# user it installed (without uninstalling chrony).  That unblocks\\n# us for a subsequent install of systemd-timesyncd (which /does/\\n# uninstall chrony).\\nif /usr/bin/dpkg -l chrony | grep -q ^ii; then\\n    /usr/sbin/userdel systemd-timesync\\n    /usr/sbin/userdel systemd-coredump\\n    apt install -y systemd-timesyncd\\nfi\\n\\nsystemctl restart nscd.service\\ndpkg-reconfigure -fnoninteractive -pcritical openssh-server\\nsystemctl restart ssh.service\\nnscd -i hosts\\n\\n# This VM might have puppet installed in its base image, or it\\n#  might have the metadata install_puppet flag set. If neither\\n#  of these is true then skip puppet entirely.\\n#\\n# Note that in Bookworm and after the puppet client\\n#  is in a package named puppet-agent; earlier it was\\n#  just named 'puppet'.\\nif dpkg -s puppet || dpkg -s puppet-agent; then\\n    # Run puppet, twice.  The second time is just to pick up packages\\n    #  that may have been unavailable in apt before the first puppet run\\n    #  updated sources.list\\n    puppet agent --enable\\n\\n    # Just in case puppet is already running for some reason, wait\\n    #  for it to finish.\\n    wait_for_puppet 120\\n    service puppet stop\\n    wait_for_puppet 120\\n\\n    # Sometimes the previous --enable doesn't take. Don't know why but\\n    #  it's harmless to try again\\n    puppet agent --enable\\n    puppet agent --onetime --verbose --no-daemonize --no-splay --show_diff --certname `hostname -f`\\n    # Refresh ldap now that puppet has updated our ldap.conf\\n    systemctl restart nslcd.service\\n\\n    # Ensure all NFS mounts are mounted\\n    mount_attempts=1\\n    until [ $mount_attempts -gt 10 ]\\n    do\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\"\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\" >> /etc/block-ldap-key-lookup\\n        ((mount_attempts++))\\n        /usr/bin/timeout --preserve-status -k 10s 20s /bin/mount -a && break\\n        # Sleep for 10s before next attempt\\n        sleep 10\\n    done\\n\\n    # Run puppet again post mounting NFS mounts (if all the mounts hadn't been mounted\\n    # before, the puppet code that ensures the symlinks are created, etc may not\\n    # have run)\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # The following will be a no-op on most distros, but\\n    #  as of 08-2025 Trixie comes packaged with puppet 8\\n    #  and the puppet manifests themselves seek to downgrade\\n    #  to 7.\\n    apt install -y --allow-downgrades puppet-agent\\n    apt-get -y autoremove\\n\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # sssd might be caching sudo rules and other ldap things from before puppet\\n    # finished setting up ldap integration. Restart sssd and clear its cache.\\n    # On some old images we won't have sssd installed; in that case this will\\n    # throw some harmless errors.\\n    systemctl stop sssd\\n    rm -rf /var/lib/sss/db/*\\n    systemctl restart sssd\\nfi\\n\\nif test -f \\\"/.base_image_is_building\\\"; then\\n    # Now that puppet has configured apt properly, make sure\\n    # we have the right version of puppet installed.\\n    # The package may be named 'puppet' or may be named\\n    # 'puppet-agent' so one of these might fail.\\n    apt-get install --allow-downgrades -y puppet\\n    apt-get install --allow-downgrades -y puppet-agent\\n\\n    # stop puppet runs, clean up log files, and generally get this ready\\n    # for another cloud-init run.\\n    echo Cleaning up files before base image creation\\n    rm -f /.base_image_is_building\\n    puppet agent --disable \\\"waiting for initial VM setup\\\"\\n    rm -f /lib/systemd/system/puppet-agent-timer.timer\\n    rm -f /etc/cron.d/puppet\\n    rm -rf /var/lib/cloud/instances\\n    rm -rf /var/lib/puppet/ssl\\n    rm -rf /etc/dhcp/dhclient-enter-hooks.d/nodnsupdate\\n    cloud-init clean\\n    rm -f /.cloud-init-finished\\n    echo 'uninitialized' > etc/machine-id\\n    find /var/log -type f -exec rm '{}' \\\\;\\nelse\\n    # Remove the non-root login restriction. If this is the\\n    # initial base image, we just leave logins disabled.\\n    rm /etc/block-ldap-key-lookup\\n\\n    # mark the host as ready, from now on alerts and checks will send notifications (ex. puppet_alert.py)\\n    touch /.cloud-init-finished\\nfi\\n\\nexit 0\\n--XXXXboundary text\\n\"\n+  \"cloud-init\": \"MIME-Version: 1.0\\nContent-Type: multipart/mixed; boundary=\\\"XXXXboundary text\\\"\\n\\nThis is a multipart config in MIME format.\\nIt contains a cloud-init config followed by\\na first boot script.\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/jinja2; charset=\\\"us-ascii\\\"\\n\\n## template: jinja\\n#cloud-config\\n\\ndatasource:\\n  OpenStack:\\n    timeout: 10\\n    max_wait: 120\\n    retries: 10\\n\\nhostname: {{ds.meta_data.name}}\\nfqdn: {{ds.meta_data.name}}.{{ds.meta_data.project_id}}.eqiad1.wikimedia.cloud\\n\\n# Prevent puppet from doing things before we're ready.\\n# APT fails to acquire GPG keys if packages dirmngr or gpg are missing\\nbootcmd:\\n  - [ cloud-init-per, once, disablepuppet, puppet, agent, --disable, \\\"prevent puppet runs during cloud-init\\\" ]\\n  - [ cloud-init-per, once, purgeresolved, /usr/bin/dpkg, --purge, systemd-resolved, libnss-resolve ]\\n  - [ cloud-init-per, once, dirmngr-aptupdate, apt-get, update ]\\n  - [ cloud-init-per, once, dirmngr-aptinstall, apt-get, install, \\\"-y\\\", dirmngr ]\\n  - [ cloud-init-per, once, gnupginstall, apt-get, install, \\\"-y\\\", gnupg ]\\n\\n\\n# /etc/block-ldap-key-lookup:\\n#   Prevent non-root logins while the VM is being setup\\n#   The ssh-key-ldap-lookup script rejects non-root user logins if this file\\n#   is present.\\n#\\n# /etc/rsyslog.d/60-puppet.conf:\\n#   Enable console logging for puppet\\n#\\n# /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf:\\n#   Enable root console on serial0\\n#   (cloud-init will create any needed parent dirs)\\n#\\n# /etc/apt/preferences.d/wikimedia.pref\\n#   Prefer packages from wikimedia repo\\nwrite_files:\\n    - content: \\\"VM is work in progress\\\"\\n      path: /etc/block-ldap-key-lookup\\n    - content: \\\"daemon.* |/dev/console\\\"\\n      path: /etc/rsyslog.d/60-puppet.conf\\n    - content: {{ds.meta_data.project_id}}\\n      path: /etc/openstack/project_id\\n    - content: {{ds.ec2_metadata.local_ipv4}}\\n      path: /etc/openstack/host_ip\\n    - content: UNKNOWN\\n      path: /etc/openstack/project_name\\n    - content: |\\n        [Service]\\n        ExecStart=\\n        ExecStart=-/sbin/agetty --autologin root --noclear %I $TERM\\n      path: /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf\\n    - content: |\\n        Package: *\\n        Pin: release o=Wikimedia\\n        Pin-Priority: 1001\\n      path: /etc/apt/preferences.d/wikimedia.pref\\n    - content: |\\n          # Cumin Controller.\\n              from=10.64.16.154,2620:0:861:102:10:64:16:154,10.192.32.49,2620:0:860:103:10:192:32:49,10.192.15.6,2620:0:860:110:10:192:15:6,no-agent-forwarding,no-port-forwarding,no-x11-forwarding,no-user-rc ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICcav+ECiF6hW2XRuP7R8nqDw4hPlD0OChsGvB6K27jK root@cloudinfra-internal-puppetmaster-02\\n\\n      path: /etc/ssh/userkeys/root.d/cumin\\n      permissions: '0444'\\n      owner: root:root\\n    - content: |\\n        clouds:\\n          novaobserver:\\n            auth:\\n              auth_url: https://openstack.eqiad1.wikimediacloud.org:25000/v3\\n              username: novaobserver\\n              password: dummy_observer_password\\n              project_id: observer\\n              user_domain_id: default\\n              project_domain_id: default\\n            region_name: eqiad1-r\\n            identity_api_version: 3\\n      path: /etc/openstack/clouds.yaml\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.image_build is defined %}\\n    - content: building\\n      path: /.base_image_is_building\\n{% endif %}\\n{% endif %}\\n\\n# resetting ttys0 so root is logged in\\nruncmd:\\n    - [systemctl, enable, serial-getty@ttyS0.service]\\n    - [systemctl, restart, serial-getty@ttyS0.service]\\n\\nssh_pwauth: False\\n\\nmanage_etc_hosts: false\\n\\n\\n# gpg: needed for apt to function properly\\n# pciutils: used by puppet to determine is_virtual\\n# python3-keystoneclient and openstackclient:  used to look up project name\\npackages:\\n    - gpg\\n    - pciutils\\n    - python3-keystoneclient\\n    - python3-openstackclient\\n\\n# Mark out ephemeral0 so that cloud-init leaves it alone;\\n#  we'll let puppet mount it later.\\nmounts:\\n    - [ephemeral0]\\n\\n# You'll see that we're setting apt_preserve_sources_list twice here.  That's\\n#  because there's a bug in cloud-init where it tries to reconcile the\\n#  two settings and if they're different the stage fails. That means that\\n#  if one of them is set differently from the default (True) then nothing\\n#  works.\\n#\\n#  Setting preserve_sources_list avoids clobbering the pre-set repos\\n#  (T340814) but still lets us inject the mediawiki repos.\\napt_preserve_sources_list: True\\napt:\\n    preserve_sources_list: True\\n    sources:\\n        wikimedia.list:\\n            source: |\\n                deb http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n                deb-src http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n            filename: wikimedia.list\\n            keyid: 9D392D3FFADF18FB\\n\\npackage_update: true\\npackage_upgrade: true\\n\\nfinal_message: cloud-init has finished\\n\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.install_puppet is defined %}\\n# This configures puppet and starts the puppet agent (which we don't want).\\n#  When the boot script starts we'll kill the agent and run puppet synchronously.\\npuppet:\\n    start_service: false\\n    exec: false\\n    package_name: puppet-agent\\n    conf:\\n        agent:\\n            server: 'puppet'\\n            certname: '%f'\\n{% endif %}\\n{% endif %}\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/text/x-shellscript; charset=\\\"us-ascii\\\"\\n#!/bin/bash\\n\\nset -x\\n\\n# Set fqdn with project name.\\n# We have to do this here since project_name isn't available via metadata.\\nhostname=`hostname`\\nip=`cat /etc/openstack/host_ip`\\nproject_id=`cat /etc/openstack/project_id`\\nproject_name=`openstack project show --os-cloud novaobserver -c name $project_id -f value`\\necho $project_name > /etc/openstack/project_name\\nfqdn=$hostname.$project_name.eqiad1.wikimedia.cloud\\ncat > /etc/hosts<< EOF\\n$ip $fqdn $hostname\\n127.0.0.1 localhost\\n\\n# The following lines are desirable for IPv6 capable hosts\\n::1 ip6-localhost ip6-loopback\\nfe00::0 ip6-localnet\\nff00::0 ip6-mcastprefix\\nff02::1 ip6-allnodes\\nff02::2 ip6-allrouters\\nff02::3 ip6-allhosts\\nEOF\\n\\nif ! command -v apt-get >/dev/null; then\\n    # If we don't have apt then none of our init script will\\n    #  work so let's just get out of here\\n\\n    rm /etc/block-ldap-key-lookup\\n    touch /.cloud-init-finished\\n    exit 0\\nfi\\n\\n# Function to test if puppet is running or not\\npuppet_is_running() {\\n    PUPPETLOCK=`puppet agent --configprint agent_catalog_run_lockfile`\\n    echo \\\"checking if puppet is running by looking at ${PUPPETLOCK}\\\"\\n    # If no lockfile is defined, we can't really tell what's happening.\\n    # Assume puppet is not running in this case\\n    test -n \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # If the lockfile is not present, puppet is not running\\n    test -e \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # Now let's see if the PID at $PUPPETLOCK is indeed present and running\\n    local puppetpid=$(cat \\\"$PUPPETLOCK\\\")\\n    local cmdline_file=\\\"/proc/${puppetpid}/cmdline\\\"\\n    if [ -f \\\"$cmdline_file\\\" ]; then\\n        # Puppet is indeed running\\n        grep -q puppet \\\"$cmdline_file\\\" && return 0\\n    fi\\n\\n    # The lock file is stale, delete it so that we're all on the same page\\n    echo \\\"Clearing stale puppet agent lock for pid ${puppetpid}\\\"\\n    rm $PUPPETLOCK\\n    return 1\\n}\\n\\n# loop function to wait for puppet to finish its execution\\nwait_for_puppet() {\\n    local attempts=${1:-30}\\n    for _ in $(seq \\\"$attempts\\\"); do\\n        if ! puppet_is_running; then\\n            return 0\\n        fi\\n        echo \\\"Waiting for puppet run to complete\\\"\\n        sleep 10\\n    done\\n    # If puppet is still running at this point, report an error\\n    echo \\\"Tired of waiting for puppet run to complete; giving up\\\"\\n    return 1\\n}\\n\\n# Upstream debian cloud base images come with chrony installed,\\n# as well as a service user with an invalid (>499) ID.  We can't\\n# just uninstall chrony because systemd depends on it; installing\\n# systemd-timesyncd uninstalls chrony but then runs afoul of the\\n# user ID that was created by chrony. To work around all that,\\n# we detect if chrony is installed and if it is, remove the\\n# user it installed (without uninstalling chrony).  That unblocks\\n# us for a subsequent install of systemd-timesyncd (which /does/\\n# uninstall chrony).\\nif /usr/bin/dpkg -l chrony | grep -q ^ii; then\\n    /usr/sbin/userdel systemd-timesync\\n    /usr/sbin/userdel systemd-coredump\\n    apt install -y systemd-timesyncd\\nfi\\n\\nsystemctl restart nscd.service\\ndpkg-reconfigure -fnoninteractive -pcritical openssh-server\\nsystemctl restart ssh.service\\nnscd -i hosts\\n\\n# This VM might have puppet installed in its base image, or it\\n#  might have the metadata install_puppet flag set. If neither\\n#  of these is true then skip puppet entirely.\\n#\\n# Note that in Bookworm and after the puppet client\\n#  is in a package named puppet-agent; earlier it was\\n#  just named 'puppet'.\\nif dpkg -s puppet || dpkg -s puppet-agent; then\\n    # Run puppet, twice.  The second time is just to pick up packages\\n    #  that may have been unavailable in apt before the first puppet run\\n    #  updated sources.list\\n    puppet agent --enable\\n\\n    # Just in case puppet is already running for some reason, wait\\n    #  for it to finish.\\n    wait_for_puppet 120\\n    service puppet stop\\n    wait_for_puppet 120\\n\\n    # Sometimes the previous --enable doesn't take. Don't know why but\\n    #  it's harmless to try again\\n    puppet agent --enable\\n    puppet agent --onetime --verbose --no-daemonize --no-splay --show_diff --certname `hostname -f`\\n    # Refresh ldap now that puppet has updated our ldap.conf\\n    systemctl restart nslcd.service\\n\\n    # Ensure all NFS mounts are mounted\\n    mount_attempts=1\\n    until [ $mount_attempts -gt 10 ]\\n    do\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\"\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\" >> /etc/block-ldap-key-lookup\\n        ((mount_attempts++))\\n        /usr/bin/timeout --preserve-status -k 10s 20s /bin/mount -a && break\\n        # Sleep for 10s before next attempt\\n        sleep 10\\n    done\\n\\n    # Run puppet again post mounting NFS mounts (if all the mounts hadn't been mounted\\n    # before, the puppet code that ensures the symlinks are created, etc may not\\n    # have run)\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # The following will be a no-op on most distros, but\\n    #  as of 08-2025 Trixie comes packaged with puppet 8\\n    #  and the puppet manifests themselves seek to downgrade\\n    #  to 7.\\n    apt install -y --allow-downgrades puppet-agent\\n    apt-get -y autoremove\\n\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # sssd might be caching sudo rules and other ldap things from before puppet\\n    # finished setting up ldap integration. Restart sssd and clear its cache.\\n    # On some old images we won't have sssd installed; in that case this will\\n    # throw some harmless errors.\\n    systemctl stop sssd\\n    rm -rf /var/lib/sss/db/*\\n    systemctl restart sssd\\nfi\\n\\nif test -f \\\"/.base_image_is_building\\\"; then\\n    # Now that puppet has configured apt properly, make sure\\n    # we have the right version of puppet installed.\\n    # The package may be named 'puppet' or may be named\\n    # 'puppet-agent' so one of these might fail.\\n    apt-get install --allow-downgrades -y puppet\\n    apt-get install --allow-downgrades -y puppet-agent\\n\\n    # stop puppet runs, clean up log files, and generally get this ready\\n    # for another cloud-init run.\\n    echo Cleaning up files before base image creation\\n    rm -f /.base_image_is_building\\n    puppet agent --disable \\\"waiting for initial VM setup\\\"\\n    rm -f /lib/systemd/system/puppet-agent-timer.timer\\n    rm -f /etc/cron.d/puppet\\n    rm -rf /var/lib/cloud/instances\\n    rm -rf /var/lib/puppet/ssl\\n    rm -rf /etc/dhcp/dhclient-enter-hooks.d/nodnsupdate\\n    cloud-init clean\\n    rm -f /.cloud-init-finished\\n    echo 'uninitialized' > etc/machine-id\\n    find /var/log -type f -exec rm '{}' \\\\;\\nelse\\n    # Remove the non-root login restriction. If this is the\\n    # initial base image, we just leave logins disabled.\\n    rm /etc/block-ldap-key-lookup\\n\\n    # mark the host as ready, from now on alerts and checks will send notifications (ex. puppet_alert.py)\\n    touch /.cloud-init-finished\\nfi\\n\\nexit 0\\n--XXXXboundary text\\n\"\n }"}, {"resource": "Class[Profile::Openstack::Base::Nova::Api::Service]", "parameters": "--- Class[Profile::Openstack::Base::Nova::Api::Service].orig\n+++ Class[Profile::Openstack::Base::Nova::Api::Service]\n\n+    cumin_masters => ['10.64.16.154', '2620:0:861:102:10:64:16:154', '10.192.32.49', '2620:0:860:103:10:192:32:49', '10.192.15.6', '2620:0:860:110:10:192:15:6']\n"}, {"resource": "Class[Openstack::Nova::Api::Service]", "parameters": "--- Class[Openstack::Nova::Api::Service].orig\n+++ Class[Openstack::Nova::Api::Service]\n\n+    cumin_masters => ['10.64.16.154', '2620:0:861:102:10:64:16:154', '10.192.32.49', '2620:0:860:103:10:192:32:49', '10.192.15.6', '2620:0:860:110:10:192:15:6']\n"}], "perc_changed": "0.07%"}, "core": {"total": 4223, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/nova/vendor_data.json]", "content": "--- /etc/nova/vendor_data.json.orig\n+++ /etc/nova/vendor_data.json\n@@ -1,4 +1,4 @@\n {\n   \"domain\": \"eqiad1.wikimedia.cloud\",\n-  \"cloud-init\": \"MIME-Version: 1.0\\nContent-Type: multipart/mixed; boundary=\\\"XXXXboundary text\\\"\\n\\nThis is a multipart config in MIME format.\\nIt contains a cloud-init config followed by\\na first boot script.\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/jinja2; charset=\\\"us-ascii\\\"\\n\\n## template: jinja\\n#cloud-config\\n\\ndatasource:\\n  OpenStack:\\n    timeout: 10\\n    max_wait: 120\\n    retries: 10\\n\\nhostname: {{ds.meta_data.name}}\\nfqdn: {{ds.meta_data.name}}.{{ds.meta_data.project_id}}.eqiad1.wikimedia.cloud\\n\\n# Prevent puppet from doing things before we're ready.\\n# APT fails to acquire GPG keys if packages dirmngr or gpg are missing\\nbootcmd:\\n  - [ cloud-init-per, once, disablepuppet, puppet, agent, --disable, \\\"prevent puppet runs during cloud-init\\\" ]\\n  - [ cloud-init-per, once, purgeresolved, /usr/bin/dpkg, --purge, systemd-resolved, libnss-resolve ]\\n  - [ cloud-init-per, once, dirmngr-aptupdate, apt-get, update ]\\n  - [ cloud-init-per, once, dirmngr-aptinstall, apt-get, install, \\\"-y\\\", dirmngr ]\\n  - [ cloud-init-per, once, gnupginstall, apt-get, install, \\\"-y\\\", gnupg ]\\n\\n\\n# /etc/block-ldap-key-lookup:\\n#   Prevent non-root logins while the VM is being setup\\n#   The ssh-key-ldap-lookup script rejects non-root user logins if this file\\n#   is present.\\n#\\n# /etc/rsyslog.d/60-puppet.conf:\\n#   Enable console logging for puppet\\n#\\n# /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf:\\n#   Enable root console on serial0\\n#   (cloud-init will create any needed parent dirs)\\n#\\n# /etc/apt/preferences.d/wikimedia.pref\\n#   Prefer packages from wikimedia repo\\nwrite_files:\\n    - content: \\\"VM is work in progress\\\"\\n      path: /etc/block-ldap-key-lookup\\n    - content: \\\"daemon.* |/dev/console\\\"\\n      path: /etc/rsyslog.d/60-puppet.conf\\n    - content: {{ds.meta_data.project_id}}\\n      path: /etc/openstack/project_id\\n    - content: {{ds.ec2_metadata.local_ipv4}}\\n      path: /etc/openstack/host_ip\\n    - content: UNKNOWN\\n      path: /etc/openstack/project_name\\n    - content: |\\n        [Service]\\n        ExecStart=\\n        ExecStart=-/sbin/agetty --autologin root --noclear %I $TERM\\n      path: /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf\\n    - content: |\\n        Package: *\\n        Pin: release o=Wikimedia\\n        Pin-Priority: 1001\\n      path: /etc/apt/preferences.d/wikimedia.pref\\n    - content: |\\n        clouds:\\n          novaobserver:\\n            auth:\\n              auth_url: https://openstack.eqiad1.wikimediacloud.org:25000/v3\\n              username: novaobserver\\n              password: dummy_observer_password\\n              project_id: observer\\n              user_domain_id: default\\n              project_domain_id: default\\n            region_name: eqiad1-r\\n            identity_api_version: 3\\n      path: /etc/openstack/clouds.yaml\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.image_build is defined %}\\n    - content: building\\n      path: /.base_image_is_building\\n{% endif %}\\n{% endif %}\\n\\n# resetting ttys0 so root is logged in\\nruncmd:\\n    - [systemctl, enable, serial-getty@ttyS0.service]\\n    - [systemctl, restart, serial-getty@ttyS0.service]\\n\\nssh_pwauth: False\\n\\nmanage_etc_hosts: false\\n\\n\\n# gpg: needed for apt to function properly\\n# pciutils: used by puppet to determine is_virtual\\n# python3-keystoneclient and openstackclient:  used to look up project name\\npackages:\\n    - gpg\\n    - pciutils\\n    - python3-keystoneclient\\n    - python3-openstackclient\\n\\n# Mark out ephemeral0 so that cloud-init leaves it alone;\\n#  we'll let puppet mount it later.\\nmounts:\\n    - [ephemeral0]\\n\\n# You'll see that we're setting apt_preserve_sources_list twice here.  That's\\n#  because there's a bug in cloud-init where it tries to reconcile the\\n#  two settings and if they're different the stage fails. That means that\\n#  if one of them is set differently from the default (True) then nothing\\n#  works.\\n#\\n#  Setting preserve_sources_list avoids clobbering the pre-set repos\\n#  (T340814) but still lets us inject the mediawiki repos.\\napt_preserve_sources_list: True\\napt:\\n    preserve_sources_list: True\\n    sources:\\n        wikimedia.list:\\n            source: |\\n                deb http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n                deb-src http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n            filename: wikimedia.list\\n            keyid: 9D392D3FFADF18FB\\n\\npackage_update: true\\npackage_upgrade: true\\n\\nfinal_message: cloud-init has finished\\n\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.install_puppet is defined %}\\n# This configures puppet and starts the puppet agent (which we don't want).\\n#  When the boot script starts we'll kill the agent and run puppet synchronously.\\npuppet:\\n    start_service: false\\n    exec: false\\n    package_name: puppet-agent\\n    conf:\\n        agent:\\n            server: 'puppet'\\n            certname: '%f'\\n{% endif %}\\n{% endif %}\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/text/x-shellscript; charset=\\\"us-ascii\\\"\\n#!/bin/bash\\n\\nset -x\\n\\n# Set fqdn with project name.\\n# We have to do this here since project_name isn't available via metadata.\\nhostname=`hostname`\\nip=`cat /etc/openstack/host_ip`\\nproject_id=`cat /etc/openstack/project_id`\\nproject_name=`openstack project show --os-cloud novaobserver -c name $project_id -f value`\\necho $project_name > /etc/openstack/project_name\\nfqdn=$hostname.$project_name.eqiad1.wikimedia.cloud\\ncat > /etc/hosts<< EOF\\n$ip $fqdn $hostname\\n127.0.0.1 localhost\\n\\n# The following lines are desirable for IPv6 capable hosts\\n::1 ip6-localhost ip6-loopback\\nfe00::0 ip6-localnet\\nff00::0 ip6-mcastprefix\\nff02::1 ip6-allnodes\\nff02::2 ip6-allrouters\\nff02::3 ip6-allhosts\\nEOF\\n\\nif ! command -v apt-get >/dev/null; then\\n    # If we don't have apt then none of our init script will\\n    #  work so let's just get out of here\\n\\n    rm /etc/block-ldap-key-lookup\\n    touch /.cloud-init-finished\\n    exit 0\\nfi\\n\\n# Function to test if puppet is running or not\\npuppet_is_running() {\\n    PUPPETLOCK=`puppet agent --configprint agent_catalog_run_lockfile`\\n    echo \\\"checking if puppet is running by looking at ${PUPPETLOCK}\\\"\\n    # If no lockfile is defined, we can't really tell what's happening.\\n    # Assume puppet is not running in this case\\n    test -n \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # If the lockfile is not present, puppet is not running\\n    test -e \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # Now let's see if the PID at $PUPPETLOCK is indeed present and running\\n    local puppetpid=$(cat \\\"$PUPPETLOCK\\\")\\n    local cmdline_file=\\\"/proc/${puppetpid}/cmdline\\\"\\n    if [ -f \\\"$cmdline_file\\\" ]; then\\n        # Puppet is indeed running\\n        grep -q puppet \\\"$cmdline_file\\\" && return 0\\n    fi\\n\\n    # The lock file is stale, delete it so that we're all on the same page\\n    echo \\\"Clearing stale puppet agent lock for pid ${puppetpid}\\\"\\n    rm $PUPPETLOCK\\n    return 1\\n}\\n\\n# loop function to wait for puppet to finish its execution\\nwait_for_puppet() {\\n    local attempts=${1:-30}\\n    for _ in $(seq \\\"$attempts\\\"); do\\n        if ! puppet_is_running; then\\n            return 0\\n        fi\\n        echo \\\"Waiting for puppet run to complete\\\"\\n        sleep 10\\n    done\\n    # If puppet is still running at this point, report an error\\n    echo \\\"Tired of waiting for puppet run to complete; giving up\\\"\\n    return 1\\n}\\n\\n# Upstream debian cloud base images come with chrony installed,\\n# as well as a service user with an invalid (>499) ID.  We can't\\n# just uninstall chrony because systemd depends on it; installing\\n# systemd-timesyncd uninstalls chrony but then runs afoul of the\\n# user ID that was created by chrony. To work around all that,\\n# we detect if chrony is installed and if it is, remove the\\n# user it installed (without uninstalling chrony).  That unblocks\\n# us for a subsequent install of systemd-timesyncd (which /does/\\n# uninstall chrony).\\nif /usr/bin/dpkg -l chrony | grep -q ^ii; then\\n    /usr/sbin/userdel systemd-timesync\\n    /usr/sbin/userdel systemd-coredump\\n    apt install -y systemd-timesyncd\\nfi\\n\\nsystemctl restart nscd.service\\ndpkg-reconfigure -fnoninteractive -pcritical openssh-server\\nsystemctl restart ssh.service\\nnscd -i hosts\\n\\n# This VM might have puppet installed in its base image, or it\\n#  might have the metadata install_puppet flag set. If neither\\n#  of these is true then skip puppet entirely.\\n#\\n# Note that in Bookworm and after the puppet client\\n#  is in a package named puppet-agent; earlier it was\\n#  just named 'puppet'.\\nif dpkg -s puppet || dpkg -s puppet-agent; then\\n    # Run puppet, twice.  The second time is just to pick up packages\\n    #  that may have been unavailable in apt before the first puppet run\\n    #  updated sources.list\\n    puppet agent --enable\\n\\n    # Just in case puppet is already running for some reason, wait\\n    #  for it to finish.\\n    wait_for_puppet 120\\n    service puppet stop\\n    wait_for_puppet 120\\n\\n    # Sometimes the previous --enable doesn't take. Don't know why but\\n    #  it's harmless to try again\\n    puppet agent --enable\\n    puppet agent --onetime --verbose --no-daemonize --no-splay --show_diff --certname `hostname -f`\\n    # Refresh ldap now that puppet has updated our ldap.conf\\n    systemctl restart nslcd.service\\n\\n    # Ensure all NFS mounts are mounted\\n    mount_attempts=1\\n    until [ $mount_attempts -gt 10 ]\\n    do\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\"\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\" >> /etc/block-ldap-key-lookup\\n        ((mount_attempts++))\\n        /usr/bin/timeout --preserve-status -k 10s 20s /bin/mount -a && break\\n        # Sleep for 10s before next attempt\\n        sleep 10\\n    done\\n\\n    # Run puppet again post mounting NFS mounts (if all the mounts hadn't been mounted\\n    # before, the puppet code that ensures the symlinks are created, etc may not\\n    # have run)\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # The following will be a no-op on most distros, but\\n    #  as of 08-2025 Trixie comes packaged with puppet 8\\n    #  and the puppet manifests themselves seek to downgrade\\n    #  to 7.\\n    apt install -y --allow-downgrades puppet-agent\\n    apt-get -y autoremove\\n\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # sssd might be caching sudo rules and other ldap things from before puppet\\n    # finished setting up ldap integration. Restart sssd and clear its cache.\\n    # On some old images we won't have sssd installed; in that case this will\\n    # throw some harmless errors.\\n    systemctl stop sssd\\n    rm -rf /var/lib/sss/db/*\\n    systemctl restart sssd\\nfi\\n\\nif test -f \\\"/.base_image_is_building\\\"; then\\n    # Now that puppet has configured apt properly, make sure\\n    # we have the right version of puppet installed.\\n    # The package may be named 'puppet' or may be named\\n    # 'puppet-agent' so one of these might fail.\\n    apt-get install --allow-downgrades -y puppet\\n    apt-get install --allow-downgrades -y puppet-agent\\n\\n    # stop puppet runs, clean up log files, and generally get this ready\\n    # for another cloud-init run.\\n    echo Cleaning up files before base image creation\\n    rm -f /.base_image_is_building\\n    puppet agent --disable \\\"waiting for initial VM setup\\\"\\n    rm -f /lib/systemd/system/puppet-agent-timer.timer\\n    rm -f /etc/cron.d/puppet\\n    rm -rf /var/lib/cloud/instances\\n    rm -rf /var/lib/puppet/ssl\\n    rm -rf /etc/dhcp/dhclient-enter-hooks.d/nodnsupdate\\n    cloud-init clean\\n    rm -f /.cloud-init-finished\\n    echo 'uninitialized' > etc/machine-id\\n    find /var/log -type f -exec rm '{}' \\\\;\\nelse\\n    # Remove the non-root login restriction. If this is the\\n    # initial base image, we just leave logins disabled.\\n    rm /etc/block-ldap-key-lookup\\n\\n    # mark the host as ready, from now on alerts and checks will send notifications (ex. puppet_alert.py)\\n    touch /.cloud-init-finished\\nfi\\n\\nexit 0\\n--XXXXboundary text\\n\"\n+  \"cloud-init\": \"MIME-Version: 1.0\\nContent-Type: multipart/mixed; boundary=\\\"XXXXboundary text\\\"\\n\\nThis is a multipart config in MIME format.\\nIt contains a cloud-init config followed by\\na first boot script.\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/jinja2; charset=\\\"us-ascii\\\"\\n\\n## template: jinja\\n#cloud-config\\n\\ndatasource:\\n  OpenStack:\\n    timeout: 10\\n    max_wait: 120\\n    retries: 10\\n\\nhostname: {{ds.meta_data.name}}\\nfqdn: {{ds.meta_data.name}}.{{ds.meta_data.project_id}}.eqiad1.wikimedia.cloud\\n\\n# Prevent puppet from doing things before we're ready.\\n# APT fails to acquire GPG keys if packages dirmngr or gpg are missing\\nbootcmd:\\n  - [ cloud-init-per, once, disablepuppet, puppet, agent, --disable, \\\"prevent puppet runs during cloud-init\\\" ]\\n  - [ cloud-init-per, once, purgeresolved, /usr/bin/dpkg, --purge, systemd-resolved, libnss-resolve ]\\n  - [ cloud-init-per, once, dirmngr-aptupdate, apt-get, update ]\\n  - [ cloud-init-per, once, dirmngr-aptinstall, apt-get, install, \\\"-y\\\", dirmngr ]\\n  - [ cloud-init-per, once, gnupginstall, apt-get, install, \\\"-y\\\", gnupg ]\\n\\n\\n# /etc/block-ldap-key-lookup:\\n#   Prevent non-root logins while the VM is being setup\\n#   The ssh-key-ldap-lookup script rejects non-root user logins if this file\\n#   is present.\\n#\\n# /etc/rsyslog.d/60-puppet.conf:\\n#   Enable console logging for puppet\\n#\\n# /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf:\\n#   Enable root console on serial0\\n#   (cloud-init will create any needed parent dirs)\\n#\\n# /etc/apt/preferences.d/wikimedia.pref\\n#   Prefer packages from wikimedia repo\\nwrite_files:\\n    - content: \\\"VM is work in progress\\\"\\n      path: /etc/block-ldap-key-lookup\\n    - content: \\\"daemon.* |/dev/console\\\"\\n      path: /etc/rsyslog.d/60-puppet.conf\\n    - content: {{ds.meta_data.project_id}}\\n      path: /etc/openstack/project_id\\n    - content: {{ds.ec2_metadata.local_ipv4}}\\n      path: /etc/openstack/host_ip\\n    - content: UNKNOWN\\n      path: /etc/openstack/project_name\\n    - content: |\\n        [Service]\\n        ExecStart=\\n        ExecStart=-/sbin/agetty --autologin root --noclear %I $TERM\\n      path: /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf\\n    - content: |\\n        Package: *\\n        Pin: release o=Wikimedia\\n        Pin-Priority: 1001\\n      path: /etc/apt/preferences.d/wikimedia.pref\\n    - content: |\\n          # Cumin Controller.\\n              from=10.64.16.154,2620:0:861:102:10:64:16:154,10.192.32.49,2620:0:860:103:10:192:32:49,10.192.15.6,2620:0:860:110:10:192:15:6,no-agent-forwarding,no-port-forwarding,no-x11-forwarding,no-user-rc ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICcav+ECiF6hW2XRuP7R8nqDw4hPlD0OChsGvB6K27jK root@cloudinfra-internal-puppetmaster-02\\n\\n      path: /etc/ssh/userkeys/root.d/cumin\\n      permissions: '0444'\\n      owner: root:root\\n    - content: |\\n        clouds:\\n          novaobserver:\\n            auth:\\n              auth_url: https://openstack.eqiad1.wikimediacloud.org:25000/v3\\n              username: novaobserver\\n              password: dummy_observer_password\\n              project_id: observer\\n              user_domain_id: default\\n              project_domain_id: default\\n            region_name: eqiad1-r\\n            identity_api_version: 3\\n      path: /etc/openstack/clouds.yaml\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.image_build is defined %}\\n    - content: building\\n      path: /.base_image_is_building\\n{% endif %}\\n{% endif %}\\n\\n# resetting ttys0 so root is logged in\\nruncmd:\\n    - [systemctl, enable, serial-getty@ttyS0.service]\\n    - [systemctl, restart, serial-getty@ttyS0.service]\\n\\nssh_pwauth: False\\n\\nmanage_etc_hosts: false\\n\\n\\n# gpg: needed for apt to function properly\\n# pciutils: used by puppet to determine is_virtual\\n# python3-keystoneclient and openstackclient:  used to look up project name\\npackages:\\n    - gpg\\n    - pciutils\\n    - python3-keystoneclient\\n    - python3-openstackclient\\n\\n# Mark out ephemeral0 so that cloud-init leaves it alone;\\n#  we'll let puppet mount it later.\\nmounts:\\n    - [ephemeral0]\\n\\n# You'll see that we're setting apt_preserve_sources_list twice here.  That's\\n#  because there's a bug in cloud-init where it tries to reconcile the\\n#  two settings and if they're different the stage fails. That means that\\n#  if one of them is set differently from the default (True) then nothing\\n#  works.\\n#\\n#  Setting preserve_sources_list avoids clobbering the pre-set repos\\n#  (T340814) but still lets us inject the mediawiki repos.\\napt_preserve_sources_list: True\\napt:\\n    preserve_sources_list: True\\n    sources:\\n        wikimedia.list:\\n            source: |\\n                deb http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n                deb-src http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n            filename: wikimedia.list\\n            keyid: 9D392D3FFADF18FB\\n\\npackage_update: true\\npackage_upgrade: true\\n\\nfinal_message: cloud-init has finished\\n\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.install_puppet is defined %}\\n# This configures puppet and starts the puppet agent (which we don't want).\\n#  When the boot script starts we'll kill the agent and run puppet synchronously.\\npuppet:\\n    start_service: false\\n    exec: false\\n    package_name: puppet-agent\\n    conf:\\n        agent:\\n            server: 'puppet'\\n            certname: '%f'\\n{% endif %}\\n{% endif %}\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/text/x-shellscript; charset=\\\"us-ascii\\\"\\n#!/bin/bash\\n\\nset -x\\n\\n# Set fqdn with project name.\\n# We have to do this here since project_name isn't available via metadata.\\nhostname=`hostname`\\nip=`cat /etc/openstack/host_ip`\\nproject_id=`cat /etc/openstack/project_id`\\nproject_name=`openstack project show --os-cloud novaobserver -c name $project_id -f value`\\necho $project_name > /etc/openstack/project_name\\nfqdn=$hostname.$project_name.eqiad1.wikimedia.cloud\\ncat > /etc/hosts<< EOF\\n$ip $fqdn $hostname\\n127.0.0.1 localhost\\n\\n# The following lines are desirable for IPv6 capable hosts\\n::1 ip6-localhost ip6-loopback\\nfe00::0 ip6-localnet\\nff00::0 ip6-mcastprefix\\nff02::1 ip6-allnodes\\nff02::2 ip6-allrouters\\nff02::3 ip6-allhosts\\nEOF\\n\\nif ! command -v apt-get >/dev/null; then\\n    # If we don't have apt then none of our init script will\\n    #  work so let's just get out of here\\n\\n    rm /etc/block-ldap-key-lookup\\n    touch /.cloud-init-finished\\n    exit 0\\nfi\\n\\n# Function to test if puppet is running or not\\npuppet_is_running() {\\n    PUPPETLOCK=`puppet agent --configprint agent_catalog_run_lockfile`\\n    echo \\\"checking if puppet is running by looking at ${PUPPETLOCK}\\\"\\n    # If no lockfile is defined, we can't really tell what's happening.\\n    # Assume puppet is not running in this case\\n    test -n \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # If the lockfile is not present, puppet is not running\\n    test -e \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # Now let's see if the PID at $PUPPETLOCK is indeed present and running\\n    local puppetpid=$(cat \\\"$PUPPETLOCK\\\")\\n    local cmdline_file=\\\"/proc/${puppetpid}/cmdline\\\"\\n    if [ -f \\\"$cmdline_file\\\" ]; then\\n        # Puppet is indeed running\\n        grep -q puppet \\\"$cmdline_file\\\" && return 0\\n    fi\\n\\n    # The lock file is stale, delete it so that we're all on the same page\\n    echo \\\"Clearing stale puppet agent lock for pid ${puppetpid}\\\"\\n    rm $PUPPETLOCK\\n    return 1\\n}\\n\\n# loop function to wait for puppet to finish its execution\\nwait_for_puppet() {\\n    local attempts=${1:-30}\\n    for _ in $(seq \\\"$attempts\\\"); do\\n        if ! puppet_is_running; then\\n            return 0\\n        fi\\n        echo \\\"Waiting for puppet run to complete\\\"\\n        sleep 10\\n    done\\n    # If puppet is still running at this point, report an error\\n    echo \\\"Tired of waiting for puppet run to complete; giving up\\\"\\n    return 1\\n}\\n\\n# Upstream debian cloud base images come with chrony installed,\\n# as well as a service user with an invalid (>499) ID.  We can't\\n# just uninstall chrony because systemd depends on it; installing\\n# systemd-timesyncd uninstalls chrony but then runs afoul of the\\n# user ID that was created by chrony. To work around all that,\\n# we detect if chrony is installed and if it is, remove the\\n# user it installed (without uninstalling chrony).  That unblocks\\n# us for a subsequent install of systemd-timesyncd (which /does/\\n# uninstall chrony).\\nif /usr/bin/dpkg -l chrony | grep -q ^ii; then\\n    /usr/sbin/userdel systemd-timesync\\n    /usr/sbin/userdel systemd-coredump\\n    apt install -y systemd-timesyncd\\nfi\\n\\nsystemctl restart nscd.service\\ndpkg-reconfigure -fnoninteractive -pcritical openssh-server\\nsystemctl restart ssh.service\\nnscd -i hosts\\n\\n# This VM might have puppet installed in its base image, or it\\n#  might have the metadata install_puppet flag set. If neither\\n#  of these is true then skip puppet entirely.\\n#\\n# Note that in Bookworm and after the puppet client\\n#  is in a package named puppet-agent; earlier it was\\n#  just named 'puppet'.\\nif dpkg -s puppet || dpkg -s puppet-agent; then\\n    # Run puppet, twice.  The second time is just to pick up packages\\n    #  that may have been unavailable in apt before the first puppet run\\n    #  updated sources.list\\n    puppet agent --enable\\n\\n    # Just in case puppet is already running for some reason, wait\\n    #  for it to finish.\\n    wait_for_puppet 120\\n    service puppet stop\\n    wait_for_puppet 120\\n\\n    # Sometimes the previous --enable doesn't take. Don't know why but\\n    #  it's harmless to try again\\n    puppet agent --enable\\n    puppet agent --onetime --verbose --no-daemonize --no-splay --show_diff --certname `hostname -f`\\n    # Refresh ldap now that puppet has updated our ldap.conf\\n    systemctl restart nslcd.service\\n\\n    # Ensure all NFS mounts are mounted\\n    mount_attempts=1\\n    until [ $mount_attempts -gt 10 ]\\n    do\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\"\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\" >> /etc/block-ldap-key-lookup\\n        ((mount_attempts++))\\n        /usr/bin/timeout --preserve-status -k 10s 20s /bin/mount -a && break\\n        # Sleep for 10s before next attempt\\n        sleep 10\\n    done\\n\\n    # Run puppet again post mounting NFS mounts (if all the mounts hadn't been mounted\\n    # before, the puppet code that ensures the symlinks are created, etc may not\\n    # have run)\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # The following will be a no-op on most distros, but\\n    #  as of 08-2025 Trixie comes packaged with puppet 8\\n    #  and the puppet manifests themselves seek to downgrade\\n    #  to 7.\\n    apt install -y --allow-downgrades puppet-agent\\n    apt-get -y autoremove\\n\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # sssd might be caching sudo rules and other ldap things from before puppet\\n    # finished setting up ldap integration. Restart sssd and clear its cache.\\n    # On some old images we won't have sssd installed; in that case this will\\n    # throw some harmless errors.\\n    systemctl stop sssd\\n    rm -rf /var/lib/sss/db/*\\n    systemctl restart sssd\\nfi\\n\\nif test -f \\\"/.base_image_is_building\\\"; then\\n    # Now that puppet has configured apt properly, make sure\\n    # we have the right version of puppet installed.\\n    # The package may be named 'puppet' or may be named\\n    # 'puppet-agent' so one of these might fail.\\n    apt-get install --allow-downgrades -y puppet\\n    apt-get install --allow-downgrades -y puppet-agent\\n\\n    # stop puppet runs, clean up log files, and generally get this ready\\n    # for another cloud-init run.\\n    echo Cleaning up files before base image creation\\n    rm -f /.base_image_is_building\\n    puppet agent --disable \\\"waiting for initial VM setup\\\"\\n    rm -f /lib/systemd/system/puppet-agent-timer.timer\\n    rm -f /etc/cron.d/puppet\\n    rm -rf /var/lib/cloud/instances\\n    rm -rf /var/lib/puppet/ssl\\n    rm -rf /etc/dhcp/dhclient-enter-hooks.d/nodnsupdate\\n    cloud-init clean\\n    rm -f /.cloud-init-finished\\n    echo 'uninitialized' > etc/machine-id\\n    find /var/log -type f -exec rm '{}' \\\\;\\nelse\\n    # Remove the non-root login restriction. If this is the\\n    # initial base image, we just leave logins disabled.\\n    rm /etc/block-ldap-key-lookup\\n\\n    # mark the host as ready, from now on alerts and checks will send notifications (ex. puppet_alert.py)\\n    touch /.cloud-init-finished\\nfi\\n\\nexit 0\\n--XXXXboundary text\\n\"\n }"}], "perc_changed": "0.02%"}, "main": {"total": 4223, "only_in_self": [], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/nova/vendor_data.json]", "content": "--- /etc/nova/vendor_data.json.orig\n+++ /etc/nova/vendor_data.json\n@@ -1,4 +1,4 @@\n {\n   \"domain\": \"eqiad1.wikimedia.cloud\",\n-  \"cloud-init\": \"MIME-Version: 1.0\\nContent-Type: multipart/mixed; boundary=\\\"XXXXboundary text\\\"\\n\\nThis is a multipart config in MIME format.\\nIt contains a cloud-init config followed by\\na first boot script.\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/jinja2; charset=\\\"us-ascii\\\"\\n\\n## template: jinja\\n#cloud-config\\n\\ndatasource:\\n  OpenStack:\\n    timeout: 10\\n    max_wait: 120\\n    retries: 10\\n\\nhostname: {{ds.meta_data.name}}\\nfqdn: {{ds.meta_data.name}}.{{ds.meta_data.project_id}}.eqiad1.wikimedia.cloud\\n\\n# Prevent puppet from doing things before we're ready.\\n# APT fails to acquire GPG keys if packages dirmngr or gpg are missing\\nbootcmd:\\n  - [ cloud-init-per, once, disablepuppet, puppet, agent, --disable, \\\"prevent puppet runs during cloud-init\\\" ]\\n  - [ cloud-init-per, once, purgeresolved, /usr/bin/dpkg, --purge, systemd-resolved, libnss-resolve ]\\n  - [ cloud-init-per, once, dirmngr-aptupdate, apt-get, update ]\\n  - [ cloud-init-per, once, dirmngr-aptinstall, apt-get, install, \\\"-y\\\", dirmngr ]\\n  - [ cloud-init-per, once, gnupginstall, apt-get, install, \\\"-y\\\", gnupg ]\\n\\n\\n# /etc/block-ldap-key-lookup:\\n#   Prevent non-root logins while the VM is being setup\\n#   The ssh-key-ldap-lookup script rejects non-root user logins if this file\\n#   is present.\\n#\\n# /etc/rsyslog.d/60-puppet.conf:\\n#   Enable console logging for puppet\\n#\\n# /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf:\\n#   Enable root console on serial0\\n#   (cloud-init will create any needed parent dirs)\\n#\\n# /etc/apt/preferences.d/wikimedia.pref\\n#   Prefer packages from wikimedia repo\\nwrite_files:\\n    - content: \\\"VM is work in progress\\\"\\n      path: /etc/block-ldap-key-lookup\\n    - content: \\\"daemon.* |/dev/console\\\"\\n      path: /etc/rsyslog.d/60-puppet.conf\\n    - content: {{ds.meta_data.project_id}}\\n      path: /etc/openstack/project_id\\n    - content: {{ds.ec2_metadata.local_ipv4}}\\n      path: /etc/openstack/host_ip\\n    - content: UNKNOWN\\n      path: /etc/openstack/project_name\\n    - content: |\\n        [Service]\\n        ExecStart=\\n        ExecStart=-/sbin/agetty --autologin root --noclear %I $TERM\\n      path: /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf\\n    - content: |\\n        Package: *\\n        Pin: release o=Wikimedia\\n        Pin-Priority: 1001\\n      path: /etc/apt/preferences.d/wikimedia.pref\\n    - content: |\\n        clouds:\\n          novaobserver:\\n            auth:\\n              auth_url: https://openstack.eqiad1.wikimediacloud.org:25000/v3\\n              username: novaobserver\\n              password: dummy_observer_password\\n              project_id: observer\\n              user_domain_id: default\\n              project_domain_id: default\\n            region_name: eqiad1-r\\n            identity_api_version: 3\\n      path: /etc/openstack/clouds.yaml\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.image_build is defined %}\\n    - content: building\\n      path: /.base_image_is_building\\n{% endif %}\\n{% endif %}\\n\\n# resetting ttys0 so root is logged in\\nruncmd:\\n    - [systemctl, enable, serial-getty@ttyS0.service]\\n    - [systemctl, restart, serial-getty@ttyS0.service]\\n\\nssh_pwauth: False\\n\\nmanage_etc_hosts: false\\n\\n\\n# gpg: needed for apt to function properly\\n# pciutils: used by puppet to determine is_virtual\\n# python3-keystoneclient and openstackclient:  used to look up project name\\npackages:\\n    - gpg\\n    - pciutils\\n    - python3-keystoneclient\\n    - python3-openstackclient\\n\\n# Mark out ephemeral0 so that cloud-init leaves it alone;\\n#  we'll let puppet mount it later.\\nmounts:\\n    - [ephemeral0]\\n\\n# You'll see that we're setting apt_preserve_sources_list twice here.  That's\\n#  because there's a bug in cloud-init where it tries to reconcile the\\n#  two settings and if they're different the stage fails. That means that\\n#  if one of them is set differently from the default (True) then nothing\\n#  works.\\n#\\n#  Setting preserve_sources_list avoids clobbering the pre-set repos\\n#  (T340814) but still lets us inject the mediawiki repos.\\napt_preserve_sources_list: True\\napt:\\n    preserve_sources_list: True\\n    sources:\\n        wikimedia.list:\\n            source: |\\n                deb http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n                deb-src http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n            filename: wikimedia.list\\n            keyid: 9D392D3FFADF18FB\\n\\npackage_update: true\\npackage_upgrade: true\\n\\nfinal_message: cloud-init has finished\\n\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.install_puppet is defined %}\\n# This configures puppet and starts the puppet agent (which we don't want).\\n#  When the boot script starts we'll kill the agent and run puppet synchronously.\\npuppet:\\n    start_service: false\\n    exec: false\\n    package_name: puppet-agent\\n    conf:\\n        agent:\\n            server: 'puppet'\\n            certname: '%f'\\n{% endif %}\\n{% endif %}\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/text/x-shellscript; charset=\\\"us-ascii\\\"\\n#!/bin/bash\\n\\nset -x\\n\\n# Set fqdn with project name.\\n# We have to do this here since project_name isn't available via metadata.\\nhostname=`hostname`\\nip=`cat /etc/openstack/host_ip`\\nproject_id=`cat /etc/openstack/project_id`\\nproject_name=`openstack project show --os-cloud novaobserver -c name $project_id -f value`\\necho $project_name > /etc/openstack/project_name\\nfqdn=$hostname.$project_name.eqiad1.wikimedia.cloud\\ncat > /etc/hosts<< EOF\\n$ip $fqdn $hostname\\n127.0.0.1 localhost\\n\\n# The following lines are desirable for IPv6 capable hosts\\n::1 ip6-localhost ip6-loopback\\nfe00::0 ip6-localnet\\nff00::0 ip6-mcastprefix\\nff02::1 ip6-allnodes\\nff02::2 ip6-allrouters\\nff02::3 ip6-allhosts\\nEOF\\n\\nif ! command -v apt-get >/dev/null; then\\n    # If we don't have apt then none of our init script will\\n    #  work so let's just get out of here\\n\\n    rm /etc/block-ldap-key-lookup\\n    touch /.cloud-init-finished\\n    exit 0\\nfi\\n\\n# Function to test if puppet is running or not\\npuppet_is_running() {\\n    PUPPETLOCK=`puppet agent --configprint agent_catalog_run_lockfile`\\n    echo \\\"checking if puppet is running by looking at ${PUPPETLOCK}\\\"\\n    # If no lockfile is defined, we can't really tell what's happening.\\n    # Assume puppet is not running in this case\\n    test -n \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # If the lockfile is not present, puppet is not running\\n    test -e \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # Now let's see if the PID at $PUPPETLOCK is indeed present and running\\n    local puppetpid=$(cat \\\"$PUPPETLOCK\\\")\\n    local cmdline_file=\\\"/proc/${puppetpid}/cmdline\\\"\\n    if [ -f \\\"$cmdline_file\\\" ]; then\\n        # Puppet is indeed running\\n        grep -q puppet \\\"$cmdline_file\\\" && return 0\\n    fi\\n\\n    # The lock file is stale, delete it so that we're all on the same page\\n    echo \\\"Clearing stale puppet agent lock for pid ${puppetpid}\\\"\\n    rm $PUPPETLOCK\\n    return 1\\n}\\n\\n# loop function to wait for puppet to finish its execution\\nwait_for_puppet() {\\n    local attempts=${1:-30}\\n    for _ in $(seq \\\"$attempts\\\"); do\\n        if ! puppet_is_running; then\\n            return 0\\n        fi\\n        echo \\\"Waiting for puppet run to complete\\\"\\n        sleep 10\\n    done\\n    # If puppet is still running at this point, report an error\\n    echo \\\"Tired of waiting for puppet run to complete; giving up\\\"\\n    return 1\\n}\\n\\n# Upstream debian cloud base images come with chrony installed,\\n# as well as a service user with an invalid (>499) ID.  We can't\\n# just uninstall chrony because systemd depends on it; installing\\n# systemd-timesyncd uninstalls chrony but then runs afoul of the\\n# user ID that was created by chrony. To work around all that,\\n# we detect if chrony is installed and if it is, remove the\\n# user it installed (without uninstalling chrony).  That unblocks\\n# us for a subsequent install of systemd-timesyncd (which /does/\\n# uninstall chrony).\\nif /usr/bin/dpkg -l chrony | grep -q ^ii; then\\n    /usr/sbin/userdel systemd-timesync\\n    /usr/sbin/userdel systemd-coredump\\n    apt install -y systemd-timesyncd\\nfi\\n\\nsystemctl restart nscd.service\\ndpkg-reconfigure -fnoninteractive -pcritical openssh-server\\nsystemctl restart ssh.service\\nnscd -i hosts\\n\\n# This VM might have puppet installed in its base image, or it\\n#  might have the metadata install_puppet flag set. If neither\\n#  of these is true then skip puppet entirely.\\n#\\n# Note that in Bookworm and after the puppet client\\n#  is in a package named puppet-agent; earlier it was\\n#  just named 'puppet'.\\nif dpkg -s puppet || dpkg -s puppet-agent; then\\n    # Run puppet, twice.  The second time is just to pick up packages\\n    #  that may have been unavailable in apt before the first puppet run\\n    #  updated sources.list\\n    puppet agent --enable\\n\\n    # Just in case puppet is already running for some reason, wait\\n    #  for it to finish.\\n    wait_for_puppet 120\\n    service puppet stop\\n    wait_for_puppet 120\\n\\n    # Sometimes the previous --enable doesn't take. Don't know why but\\n    #  it's harmless to try again\\n    puppet agent --enable\\n    puppet agent --onetime --verbose --no-daemonize --no-splay --show_diff --certname `hostname -f`\\n    # Refresh ldap now that puppet has updated our ldap.conf\\n    systemctl restart nslcd.service\\n\\n    # Ensure all NFS mounts are mounted\\n    mount_attempts=1\\n    until [ $mount_attempts -gt 10 ]\\n    do\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\"\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\" >> /etc/block-ldap-key-lookup\\n        ((mount_attempts++))\\n        /usr/bin/timeout --preserve-status -k 10s 20s /bin/mount -a && break\\n        # Sleep for 10s before next attempt\\n        sleep 10\\n    done\\n\\n    # Run puppet again post mounting NFS mounts (if all the mounts hadn't been mounted\\n    # before, the puppet code that ensures the symlinks are created, etc may not\\n    # have run)\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # The following will be a no-op on most distros, but\\n    #  as of 08-2025 Trixie comes packaged with puppet 8\\n    #  and the puppet manifests themselves seek to downgrade\\n    #  to 7.\\n    apt install -y --allow-downgrades puppet-agent\\n    apt-get -y autoremove\\n\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # sssd might be caching sudo rules and other ldap things from before puppet\\n    # finished setting up ldap integration. Restart sssd and clear its cache.\\n    # On some old images we won't have sssd installed; in that case this will\\n    # throw some harmless errors.\\n    systemctl stop sssd\\n    rm -rf /var/lib/sss/db/*\\n    systemctl restart sssd\\nfi\\n\\nif test -f \\\"/.base_image_is_building\\\"; then\\n    # Now that puppet has configured apt properly, make sure\\n    # we have the right version of puppet installed.\\n    # The package may be named 'puppet' or may be named\\n    # 'puppet-agent' so one of these might fail.\\n    apt-get install --allow-downgrades -y puppet\\n    apt-get install --allow-downgrades -y puppet-agent\\n\\n    # stop puppet runs, clean up log files, and generally get this ready\\n    # for another cloud-init run.\\n    echo Cleaning up files before base image creation\\n    rm -f /.base_image_is_building\\n    puppet agent --disable \\\"waiting for initial VM setup\\\"\\n    rm -f /lib/systemd/system/puppet-agent-timer.timer\\n    rm -f /etc/cron.d/puppet\\n    rm -rf /var/lib/cloud/instances\\n    rm -rf /var/lib/puppet/ssl\\n    rm -rf /etc/dhcp/dhclient-enter-hooks.d/nodnsupdate\\n    cloud-init clean\\n    rm -f /.cloud-init-finished\\n    echo 'uninitialized' > etc/machine-id\\n    find /var/log -type f -exec rm '{}' \\\\;\\nelse\\n    # Remove the non-root login restriction. If this is the\\n    # initial base image, we just leave logins disabled.\\n    rm /etc/block-ldap-key-lookup\\n\\n    # mark the host as ready, from now on alerts and checks will send notifications (ex. puppet_alert.py)\\n    touch /.cloud-init-finished\\nfi\\n\\nexit 0\\n--XXXXboundary text\\n\"\n+  \"cloud-init\": \"MIME-Version: 1.0\\nContent-Type: multipart/mixed; boundary=\\\"XXXXboundary text\\\"\\n\\nThis is a multipart config in MIME format.\\nIt contains a cloud-init config followed by\\na first boot script.\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/jinja2; charset=\\\"us-ascii\\\"\\n\\n## template: jinja\\n#cloud-config\\n\\ndatasource:\\n  OpenStack:\\n    timeout: 10\\n    max_wait: 120\\n    retries: 10\\n\\nhostname: {{ds.meta_data.name}}\\nfqdn: {{ds.meta_data.name}}.{{ds.meta_data.project_id}}.eqiad1.wikimedia.cloud\\n\\n# Prevent puppet from doing things before we're ready.\\n# APT fails to acquire GPG keys if packages dirmngr or gpg are missing\\nbootcmd:\\n  - [ cloud-init-per, once, disablepuppet, puppet, agent, --disable, \\\"prevent puppet runs during cloud-init\\\" ]\\n  - [ cloud-init-per, once, purgeresolved, /usr/bin/dpkg, --purge, systemd-resolved, libnss-resolve ]\\n  - [ cloud-init-per, once, dirmngr-aptupdate, apt-get, update ]\\n  - [ cloud-init-per, once, dirmngr-aptinstall, apt-get, install, \\\"-y\\\", dirmngr ]\\n  - [ cloud-init-per, once, gnupginstall, apt-get, install, \\\"-y\\\", gnupg ]\\n\\n\\n# /etc/block-ldap-key-lookup:\\n#   Prevent non-root logins while the VM is being setup\\n#   The ssh-key-ldap-lookup script rejects non-root user logins if this file\\n#   is present.\\n#\\n# /etc/rsyslog.d/60-puppet.conf:\\n#   Enable console logging for puppet\\n#\\n# /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf:\\n#   Enable root console on serial0\\n#   (cloud-init will create any needed parent dirs)\\n#\\n# /etc/apt/preferences.d/wikimedia.pref\\n#   Prefer packages from wikimedia repo\\nwrite_files:\\n    - content: \\\"VM is work in progress\\\"\\n      path: /etc/block-ldap-key-lookup\\n    - content: \\\"daemon.* |/dev/console\\\"\\n      path: /etc/rsyslog.d/60-puppet.conf\\n    - content: {{ds.meta_data.project_id}}\\n      path: /etc/openstack/project_id\\n    - content: {{ds.ec2_metadata.local_ipv4}}\\n      path: /etc/openstack/host_ip\\n    - content: UNKNOWN\\n      path: /etc/openstack/project_name\\n    - content: |\\n        [Service]\\n        ExecStart=\\n        ExecStart=-/sbin/agetty --autologin root --noclear %I $TERM\\n      path: /etc/systemd/system/serial-getty@ttyS0.service.d/override.conf\\n    - content: |\\n        Package: *\\n        Pin: release o=Wikimedia\\n        Pin-Priority: 1001\\n      path: /etc/apt/preferences.d/wikimedia.pref\\n    - content: |\\n          # Cumin Controller.\\n              from=10.64.16.154,2620:0:861:102:10:64:16:154,10.192.32.49,2620:0:860:103:10:192:32:49,10.192.15.6,2620:0:860:110:10:192:15:6,no-agent-forwarding,no-port-forwarding,no-x11-forwarding,no-user-rc ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICcav+ECiF6hW2XRuP7R8nqDw4hPlD0OChsGvB6K27jK root@cloudinfra-internal-puppetmaster-02\\n\\n      path: /etc/ssh/userkeys/root.d/cumin\\n      permissions: '0444'\\n      owner: root:root\\n    - content: |\\n        clouds:\\n          novaobserver:\\n            auth:\\n              auth_url: https://openstack.eqiad1.wikimediacloud.org:25000/v3\\n              username: novaobserver\\n              password: dummy_observer_password\\n              project_id: observer\\n              user_domain_id: default\\n              project_domain_id: default\\n            region_name: eqiad1-r\\n            identity_api_version: 3\\n      path: /etc/openstack/clouds.yaml\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.image_build is defined %}\\n    - content: building\\n      path: /.base_image_is_building\\n{% endif %}\\n{% endif %}\\n\\n# resetting ttys0 so root is logged in\\nruncmd:\\n    - [systemctl, enable, serial-getty@ttyS0.service]\\n    - [systemctl, restart, serial-getty@ttyS0.service]\\n\\nssh_pwauth: False\\n\\nmanage_etc_hosts: false\\n\\n\\n# gpg: needed for apt to function properly\\n# pciutils: used by puppet to determine is_virtual\\n# python3-keystoneclient and openstackclient:  used to look up project name\\npackages:\\n    - gpg\\n    - pciutils\\n    - python3-keystoneclient\\n    - python3-openstackclient\\n\\n# Mark out ephemeral0 so that cloud-init leaves it alone;\\n#  we'll let puppet mount it later.\\nmounts:\\n    - [ephemeral0]\\n\\n# You'll see that we're setting apt_preserve_sources_list twice here.  That's\\n#  because there's a bug in cloud-init where it tries to reconcile the\\n#  two settings and if they're different the stage fails. That means that\\n#  if one of them is set differently from the default (True) then nothing\\n#  works.\\n#\\n#  Setting preserve_sources_list avoids clobbering the pre-set repos\\n#  (T340814) but still lets us inject the mediawiki repos.\\napt_preserve_sources_list: True\\napt:\\n    preserve_sources_list: True\\n    sources:\\n        wikimedia.list:\\n            source: |\\n                deb http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n                deb-src http://apt.wikimedia.org/wikimedia $RELEASE-wikimedia main\\n            filename: wikimedia.list\\n            keyid: 9D392D3FFADF18FB\\n\\npackage_update: true\\npackage_upgrade: true\\n\\nfinal_message: cloud-init has finished\\n\\n{% if ds.meta_data.meta is defined %}\\n{% if ds.meta_data.meta.install_puppet is defined %}\\n# This configures puppet and starts the puppet agent (which we don't want).\\n#  When the boot script starts we'll kill the agent and run puppet synchronously.\\npuppet:\\n    start_service: false\\n    exec: false\\n    package_name: puppet-agent\\n    conf:\\n        agent:\\n            server: 'puppet'\\n            certname: '%f'\\n{% endif %}\\n{% endif %}\\n\\n--XXXXboundary text\\nMIME-Version: 1.0\\nContent-Type: text/text/x-shellscript; charset=\\\"us-ascii\\\"\\n#!/bin/bash\\n\\nset -x\\n\\n# Set fqdn with project name.\\n# We have to do this here since project_name isn't available via metadata.\\nhostname=`hostname`\\nip=`cat /etc/openstack/host_ip`\\nproject_id=`cat /etc/openstack/project_id`\\nproject_name=`openstack project show --os-cloud novaobserver -c name $project_id -f value`\\necho $project_name > /etc/openstack/project_name\\nfqdn=$hostname.$project_name.eqiad1.wikimedia.cloud\\ncat > /etc/hosts<< EOF\\n$ip $fqdn $hostname\\n127.0.0.1 localhost\\n\\n# The following lines are desirable for IPv6 capable hosts\\n::1 ip6-localhost ip6-loopback\\nfe00::0 ip6-localnet\\nff00::0 ip6-mcastprefix\\nff02::1 ip6-allnodes\\nff02::2 ip6-allrouters\\nff02::3 ip6-allhosts\\nEOF\\n\\nif ! command -v apt-get >/dev/null; then\\n    # If we don't have apt then none of our init script will\\n    #  work so let's just get out of here\\n\\n    rm /etc/block-ldap-key-lookup\\n    touch /.cloud-init-finished\\n    exit 0\\nfi\\n\\n# Function to test if puppet is running or not\\npuppet_is_running() {\\n    PUPPETLOCK=`puppet agent --configprint agent_catalog_run_lockfile`\\n    echo \\\"checking if puppet is running by looking at ${PUPPETLOCK}\\\"\\n    # If no lockfile is defined, we can't really tell what's happening.\\n    # Assume puppet is not running in this case\\n    test -n \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # If the lockfile is not present, puppet is not running\\n    test -e \\\"$PUPPETLOCK\\\" || return 1\\n\\n    # Now let's see if the PID at $PUPPETLOCK is indeed present and running\\n    local puppetpid=$(cat \\\"$PUPPETLOCK\\\")\\n    local cmdline_file=\\\"/proc/${puppetpid}/cmdline\\\"\\n    if [ -f \\\"$cmdline_file\\\" ]; then\\n        # Puppet is indeed running\\n        grep -q puppet \\\"$cmdline_file\\\" && return 0\\n    fi\\n\\n    # The lock file is stale, delete it so that we're all on the same page\\n    echo \\\"Clearing stale puppet agent lock for pid ${puppetpid}\\\"\\n    rm $PUPPETLOCK\\n    return 1\\n}\\n\\n# loop function to wait for puppet to finish its execution\\nwait_for_puppet() {\\n    local attempts=${1:-30}\\n    for _ in $(seq \\\"$attempts\\\"); do\\n        if ! puppet_is_running; then\\n            return 0\\n        fi\\n        echo \\\"Waiting for puppet run to complete\\\"\\n        sleep 10\\n    done\\n    # If puppet is still running at this point, report an error\\n    echo \\\"Tired of waiting for puppet run to complete; giving up\\\"\\n    return 1\\n}\\n\\n# Upstream debian cloud base images come with chrony installed,\\n# as well as a service user with an invalid (>499) ID.  We can't\\n# just uninstall chrony because systemd depends on it; installing\\n# systemd-timesyncd uninstalls chrony but then runs afoul of the\\n# user ID that was created by chrony. To work around all that,\\n# we detect if chrony is installed and if it is, remove the\\n# user it installed (without uninstalling chrony).  That unblocks\\n# us for a subsequent install of systemd-timesyncd (which /does/\\n# uninstall chrony).\\nif /usr/bin/dpkg -l chrony | grep -q ^ii; then\\n    /usr/sbin/userdel systemd-timesync\\n    /usr/sbin/userdel systemd-coredump\\n    apt install -y systemd-timesyncd\\nfi\\n\\nsystemctl restart nscd.service\\ndpkg-reconfigure -fnoninteractive -pcritical openssh-server\\nsystemctl restart ssh.service\\nnscd -i hosts\\n\\n# This VM might have puppet installed in its base image, or it\\n#  might have the metadata install_puppet flag set. If neither\\n#  of these is true then skip puppet entirely.\\n#\\n# Note that in Bookworm and after the puppet client\\n#  is in a package named puppet-agent; earlier it was\\n#  just named 'puppet'.\\nif dpkg -s puppet || dpkg -s puppet-agent; then\\n    # Run puppet, twice.  The second time is just to pick up packages\\n    #  that may have been unavailable in apt before the first puppet run\\n    #  updated sources.list\\n    puppet agent --enable\\n\\n    # Just in case puppet is already running for some reason, wait\\n    #  for it to finish.\\n    wait_for_puppet 120\\n    service puppet stop\\n    wait_for_puppet 120\\n\\n    # Sometimes the previous --enable doesn't take. Don't know why but\\n    #  it's harmless to try again\\n    puppet agent --enable\\n    puppet agent --onetime --verbose --no-daemonize --no-splay --show_diff --certname `hostname -f`\\n    # Refresh ldap now that puppet has updated our ldap.conf\\n    systemctl restart nslcd.service\\n\\n    # Ensure all NFS mounts are mounted\\n    mount_attempts=1\\n    until [ $mount_attempts -gt 10 ]\\n    do\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\"\\n        echo \\\"Ensuring all NFS mounts are mounted, attempt ${mount_attempts}\\\" >> /etc/block-ldap-key-lookup\\n        ((mount_attempts++))\\n        /usr/bin/timeout --preserve-status -k 10s 20s /bin/mount -a && break\\n        # Sleep for 10s before next attempt\\n        sleep 10\\n    done\\n\\n    # Run puppet again post mounting NFS mounts (if all the mounts hadn't been mounted\\n    # before, the puppet code that ensures the symlinks are created, etc may not\\n    # have run)\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # The following will be a no-op on most distros, but\\n    #  as of 08-2025 Trixie comes packaged with puppet 8\\n    #  and the puppet manifests themselves seek to downgrade\\n    #  to 7.\\n    apt install -y --allow-downgrades puppet-agent\\n    apt-get -y autoremove\\n\\n    wait_for_puppet 10\\n    apt-get update\\n    puppet agent -t\\n\\n    # sssd might be caching sudo rules and other ldap things from before puppet\\n    # finished setting up ldap integration. Restart sssd and clear its cache.\\n    # On some old images we won't have sssd installed; in that case this will\\n    # throw some harmless errors.\\n    systemctl stop sssd\\n    rm -rf /var/lib/sss/db/*\\n    systemctl restart sssd\\nfi\\n\\nif test -f \\\"/.base_image_is_building\\\"; then\\n    # Now that puppet has configured apt properly, make sure\\n    # we have the right version of puppet installed.\\n    # The package may be named 'puppet' or may be named\\n    # 'puppet-agent' so one of these might fail.\\n    apt-get install --allow-downgrades -y puppet\\n    apt-get install --allow-downgrades -y puppet-agent\\n\\n    # stop puppet runs, clean up log files, and generally get this ready\\n    # for another cloud-init run.\\n    echo Cleaning up files before base image creation\\n    rm -f /.base_image_is_building\\n    puppet agent --disable \\\"waiting for initial VM setup\\\"\\n    rm -f /lib/systemd/system/puppet-agent-timer.timer\\n    rm -f /etc/cron.d/puppet\\n    rm -rf /var/lib/cloud/instances\\n    rm -rf /var/lib/puppet/ssl\\n    rm -rf /etc/dhcp/dhclient-enter-hooks.d/nodnsupdate\\n    cloud-init clean\\n    rm -f /.cloud-init-finished\\n    echo 'uninitialized' > etc/machine-id\\n    find /var/log -type f -exec rm '{}' \\\\;\\nelse\\n    # Remove the non-root login restriction. If this is the\\n    # initial base image, we just leave logins disabled.\\n    rm /etc/block-ldap-key-lookup\\n\\n    # mark the host as ready, from now on alerts and checks will send notifications (ex. puppet_alert.py)\\n    touch /.cloud-init-finished\\nfi\\n\\nexit 0\\n--XXXXboundary text\\n\"\n }"}, {"resource": "Class[Profile::Openstack::Base::Nova::Api::Service]", "parameters": "--- Class[Profile::Openstack::Base::Nova::Api::Service].orig\n+++ Class[Profile::Openstack::Base::Nova::Api::Service]\n\n+    cumin_masters => ['10.64.16.154', '2620:0:861:102:10:64:16:154', '10.192.32.49', '2620:0:860:103:10:192:32:49', '10.192.15.6', '2620:0:860:110:10:192:15:6']\n"}, {"resource": "Class[Openstack::Nova::Api::Service]", "parameters": "--- Class[Openstack::Nova::Api::Service].orig\n+++ Class[Openstack::Nova::Api::Service]\n\n+    cumin_masters => ['10.64.16.154', '2620:0:861:102:10:64:16:154', '10.192.32.49', '2620:0:860:103:10:192:32:49', '10.192.15.6', '2620:0:860:110:10:192:15:6']\n"}], "perc_changed": "0.07%"}}}