Compilation results for an-test-client1002.eqiad.wmnet: System changes detected

You can retrieve this result from host.json.

Catalog differences

Summary

Total Resources:	4896
Resources added:	22
Resources removed:	0
Resources modified:	23
Change percentage:	0.92%

Resources only in the new catalog

File[/usr/bin/spark35-class]
File[/etc/ferm/conf.d/10_spark35_ui_port]
File[/usr/bin/spark35-shell]
Firewall::Service[spark35-ui-port]
Ferm::Service[spark35_driver_blockmanager]
Firewall::Service[spark35-driver-blockmanager]
File[/etc/spark35/conf/spark-env.sh]
File[/etc/spark35]
Ferm::Service[spark35_ui_port]
File[/usr/lib/spark35]
File[/usr/bin/spark35-sql]
File[/etc/ferm/conf.d/10_spark35_driver]
File[/usr/bin/spark35-submit]
File[/etc/spark35/conf]
Ferm::Service[spark35_driver]
Firewall::Service[spark35-driver]
File[/etc/spark35/conf/spark-defaults.conf]
File[/etc/spark35/conf/log4j2.properties]
File[/etc/spark35/conf/hive-site.xml]
File[/usr/bin/spark35-pyspark]
File[/etc/ferm/conf.d/10_spark35_driver_blockmanager]
Class[Profile::Hadoop::Spark35]

Resources modified

File[/usr/bin/spark35-pyspark]

Parameters differences:

--- File[/usr/bin/spark35-pyspark].orig
+++ File[/usr/bin/spark35-pyspark]

+    owner  => root
+    group  => root
+    ensure => file
+    mode   => 0755

Content differences:

--- /usr/bin/spark35-pyspark.orig
+++ /usr/bin/spark35-pyspark
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Managed by Puppet
+export SPARK_CONF_DIR=/etc/spark35/conf
+exec /usr/lib/spark35/bin/pyspark "$@"

File[/usr/bin/spark35-submit]

Parameters differences:

--- File[/usr/bin/spark35-submit].orig
+++ File[/usr/bin/spark35-submit]

+    owner  => root
+    group  => root
+    ensure => file
+    mode   => 0755

Content differences:

--- /usr/bin/spark35-submit.orig
+++ /usr/bin/spark35-submit
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Managed by Puppet
+export SPARK_CONF_DIR=/etc/spark35/conf
+exec /usr/lib/spark35/bin/spark-submit "$@"

File[/usr/lib/spark35]

Parameters differences:

--- File[/usr/lib/spark35].orig
+++ File[/usr/lib/spark35]

+    owner  => root
+    group  => root
+    ensure => link

Class[Profile::Analytics::Cluster::Client]

Parameters differences:

--- Class[Profile::Analytics::Cluster::Client].orig
+++ Class[Profile::Analytics::Cluster::Client]

@@
-    require => ['Class[Profile::Analytics::Cluster::Packages::Common]', 'Class[Profile::Hadoop::Common]', 'Class[Profile::Hive::Client]', 'Class[Profile::Hadoop::Spark3]']
+    require => ['Class[Profile::Analytics::Cluster::Packages::Common]', 'Class[Profile::Hadoop::Common]', 'Class[Profile::Hive::Client]']

File[/etc/ferm/conf.d/10_spark35_driver]

Parameters differences:

--- File[/etc/ferm/conf.d/10_spark35_driver].orig
+++ File[/etc/ferm/conf.d/10_spark35_driver]

+    require => File[/etc/ferm/conf.d]
+    notify  => Service[ferm]
+    ensure  => present
+    owner   => root
+    group   => root
+    tag     => ferm
+    mode    => 0400

Content differences:

--- /etc/ferm/conf.d/10_spark35_driver.orig
+++ /etc/ferm/conf.d/10_spark35_driver
@@ -0,0 +1,6 @@
+# Autogenerated by puppet. DO NOT EDIT BY HAND!
+#
+# 
+&R_SERVICE(tcp, 12000:12100, $ANALYTICS_NETWORKS);
+
+

File[/etc/ferm/conf.d/10_spark35_driver_blockmanager]

Parameters differences:

--- File[/etc/ferm/conf.d/10_spark35_driver_blockmanager].orig
+++ File[/etc/ferm/conf.d/10_spark35_driver_blockmanager]

+    require => File[/etc/ferm/conf.d]
+    notify  => Service[ferm]
+    ensure  => present
+    owner   => root
+    group   => root
+    tag     => ferm
+    mode    => 0400

Content differences:

--- /etc/ferm/conf.d/10_spark35_driver_blockmanager.orig
+++ /etc/ferm/conf.d/10_spark35_driver_blockmanager
@@ -0,0 +1,6 @@
+# Autogenerated by puppet. DO NOT EDIT BY HAND!
+#
+# 
+&R_SERVICE(tcp, 13000:13100, $ANALYTICS_NETWORKS);
+
+

File[/etc/spark35]

Parameters differences:

--- File[/etc/spark35].orig
+++ File[/etc/spark35]

+    owner  => root
+    group  => root
+    ensure => directory
+    mode   => 0755

File[/usr/bin/spark35-sql]

Parameters differences:

--- File[/usr/bin/spark35-sql].orig
+++ File[/usr/bin/spark35-sql]

+    owner  => root
+    group  => root
+    ensure => file
+    mode   => 0755

Content differences:

--- /usr/bin/spark35-sql.orig
+++ /usr/bin/spark35-sql
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Managed by Puppet
+export SPARK_CONF_DIR=/etc/spark35/conf
+exec /usr/lib/spark35/bin/spark-sql "$@"

Ferm::Service[spark35_driver_blockmanager]

Parameters differences:

--- Ferm::Service[spark35_driver_blockmanager].orig
+++ Ferm::Service[spark35_driver_blockmanager]

+    notrack             => False
+    proto               => tcp
+    prio                => 10
+    port_range          => [13000, 13100]
+    unrestricted_access => False
+    src_sets            => ['ANALYTICS_NETWORKS']
+    ensure              => present
+    desc                =>

File[/etc/spark35/conf]

Parameters differences:

--- File[/etc/spark35/conf].orig
+++ File[/etc/spark35/conf]

+    owner  => root
+    group  => root
+    ensure => directory
+    mode   => 0755

Ferm::Service[spark35_driver]

Parameters differences:

--- Ferm::Service[spark35_driver].orig
+++ Ferm::Service[spark35_driver]

+    notrack             => False
+    proto               => tcp
+    prio                => 10
+    port_range          => [12000, 12100]
+    unrestricted_access => False
+    src_sets            => ['ANALYTICS_NETWORKS']
+    ensure              => present
+    desc                =>

File[/etc/spark35/conf/hive-site.xml]

Parameters differences:

--- File[/etc/spark35/conf/hive-site.xml].orig
+++ File[/etc/spark35/conf/hive-site.xml]

+    target => /etc/hive/conf.analytics-test-hadoop/hive-site.xml
+    owner  => root
+    group  => root
+    ensure => link

File[/etc/ferm/conf.d/10_spark35_ui_port]

Parameters differences:

--- File[/etc/ferm/conf.d/10_spark35_ui_port].orig
+++ File[/etc/ferm/conf.d/10_spark35_ui_port]

+    require => File[/etc/ferm/conf.d]
+    notify  => Service[ferm]
+    ensure  => present
+    owner   => root
+    group   => root
+    tag     => ferm
+    mode    => 0400

Content differences:

--- /etc/ferm/conf.d/10_spark35_ui_port.orig
+++ /etc/ferm/conf.d/10_spark35_ui_port
@@ -0,0 +1,6 @@
+# Autogenerated by puppet. DO NOT EDIT BY HAND!
+#
+# 
+&R_SERVICE(tcp, 4040:4140, $ANALYTICS_NETWORKS);
+
+

Firewall::Service[spark35-driver]

Parameters differences:

--- Firewall::Service[spark35-driver].orig
+++ Firewall::Service[spark35-driver]

+    notrack             => False
+    proto               => tcp
+    prio                => 10
+    port_range          => [12000, 12100]
+    unrestricted_access => False
+    src_sets            => ['ANALYTICS_NETWORKS']
+    ensure              => present
+    desc                =>

Firewall::Service[spark35-ui-port]

Parameters differences:

--- Firewall::Service[spark35-ui-port].orig
+++ Firewall::Service[spark35-ui-port]

+    notrack             => False
+    proto               => tcp
+    prio                => 10
+    port_range          => [4040, 4140]
+    unrestricted_access => False
+    src_sets            => ['ANALYTICS_NETWORKS']
+    ensure              => present
+    desc                =>

Ferm::Service[spark35_ui_port]

Parameters differences:

--- Ferm::Service[spark35_ui_port].orig
+++ Ferm::Service[spark35_ui_port]

+    notrack             => False
+    proto               => tcp
+    prio                => 10
+    port_range          => [4040, 4140]
+    unrestricted_access => False
+    src_sets            => ['ANALYTICS_NETWORKS']
+    ensure              => present
+    desc                =>

File[/etc/spark35/conf/spark-defaults.conf]

Parameters differences:

--- File[/etc/spark35/conf/spark-defaults.conf].orig
+++ File[/etc/spark35/conf/spark-defaults.conf]

+    owner => root
+    group => root
+    mode  => 0644

Content differences:

--- /etc/spark35/conf/spark-defaults.conf.orig
+++ /etc/spark35/conf/spark-defaults.conf
@@ -0,0 +1,74 @@
+# NOTE: This file is managed by Puppet.
+
+# Default system properties included when running spark-submit.
+# This is useful for setting default environmental settings.
+
+# Example:
+# spark.master                     spark://master:7077
+# spark.eventLog.enabled           true
+# spark.eventLog.dir               hdfs://namenode:8021/directory
+# spark.serializer                 org.apache.spark.serializer.KryoSerializer
+# spark.driver.memory              5g
+# spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
+
+# We configure the default value of spark.mater to be yarn since this is by far the most common
+# type of usage in our environment. It can be overridden on the command-line to use 'local' and to
+# limit the number of cores allocated. Please see #T393181 and the following docs for more info.
+# https://wikitech.wikimedia.org/wiki/Data_Platform/Systems/Spark#Application_Master
+spark.master                                        yarn
+# Dynamic allocation allows Spark to dynamically scale the cluster resources
+# allocated for an application based on the workload. Only available in YARN mode.
+# More info: https://spark.apache.org/docs/2.1.2/configuration.html#dynamic-allocation
+spark.dynamicAllocation.enabled                     true
+spark.shuffle.service.enabled                       true
+spark.dynamicAllocation.executorIdleTimeout         60s
+spark.dynamicAllocation.cachedExecutorIdleTimeout   3600s
+spark.shuffle.io.maxRetries                         10
+spark.shuffle.io.retryWait                          10s
+# The following two settings are only set if the cluster is configured
+# to have multiple yarn shufflers available and the version is greater than 3.1
+spark.shuffle.service.name = spark_shuffle_3_5
+spark.shuffle.service.port = 7340
+spark.executorEnv.LD_LIBRARY_PATH                   /usr/lib/hadoop/lib/native
+spark.sql.catalogImplementation                     hive
+# This setting adds support for Iceberg SQL extensions like CALL for stored procedures or ALTER TABLE ... WRITE ORDERED BY.
+# See https://iceberg.apache.org/docs/1.2.1/spark-configuration/#sql-extensions
+spark.sql.extensions                                org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
+# These 'spark.sql.catalog.spark_catalog*' settings add support for Iceberg to the default Spark Catalog by wrapping it.
+# See https://iceberg.apache.org/docs/1.2.1/spark-configuration/#replacing-the-session-catalog
+spark.sql.catalog.spark_catalog                     org.apache.iceberg.spark.SparkSessionCatalog
+spark.sql.catalog.spark_catalog.type                hive
+spark.driver.port                                   12000
+spark.port.maxRetries                               100
+spark.ui.port                                       4040
+spark.driver.blockManager.port                      13000
+spark.sql.files.maxPartitionBytes                   268435456
+spark.sql.warehouse.dir                             hdfs:///user/hive/warehouse
+spark.yarn.archive                                  hdfs:///user/spark/share/lib/spark-3.5.8-assembly.jar
+
+# JVMs should use system proxy settings.
+# The system proxy settings are configured via the env vars http_proxy, https_proxy, and no_proxy.
+spark.driver.defaultJavaOptions                     -Djava.net.useSystemProxies=True
+spark.executor.defaultJavaOptions                   -Djava.net.useSystemProxies=True
+
+
+spark.authenticate                                  true
+# Spark IO encryption settings are not enabled (but listed anyway)
+# since in some use cases (like Refine) they caused exceptions like
+# 'java.io.IOException: Stream is corrupted' when shuffle files were
+# compressed with lz4.
+# spark.io.encryption.enabled                         true
+# spark.io.encryption.keySizeBits                     256
+# spark.io.encryption.keygen.algorithm                HmacSHA256
+spark.network.crypto.enabled                        true
+spark.network.crypto.keyFactoryAlgorithm            PBKDF2WithHmacSHA256
+spark.network.crypto.keyLength                      256
+spark.network.crypto.saslFallback                   false
+
+# Ensure that Python requests lib always use system CA certificates.
+spark.yarn.appMasterEnv.REQUESTS_CA_BUNDLE          /etc/ssl/certs/ca-certificates.crt
+spark.executorEnv.REQUESTS_CA_BUNDLE                /etc/ssl/certs/ca-certificates.crt
+spark.eventLog.enabled                              true
+spark.eventLog.dir                                  hdfs:///var/log/spark
+spark.eventLog.compress                             true
+spark.yarn.historyServer.address                    spark-history-test.svc.eqiad.wmnet:30443

Firewall::Service[spark35-driver-blockmanager]

Parameters differences:

--- Firewall::Service[spark35-driver-blockmanager].orig
+++ Firewall::Service[spark35-driver-blockmanager]

+    notrack             => False
+    proto               => tcp
+    prio                => 10
+    port_range          => [13000, 13100]
+    unrestricted_access => False
+    src_sets            => ['ANALYTICS_NETWORKS']
+    ensure              => present
+    desc                =>

Class[Profile::Hadoop::Spark35]

Parameters differences:

--- Class[Profile::Hadoop::Spark35].orig
+++ Class[Profile::Hadoop::Spark35]

+    spark_yarn_history_address => spark-history-test.svc.eqiad.wmnet:30443
+    extra_settings             => {}
+    encryption_enabled         => True
+    driver_blockmanager_port   => 13000
+    port_max_retries           => 100
+    event_log_compress         => True
+    default_shuffler_version   => 3.5
+    require                    => ['Class[Profile::Hadoop::Common]', 'Class[Profile::Analytics::Conda_analytics]']
+    driver_port                => 12000
+    executor_env_ld_lib_path   => /usr/lib/hadoop/lib/native
+    event_log_dir              => hdfs:///var/log/spark
+    ui_port                    => 4040
+    spark_version              => 3.5.8

File[/etc/spark35/conf/spark-env.sh]

Parameters differences:

--- File[/etc/spark35/conf/spark-env.sh].orig
+++ File[/etc/spark35/conf/spark-env.sh]

+    owner => root
+    group => root
+    mode  => 0644

Content differences:

--- /etc/spark35/conf/spark-env.sh.orig
+++ /etc/spark35/conf/spark-env.sh
@@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+
+# NOTE: This file is managed by Puppet.
+
+# This file is sourced when running various Spark programs.
+# Copy it as spark-env.sh and edit that to configure Spark for your site.
+
+# Options read when launching programs locally with
+# ./bin/run-example or ./bin/spark-submit
+# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
+# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
+# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program
+# - SPARK_CLASSPATH, default classpath entries to append
+
+# Options read by executors and drivers running inside the cluster
+# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
+# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program
+# - SPARK_CLASSPATH, default classpath entries to append
+# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data
+# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos
+
+# Options read in YARN client mode
+# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
+# - SPARK_EXECUTOR_INSTANCES, Number of executors to start (Default: 2)
+# - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1).
+# - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G)
+# - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G)
+
+# Options for the daemons used in the standalone deploy mode
+# - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname
+# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master
+# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y")
+# - SPARK_WORKER_CORES, to set the number of cores to use on this machine
+# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g)
+# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker
+# - SPARK_WORKER_INSTANCES, to set the number of worker processes per node
+# - SPARK_WORKER_DIR, to set the working directory of worker processes
+# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y")
+# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g).
+# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y")
+# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y")
+# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y")
+# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers
+
+# Generic options for the daemons used in the standalone deploy mode
+# - SPARK_CONF_DIR      Alternate conf dir. (Default: ${SPARK_HOME}/conf)
+# - SPARK_LOG_DIR       Where log files are stored.  (Default: ${SPARK_HOME}/logs)
+# - SPARK_PID_DIR       Where the pid file is stored. (Default: /tmp)
+# - SPARK_IDENT_STRING  A string representing this instance of spark. (Default: $USER)
+# - SPARK_NICENESS      The scheduling priority for daemons. (Default: 0)
+# - SPARK_NO_DAEMONIZE  Run the proposed command in the foreground. It will not output a PID file.
+
+# = WMF customizations below =
+#
+# If /etc/hadoop/conf exists, use it as HADOOP_CONF_DIR
+if [ -z "${HADOOP_CONF_DIR}" -a -e "/etc/hadoop/conf" ]; then
+  export HADOOP_CONF_DIR=/etc/hadoop/conf
+fi
+
+# If /usr/lib/hadoop/native exists, use Hadoop native libs from there
+if [ -z "${LD_LIBRARY_PATH}" -a -e /usr/lib/hadoop/lib/native ]; then
+    export LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native
+fi
+
+if [ -z "${SPARK_HOME}" ]; then
+    echo "ERROR: SPARK_HOME must be set before loading ${0}"
+    exit 1
+else
+    echo "SPARK_HOME: ${SPARK_HOME}"
+fi
+
+# Select Hadoop library
+#
+# if [ "${SPARK_HOME}" != /usr/lib/airflow* -a -x "$(command -v hadoop)" ]; then
+#    SPARK_DIST_CLASSPATH=$(hadoop classpath)
+# fi
+HADOOP_VERSION="$(ls $SPARK_HOME/jars/ \
+  | grep hadoop-client \
+  |  sed 's/^hadoop-client-\([0-9]*\.[0-9]*\.[0-9]*\).*$/\1/')"
+if [ -z "${HADOOP_VERSION}" ]; then
+    echo "ERROR: unknown HADOOP_VERSION"
+    exit 1
+fi
+# 2022-09 We are using Hadoop 3 client lib to connect to an Hadoop 2.10 cluster.
+# Hadoop 2.10 version is a backport of version 3 api. That means we can use
+# a client lib at version 3 to access it.
+# This echo is just to remember the explicit version we are using.
+echo "Using Hadoop client lib jars at ${HADOOP_VERSION}, provided by Spark."
+
+# == Determine useful default values for PYSPARK_DRIVER_PYTHON and PYSPARK_PYTHON ==
+#
+# PYSPARK_DRIVER_PYTHON:
+#
+# - User's python env.  If CONDA_PREFIX is set, or if SPARK_HOME is a directory
+#   site-packages/pyspark directory, assume we want to use ipython (or python)
+#   from the user's python env.
+#
+# - Else, use $PYTHON_PREFIX_GLOBAL/bin/ipython3
+#
+# NOTE: If we don't end up setting PYSPARK_DRIVER_PYTHON, then built in spark scripts will do
+# what they usually do to set it (use system python). I.e. it will default to the
+# value of PYSPARK_PYTHON, or the system 'python'.
+#
+# PYSPARK_PYTHON:
+#
+# This must be available on worker nodes.
+#
+# - If we are not running in YARN, then we can safely use the user's python env.
+#
+# - Else if we are running in YARN, we have no way of using a user's local python env.
+#   as the default value here.  It is possible to for users to use a custom python env,
+#   but the user will have ship their packed python env to workers, and handle setting
+#   PYSPARK_PYTHON to the correct value themselves,
+#
+#   So, we default PYSPARK_PYTHON to $PYTHON_PREFIX_GLOBAL/bin/python3
+
+
+# This python env prefix is expected to exist on all potential spark nodes, both masters and workers.
+# This will be used for default PYSPARK_*_PYTHON unless a conda env is currently active.
+PYTHON_PREFIX_GLOBAL=$(realpath ${PYTHON_PREFIX_GLOBAL:-"/opt/conda-analytics-next"})
+
+# If a conda environment is active (CONDA_PREFIX is set),
+# then assume this is the user's python environment
+# If it isn't then check to see if $SPARK_HOME is currently in a pip installed pyspark site-packages
+# inside a python environment.  If it is, then assume that is the user's python environment.
+# NOTE: At this point SPARK_HOME should be set, either by the user or by the find-spark-home
+# script that comes with spark.
+PYTHON_PREFIX_USER=$(realpath "${CONDA_PREFIX:-$(dirname $(dirname $(dirname $(dirname $SPARK_HOME))))}")
+
+# Default PYSPARK_DRIVER_PYTHON to ipython3 if running pyspark CLI directly.
+if [[ -z "${PYSPARK_DRIVER_PYTHON}" && "${0}" == *pyspark ]]; then
+
+    # If a user's python environment exists and has ipython3, use it
+    if [ -n "${PYTHON_PREFIX_USER}" -a -e "${PYTHON_PREFIX_USER}/bin/ipython3" ]; then
+        export PYSPARK_DRIVER_PYTHON="${PYTHON_PREFIX_USER}/bin/ipython3"
+
+    # in case the user's python environment exists without ipython3, but does have python3, use it.
+    elif [ -n "${PYTHON_PREFIX_USER}" -a -e "${PYTHON_PREFIX_USER}/bin/python3" ]; then
+        export PYSPARK_DRIVER_PYTHON="${PYTHON_PREFIX_USER}/bin/python3"
+
+    # Else use PYTHON_PREFIX_GLOBAL's ipython3.
+    else
+        export PYSPARK_DRIVER_PYTHON="${PYTHON_PREFIX_GLOBAL}/bin/ipython3"
+
+    fi
+fi
+
+if [ -z "${PYSPARK_PYTHON}" ]; then
+    # Search the CLI opts to find the master option, if it is given.
+    # PYSPARK_PYTHON needs to be set to something that will work on remote executors
+    # if a conda environment is active.  This will only work if
+    # the SparkSession master is being set via the CLI.
+    spark_master=''
+    for ((i = 1; i <= $#; i++ )); do
+       arg="${!i}"
+       if [[ "${arg}" == --master ]]; then
+           master_index=$((i+1))
+           spark_master="${!master_index}"
+       elif [[ "${arg}" == --master=* ]]; then
+           spark_master="$(echo ${arg} | cut -f2 -d=)"
+       fi
+    done
+
+    # If not YARN and user env's python exists, we can safely use the user env's python.
+    if [[ "${spark_master}" != yarn* && -n "${PYTHON_PREFIX_USER}" && -e "${PYTHON_PREFIX_USER}/bin/python3" ]]; then
+        export PYSPARK_PYTHON="${PYTHON_PREFIX_USER}/bin/python3"
+
+    # else use the PYTHON_PREFIX_GLOBAL's python.
+    # This MUST exist on all worker nodes.
+    else
+        export PYSPARK_PYTHON="${PYTHON_PREFIX_GLOBAL}/bin/python3"
+
+    fi
+fi
+
+# Note: If PYSPARK_DRIVER_PYTHON and PYSPARK_PYTHON are not set at this point,
+# the pyspark script will just use system 'python'.
+test -n "${PYSPARK_DRIVER_PYTHON}" && echo "PYSPARK_DRIVER_PYTHON=${PYSPARK_DRIVER_PYTHON}"
+test -n "${PYSPARK_PYTHON}" && echo "PYSPARK_PYTHON=${PYSPARK_PYTHON}"

File[/etc/spark35/conf/log4j2.properties]

Parameters differences:

--- File[/etc/spark35/conf/log4j2.properties].orig
+++ File[/etc/spark35/conf/log4j2.properties]

+    source => puppet:///modules/profile/hadoop/spark3/spark35_log4j2.properties
+    owner  => root
+    group  => root
+    mode   => 0644

File[/usr/bin/spark35-shell]

Parameters differences:

--- File[/usr/bin/spark35-shell].orig
+++ File[/usr/bin/spark35-shell]

+    owner  => root
+    group  => root
+    ensure => file
+    mode   => 0755

Content differences:

--- /usr/bin/spark35-shell.orig
+++ /usr/bin/spark35-shell
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Managed by Puppet
+export SPARK_CONF_DIR=/etc/spark35/conf
+exec /usr/lib/spark35/bin/spark-shell "$@"

File[/usr/bin/spark35-class]

Parameters differences:

--- File[/usr/bin/spark35-class].orig
+++ File[/usr/bin/spark35-class]

+    owner  => root
+    group  => root
+    ensure => file
+    mode   => 0755

Content differences:

--- /usr/bin/spark35-class.orig
+++ /usr/bin/spark35-class
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Managed by Puppet
+export SPARK_CONF_DIR=/etc/spark35/conf
+exec /usr/lib/spark35/bin/spark-class "$@"

Compilation results for an-test-client1002.eqiad.wmnet: System changes detected

Catalog differences

Summary

Resources only in the new catalog

Resources modified

Relevant files