--- Class[Profile::Hadoop::Common].orig
+++ Class[Profile::Hadoop::Common]
@@
- hadoop_clusters => {'analytics-test-hadoop': {'zookeeper_cluster_name': 'analytics-eqiad', 'resourcemanager_hosts': ['an-test-master1001.eqiad.wmnet', 'an-test-master1002.eqiad.wmnet'], 'namenode_hosts': ['an-test-master1001.eqiad.wmnet', 'an-test-master1002.eqiad.wmnet'], 'journalnode_hosts': ['an-test-worker1001.eqiad.wmnet', 'an-test-worker1002.eqiad.wmnet', 'an-test-worker1003.eqiad.wmnet'], 'net_topology': {'an-test-worker1001.eqiad.wmnet': '/eqiad/A/5', 'an-test-worker1002.eqiad.wmnet': '/eqiad/C/5', 'an-test-worker1003.eqiad.wmnet': '/eqiad/D/6'}, 'excluded_hosts': [], 'dfs_namenode_handler_count': 20, 'dfs_namenode_service_port': 8040, 'dfs_namenode_service_handler_count': 10, 'hadoop_var_directory': '/srv/hadoop', 'yarn_scheduler_maximum_allocation_mb': 53248, 'yarn_resourcemanager_zk_state_store_parent_path': '/yarn-rmstore/analytics-test-hadoop', 'yarn_resourcemanager_max_completed_applications': 1000, 'yarn_nodemanager_os_reserved_memory_mb': 14000, 'yarn_node_labels_enabled': True, 'hadoop_datanode_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:51010:/etc/prometheus/hdfs_datanode_jmx_exporter.yaml', 'hadoop_journalnode_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10485:/etc/prometheus/hdfs_journalnode_jmx_exporter.yaml', 'yarn_nodemanager_opts': '-Xms6144m -Xmx6144m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:8141:/etc/prometheus/yarn_nodemanager_jmx_exporter.yaml', 'hadoop_namenode_opts': '-Xms12288m -Xmx12288m -XX:+UseG1GC -XX:MaxGCPauseMillis=1000 -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10080:/etc/prometheus/hdfs_namenode_jmx_exporter.yaml', 'yarn_resourcemanager_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10083:/etc/prometheus/yarn_resourcemanager_jmx_exporter.yaml', 'mapreduce_history_java_opts': '-Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10086:/etc/prometheus/mapreduce_history_jmx_exporter.yaml', 'core_site_extra_properties': {'hadoop.http.staticuser.user': 'yarn'}, 'yarn_site_extra_properties': {'yarn.resourcemanager.scheduler.class': 'org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler', 'yarn.resourcemanager.scheduler.monitor.enable': True, 'yarn.admin.acl': 'yarn analytics-admins', 'yarn.acl.enable': True}, 'hdfs_site_extra_properties': {'dfs.client.failover.max.attempts': 3, 'dfs.namenode.acls.enabled': True, 'dfs.replication': 2}, 'yarn_use_multi_spark_shufflers': True, 'yarn_multi_spark_shuffler_versions': {'3.1': 7337, '3.3': 7338, '3.4': 7339}}, 'analytics-hadoop': {'zookeeper_cluster_name': 'analytics-eqiad', 'resourcemanager_hosts': ['an-master1003.eqiad.wmnet', 'an-master1004.eqiad.wmnet'], 'namenode_hosts': ['an-master1003.eqiad.wmnet', 'an-master1004.eqiad.wmnet'], 'journalnode_hosts': ['an-worker1178.eqiad.wmnet', 'an-worker1151.eqiad.wmnet', 'an-worker1142.eqiad.wmnet', 'an-worker1219.eqiad.wmnet', 'an-worker1213.eqiad.wmnet'], 'net_topology': {'an-master1003.eqiad.wmnet': '/eqiad/C/2', 'an-master1004.eqiad.wmnet': '/eqiad/D/7', 'an-worker1178.eqiad.wmnet': '/eqiad/A/2', 'an-worker1210.eqiad.wmnet': '/eqiad/A/2', 'an-worker1211.eqiad.wmnet': '/eqiad/A/2', 'an-worker1233.eqiad.wmnet': '/eqiad/A/2', 'an-worker1212.eqiad.wmnet': '/eqiad/A/4', 'an-worker1213.eqiad.wmnet': '/eqiad/A/4', 'an-worker1149.eqiad.wmnet': '/eqiad/A/7', 'an-worker1214.eqiad.wmnet': '/eqiad/A/7', 'an-worker1215.eqiad.wmnet': '/eqiad/A/7', 'an-worker1216.eqiad.wmnet': '/eqiad/B/2', 'an-worker1217.eqiad.wmnet': '/eqiad/B/2', 'an-worker1150.eqiad.wmnet': '/eqiad/B/7', 'an-worker1179.eqiad.wmnet': '/eqiad/B/7', 'an-worker1218.eqiad.wmnet': '/eqiad/B/7', 'an-worker1219.eqiad.wmnet': '/eqiad/B/7', 'an-worker1220.eqiad.wmnet': '/eqiad/C/2', 'an-worker1221.eqiad.wmnet': '/eqiad/C/2', 'an-worker1234.eqiad.wmnet': '/eqiad/C/2', 'an-worker1222.eqiad.wmnet': '/eqiad/C/4', 'an-worker1223.eqiad.wmnet': '/eqiad/C/4', 'an-worker1224.eqiad.wmnet': '/eqiad/C/7', 'an-worker1225.eqiad.wmnet': '/eqiad/C/7', 'an-worker1226.eqiad.wmnet': '/eqiad/C/7', 'an-worker1151.eqiad.wmnet': '/eqiad/C/7', 'an-worker1175.eqiad.wmnet': '/eqiad/D/2', 'an-worker1209.eqiad.wmnet': '/eqiad/D/2', 'an-worker1227.eqiad.wmnet': '/eqiad/D/2', 'an-worker1228.eqiad.wmnet': '/eqiad/D/2', 'an-worker1229.eqiad.wmnet': '/eqiad/D/4', 'an-worker1230.eqiad.wmnet': '/eqiad/D/4', 'an-worker1152.eqiad.wmnet': '/eqiad/D/7', 'an-worker1231.eqiad.wmnet': '/eqiad/D/7', 'an-worker1232.eqiad.wmnet': '/eqiad/D/7', 'an-worker1142.eqiad.wmnet': '/eqiad/E/1', 'an-worker1147.eqiad.wmnet': '/eqiad/E/1', 'an-worker1153.eqiad.wmnet': '/eqiad/E/1', 'an-worker1181.eqiad.wmnet': '/eqiad/E/1', 'an-worker1187.eqiad.wmnet': '/eqiad/E/1', 'an-worker1235.eqiad.wmnet': '/eqiad/E/1', 'an-worker1143.eqiad.wmnet': '/eqiad/E/2', 'an-worker1182.eqiad.wmnet': '/eqiad/E/2', 'an-worker1188.eqiad.wmnet': '/eqiad/E/2', 'an-worker1189.eqiad.wmnet': '/eqiad/E/2', 'an-worker1154.eqiad.wmnet': '/eqiad/E/3', 'an-worker1176.eqiad.wmnet': '/eqiad/E/3', 'an-worker1190.eqiad.wmnet': '/eqiad/E/3', 'an-worker1191.eqiad.wmnet': '/eqiad/E/3', 'an-worker1157.eqiad.wmnet': '/eqiad/E/5', 'an-worker1158.eqiad.wmnet': '/eqiad/E/5', 'an-worker1159.eqiad.wmnet': '/eqiad/E/5', 'an-worker1180.eqiad.wmnet': '/eqiad/E/5', 'an-worker1183.eqiad.wmnet': '/eqiad/E/5', 'an-worker1192.eqiad.wmnet': '/eqiad/E/5', 'an-worker1193.eqiad.wmnet': '/eqiad/E/5', 'an-worker1160.eqiad.wmnet': '/eqiad/E/6', 'an-worker1161.eqiad.wmnet': '/eqiad/E/6', 'an-worker1162.eqiad.wmnet': '/eqiad/E/6', 'an-worker1184.eqiad.wmnet': '/eqiad/E/6', 'an-worker1194.eqiad.wmnet': '/eqiad/E/6', 'an-worker1195.eqiad.wmnet': '/eqiad/E/6', 'an-worker1163.eqiad.wmnet': '/eqiad/E/7', 'an-worker1164.eqiad.wmnet': '/eqiad/E/7', 'an-worker1165.eqiad.wmnet': '/eqiad/E/7', 'an-worker1196.eqiad.wmnet': '/eqiad/E/7', 'an-worker1197.eqiad.wmnet': '/eqiad/E/7', 'an-worker1144.eqiad.wmnet': '/eqiad/F/1', 'an-worker1155.eqiad.wmnet': '/eqiad/F/1', 'an-worker1198.eqiad.wmnet': '/eqiad/F/1', 'an-worker1236.eqiad.wmnet': '/eqiad/F/1', 'an-worker1145.eqiad.wmnet': '/eqiad/F/2', 'an-worker1199.eqiad.wmnet': '/eqiad/F/2', 'an-worker1200.eqiad.wmnet': '/eqiad/F/2', 'an-worker1146.eqiad.wmnet': '/eqiad/F/3', 'an-worker1156.eqiad.wmnet': '/eqiad/F/3', 'an-worker1177.eqiad.wmnet': '/eqiad/F/3', 'an-worker1201.eqiad.wmnet': '/eqiad/F/3', 'an-worker1202.eqiad.wmnet': '/eqiad/F/3', 'an-worker1166.eqiad.wmnet': '/eqiad/F/5', 'an-worker1167.eqiad.wmnet': '/eqiad/F/5', 'an-worker1168.eqiad.wmnet': '/eqiad/F/5', 'an-worker1203.eqiad.wmnet': '/eqiad/F/5', 'an-worker1204.eqiad.wmnet': '/eqiad/F/5', 'an-worker1170.eqiad.wmnet': '/eqiad/F/6', 'an-worker1171.eqiad.wmnet': '/eqiad/F/6', 'an-worker1186.eqiad.wmnet': '/eqiad/F/6', 'an-worker1205.eqiad.wmnet': '/eqiad/F/6', 'an-worker1206.eqiad.wmnet': '/eqiad/F/6', 'an-worker1172.eqiad.wmnet': '/eqiad/F/7', 'an-worker1173.eqiad.wmnet': '/eqiad/F/7', 'an-worker1174.eqiad.wmnet': '/eqiad/F/7', 'an-worker1207.eqiad.wmnet': '/eqiad/F/7', 'an-worker1208.eqiad.wmnet': '/eqiad/F/7', 'an-worker1169.eqiad.wmnet': '/eqiad/F/8', 'an-worker1185.eqiad.wmnet': '/eqiad/F/8'}, 'excluded_hosts': ['an-worker1148.eqiad.wmnet'], 'core_site_extra_properties': {'hadoop.http.staticuser.user': 'yarn'}, 'yarn_node_labels_enabled': True, 'yarn_site_extra_properties': {'yarn.resourcemanager.scheduler.class': 'org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler', 'yarn.resourcemanager.scheduler.monitor.enable': True, 'yarn.admin.acl': 'yarn analytics-admins', 'yarn.acl.enable': True}, 'hdfs_site_extra_properties': {'dfs.namenode.fs-limits.max-directory-items': 4194304}, 'datanode_volumes_failed_tolerated': 2, 'dfs_namenode_handler_count': 127, 'dfs_namenode_service_port': 8040, 'dfs_namenode_service_handler_count': 100, 'yarn_scheduler_maximum_allocation_mb': 131072, 'yarn_resourcemanager_zk_state_store_parent_path': '/yarn-rmstore/analytics-hadoop', 'yarn_nodemanager_os_reserved_memory_mb': 24000, 'hadoop_datanode_opts': '-Xms8192m -Xmx8192m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:51010:/etc/prometheus/hdfs_datanode_jmx_exporter.yaml', 'hadoop_journalnode_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10485:/etc/prometheus/hdfs_journalnode_jmx_exporter.yaml', 'yarn_nodemanager_opts': '-Xms12288m -Xmx12288m -Djava.net.preferIPv4Stack=false -XX:+UseG1GC -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=1000 -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:8141:/etc/prometheus/yarn_nodemanager_jmx_exporter.yaml', 'hadoop_namenode_opts': '-Xms196608m -Xmx196608m -XX:PermSize=128m -XX:MaxPermSize=256m -XX:NewSize=28672m -XX:MaxNewSize=28672m -Djava.net.preferIPv4Stack=false -XX:+UseG1GC -XX:MaxGCPauseMillis=4000 -XX:ParallelGCThreads=20 -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10080:/etc/prometheus/hdfs_namenode_jmx_exporter.yaml', 'yarn_resourcemanager_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10083:/etc/prometheus/yarn_resourcemanager_jmx_exporter.yaml', 'mapreduce_history_java_opts': '-Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10086:/etc/prometheus/mapreduce_history_jmx_exporter.yaml', 'mapreduce_history_heap_size': 4096, 'yarn_use_multi_spark_shufflers': True, 'yarn_multi_spark_shuffler_versions': {'3.1': 7337, '3.3': 7338, '3.4': 7339}, 'yarn_spark_history_server_address': 'https://spark-history.svc.eqiad.wmnet:30443'}}
+ hadoop_clusters => {'analytics-test-hadoop': {'zookeeper_cluster_name': 'analytics-eqiad', 'resourcemanager_hosts': ['an-test-master1001.eqiad.wmnet', 'an-test-master1002.eqiad.wmnet'], 'namenode_hosts': ['an-test-master1001.eqiad.wmnet', 'an-test-master1002.eqiad.wmnet'], 'journalnode_hosts': ['an-test-worker1001.eqiad.wmnet', 'an-test-worker1002.eqiad.wmnet', 'an-test-worker1003.eqiad.wmnet'], 'net_topology': {'an-test-worker1001.eqiad.wmnet': '/eqiad/A/5', 'an-test-worker1002.eqiad.wmnet': '/eqiad/C/5', 'an-test-worker1003.eqiad.wmnet': '/eqiad/D/6'}, 'excluded_hosts': [], 'dfs_namenode_handler_count': 20, 'dfs_namenode_service_port': 8040, 'dfs_namenode_service_handler_count': 10, 'hadoop_var_directory': '/srv/hadoop', 'yarn_scheduler_maximum_allocation_mb': 53248, 'yarn_resourcemanager_zk_state_store_parent_path': '/yarn-rmstore/analytics-test-hadoop', 'yarn_resourcemanager_max_completed_applications': 1000, 'yarn_nodemanager_os_reserved_memory_mb': 14000, 'yarn_node_labels_enabled': True, 'hadoop_datanode_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:51010:/etc/prometheus/hdfs_datanode_jmx_exporter.yaml', 'hadoop_journalnode_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10485:/etc/prometheus/hdfs_journalnode_jmx_exporter.yaml', 'yarn_nodemanager_opts': '-Xms6144m -Xmx6144m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:8141:/etc/prometheus/yarn_nodemanager_jmx_exporter.yaml', 'hadoop_namenode_opts': '-Xms12288m -Xmx12288m -XX:+UseG1GC -XX:MaxGCPauseMillis=1000 -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10080:/etc/prometheus/hdfs_namenode_jmx_exporter.yaml', 'yarn_resourcemanager_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10083:/etc/prometheus/yarn_resourcemanager_jmx_exporter.yaml', 'mapreduce_history_java_opts': '-Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10086:/etc/prometheus/mapreduce_history_jmx_exporter.yaml', 'core_site_extra_properties': {'hadoop.http.staticuser.user': 'yarn'}, 'yarn_site_extra_properties': {'yarn.resourcemanager.scheduler.class': 'org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler', 'yarn.resourcemanager.scheduler.monitor.enable': True, 'yarn.admin.acl': 'yarn analytics-admins', 'yarn.acl.enable': True}, 'hdfs_site_extra_properties': {'dfs.client.failover.max.attempts': 3, 'dfs.namenode.acls.enabled': True, 'dfs.replication': 2}, 'yarn_use_multi_spark_shufflers': True, 'yarn_multi_spark_shuffler_versions': {'3.1': 7337, '3.3': 7338, '3.4': 7339, '3.5': 7340}}, 'analytics-hadoop': {'zookeeper_cluster_name': 'analytics-eqiad', 'resourcemanager_hosts': ['an-master1003.eqiad.wmnet', 'an-master1004.eqiad.wmnet'], 'namenode_hosts': ['an-master1003.eqiad.wmnet', 'an-master1004.eqiad.wmnet'], 'journalnode_hosts': ['an-worker1178.eqiad.wmnet', 'an-worker1151.eqiad.wmnet', 'an-worker1142.eqiad.wmnet', 'an-worker1219.eqiad.wmnet', 'an-worker1213.eqiad.wmnet'], 'net_topology': {'an-master1003.eqiad.wmnet': '/eqiad/C/2', 'an-master1004.eqiad.wmnet': '/eqiad/D/7', 'an-worker1178.eqiad.wmnet': '/eqiad/A/2', 'an-worker1210.eqiad.wmnet': '/eqiad/A/2', 'an-worker1211.eqiad.wmnet': '/eqiad/A/2', 'an-worker1233.eqiad.wmnet': '/eqiad/A/2', 'an-worker1212.eqiad.wmnet': '/eqiad/A/4', 'an-worker1213.eqiad.wmnet': '/eqiad/A/4', 'an-worker1149.eqiad.wmnet': '/eqiad/A/7', 'an-worker1214.eqiad.wmnet': '/eqiad/A/7', 'an-worker1215.eqiad.wmnet': '/eqiad/A/7', 'an-worker1216.eqiad.wmnet': '/eqiad/B/2', 'an-worker1217.eqiad.wmnet': '/eqiad/B/2', 'an-worker1150.eqiad.wmnet': '/eqiad/B/7', 'an-worker1179.eqiad.wmnet': '/eqiad/B/7', 'an-worker1218.eqiad.wmnet': '/eqiad/B/7', 'an-worker1219.eqiad.wmnet': '/eqiad/B/7', 'an-worker1220.eqiad.wmnet': '/eqiad/C/2', 'an-worker1221.eqiad.wmnet': '/eqiad/C/2', 'an-worker1234.eqiad.wmnet': '/eqiad/C/2', 'an-worker1222.eqiad.wmnet': '/eqiad/C/4', 'an-worker1223.eqiad.wmnet': '/eqiad/C/4', 'an-worker1224.eqiad.wmnet': '/eqiad/C/7', 'an-worker1225.eqiad.wmnet': '/eqiad/C/7', 'an-worker1226.eqiad.wmnet': '/eqiad/C/7', 'an-worker1151.eqiad.wmnet': '/eqiad/C/7', 'an-worker1175.eqiad.wmnet': '/eqiad/D/2', 'an-worker1209.eqiad.wmnet': '/eqiad/D/2', 'an-worker1227.eqiad.wmnet': '/eqiad/D/2', 'an-worker1228.eqiad.wmnet': '/eqiad/D/2', 'an-worker1229.eqiad.wmnet': '/eqiad/D/4', 'an-worker1230.eqiad.wmnet': '/eqiad/D/4', 'an-worker1152.eqiad.wmnet': '/eqiad/D/7', 'an-worker1231.eqiad.wmnet': '/eqiad/D/7', 'an-worker1232.eqiad.wmnet': '/eqiad/D/7', 'an-worker1142.eqiad.wmnet': '/eqiad/E/1', 'an-worker1147.eqiad.wmnet': '/eqiad/E/1', 'an-worker1153.eqiad.wmnet': '/eqiad/E/1', 'an-worker1181.eqiad.wmnet': '/eqiad/E/1', 'an-worker1187.eqiad.wmnet': '/eqiad/E/1', 'an-worker1235.eqiad.wmnet': '/eqiad/E/1', 'an-worker1143.eqiad.wmnet': '/eqiad/E/2', 'an-worker1182.eqiad.wmnet': '/eqiad/E/2', 'an-worker1188.eqiad.wmnet': '/eqiad/E/2', 'an-worker1189.eqiad.wmnet': '/eqiad/E/2', 'an-worker1154.eqiad.wmnet': '/eqiad/E/3', 'an-worker1176.eqiad.wmnet': '/eqiad/E/3', 'an-worker1190.eqiad.wmnet': '/eqiad/E/3', 'an-worker1191.eqiad.wmnet': '/eqiad/E/3', 'an-worker1157.eqiad.wmnet': '/eqiad/E/5', 'an-worker1158.eqiad.wmnet': '/eqiad/E/5', 'an-worker1159.eqiad.wmnet': '/eqiad/E/5', 'an-worker1180.eqiad.wmnet': '/eqiad/E/5', 'an-worker1183.eqiad.wmnet': '/eqiad/E/5', 'an-worker1192.eqiad.wmnet': '/eqiad/E/5', 'an-worker1193.eqiad.wmnet': '/eqiad/E/5', 'an-worker1160.eqiad.wmnet': '/eqiad/E/6', 'an-worker1161.eqiad.wmnet': '/eqiad/E/6', 'an-worker1162.eqiad.wmnet': '/eqiad/E/6', 'an-worker1184.eqiad.wmnet': '/eqiad/E/6', 'an-worker1194.eqiad.wmnet': '/eqiad/E/6', 'an-worker1195.eqiad.wmnet': '/eqiad/E/6', 'an-worker1163.eqiad.wmnet': '/eqiad/E/7', 'an-worker1164.eqiad.wmnet': '/eqiad/E/7', 'an-worker1165.eqiad.wmnet': '/eqiad/E/7', 'an-worker1196.eqiad.wmnet': '/eqiad/E/7', 'an-worker1197.eqiad.wmnet': '/eqiad/E/7', 'an-worker1144.eqiad.wmnet': '/eqiad/F/1', 'an-worker1155.eqiad.wmnet': '/eqiad/F/1', 'an-worker1198.eqiad.wmnet': '/eqiad/F/1', 'an-worker1236.eqiad.wmnet': '/eqiad/F/1', 'an-worker1145.eqiad.wmnet': '/eqiad/F/2', 'an-worker1199.eqiad.wmnet': '/eqiad/F/2', 'an-worker1200.eqiad.wmnet': '/eqiad/F/2', 'an-worker1146.eqiad.wmnet': '/eqiad/F/3', 'an-worker1156.eqiad.wmnet': '/eqiad/F/3', 'an-worker1177.eqiad.wmnet': '/eqiad/F/3', 'an-worker1201.eqiad.wmnet': '/eqiad/F/3', 'an-worker1202.eqiad.wmnet': '/eqiad/F/3', 'an-worker1166.eqiad.wmnet': '/eqiad/F/5', 'an-worker1167.eqiad.wmnet': '/eqiad/F/5', 'an-worker1168.eqiad.wmnet': '/eqiad/F/5', 'an-worker1203.eqiad.wmnet': '/eqiad/F/5', 'an-worker1204.eqiad.wmnet': '/eqiad/F/5', 'an-worker1170.eqiad.wmnet': '/eqiad/F/6', 'an-worker1171.eqiad.wmnet': '/eqiad/F/6', 'an-worker1186.eqiad.wmnet': '/eqiad/F/6', 'an-worker1205.eqiad.wmnet': '/eqiad/F/6', 'an-worker1206.eqiad.wmnet': '/eqiad/F/6', 'an-worker1172.eqiad.wmnet': '/eqiad/F/7', 'an-worker1173.eqiad.wmnet': '/eqiad/F/7', 'an-worker1174.eqiad.wmnet': '/eqiad/F/7', 'an-worker1207.eqiad.wmnet': '/eqiad/F/7', 'an-worker1208.eqiad.wmnet': '/eqiad/F/7', 'an-worker1169.eqiad.wmnet': '/eqiad/F/8', 'an-worker1185.eqiad.wmnet': '/eqiad/F/8'}, 'excluded_hosts': ['an-worker1148.eqiad.wmnet'], 'core_site_extra_properties': {'hadoop.http.staticuser.user': 'yarn'}, 'yarn_node_labels_enabled': True, 'yarn_site_extra_properties': {'yarn.resourcemanager.scheduler.class': 'org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler', 'yarn.resourcemanager.scheduler.monitor.enable': True, 'yarn.admin.acl': 'yarn analytics-admins', 'yarn.acl.enable': True}, 'hdfs_site_extra_properties': {'dfs.namenode.fs-limits.max-directory-items': 4194304}, 'datanode_volumes_failed_tolerated': 2, 'dfs_namenode_handler_count': 127, 'dfs_namenode_service_port': 8040, 'dfs_namenode_service_handler_count': 100, 'yarn_scheduler_maximum_allocation_mb': 131072, 'yarn_resourcemanager_zk_state_store_parent_path': '/yarn-rmstore/analytics-hadoop', 'yarn_nodemanager_os_reserved_memory_mb': 24000, 'hadoop_datanode_opts': '-Xms8192m -Xmx8192m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:51010:/etc/prometheus/hdfs_datanode_jmx_exporter.yaml', 'hadoop_journalnode_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10485:/etc/prometheus/hdfs_journalnode_jmx_exporter.yaml', 'yarn_nodemanager_opts': '-Xms12288m -Xmx12288m -Djava.net.preferIPv4Stack=false -XX:+UseG1GC -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=1000 -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:8141:/etc/prometheus/yarn_nodemanager_jmx_exporter.yaml', 'hadoop_namenode_opts': '-Xms196608m -Xmx196608m -XX:PermSize=128m -XX:MaxPermSize=256m -XX:NewSize=28672m -XX:MaxNewSize=28672m -Djava.net.preferIPv4Stack=false -XX:+UseG1GC -XX:MaxGCPauseMillis=4000 -XX:ParallelGCThreads=20 -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10080:/etc/prometheus/hdfs_namenode_jmx_exporter.yaml', 'yarn_resourcemanager_opts': '-Xms4096m -Xmx4096m -Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10083:/etc/prometheus/yarn_resourcemanager_jmx_exporter.yaml', 'mapreduce_history_java_opts': '-Djava.net.preferIPv4Stack=false -javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=[::]:10086:/etc/prometheus/mapreduce_history_jmx_exporter.yaml', 'mapreduce_history_heap_size': 4096, 'yarn_use_multi_spark_shufflers': True, 'yarn_multi_spark_shuffler_versions': {'3.1': 7337, '3.3': 7338, '3.4': 7339}, 'yarn_spark_history_server_address': 'https://spark-history.svc.eqiad.wmnet:30443'}}
Class[Bigtop::Hadoop]
- Parameters differences:
--- Class[Bigtop::Hadoop].orig
+++ Class[Bigtop::Hadoop]
@@
- yarn_multi_spark_shuffler_versions => {'3.1': 7337, '3.3': 7338, '3.4': 7339}
+ yarn_multi_spark_shuffler_versions => {'3.1': 7337, '3.3': 7338, '3.4': 7339, '3.5': 7340}
- File[/etc/hadoop/conf.analytics-test-hadoop/yarn-site.xml]
- Content differences:
--- /etc/hadoop/conf.analytics-test-hadoop/yarn-site.xml.orig
+++ /etc/hadoop/conf.analytics-test-hadoop/yarn-site.xml
@@ -139,7 +139,7 @@
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle,spark_shuffle,spark_shuffle_3_3,spark_shuffle_3_4</value>
+ <value>mapreduce_shuffle,spark_shuffle,spark_shuffle_3_3,spark_shuffle_3_4,spark_shuffle_3_5</value>
</property>
<property>
@@ -173,6 +173,15 @@
<property>
<name>yarn.nodemanager.aux-services.spark_shuffle_3_4.classpath</name>
<value>/usr/lib/hadoop-yarn/lib/spark-3.4-yarn-shuffle.jar:/etc/hadoop/conf/spark_shuffle_3_4_config</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services.spark_shuffle_3_5.class</name>
+ <value>org.apache.spark.network.yarn.YarnShuffleService</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.spark_shuffle_3_5.classpath</name>
+ <value>/usr/lib/hadoop-yarn/lib/spark-3.5-yarn-shuffle.jar:/etc/hadoop/conf/spark_shuffle_3_5_config</value>
</property>
<property>
Relevant files