Compilation results for toolsbeta-prometheus-2.toolsbeta.eqiad1.wikimedia.cloud: System changes detected

Catalog differences

Summary

Resources modified

• Prometheus::Server[tools]

  Parameters differences:

  --- Prometheus::Server[tools].orig
  +++ Prometheus::Server[tools]
  
  @@
  -    scrape_configs_extra => [{'job_name': 'node-exporter', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9100}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9100'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'ssh-banner', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 22}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:22'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}, {'source_labels': ['__address__'], 'regex': '(.*)', 'target_label': '__param_target', 'replacement': '${1}'}, {'source_labels': ['__param_target'], 'regex': '(.*)', 'target_label': 'instance', 'replacement': '${1}'}, {'source_labels': [], 'regex': '.*', 'target_label': '__address__', 'replacement': '127.0.0.1:9115'}], 'metrics_path': '/probe', 'params': {'module': ['ssh_banner']}}, {'job_name': 'haproxy', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 8404}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-test-k8s-haproxy-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:8404'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'exim', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 3903}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-mail-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:3903'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'k8s-etcd', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9051}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-test-k8s-etcd-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9051'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'k8s-apiserver', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 6443}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-test-k8s-control-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:6443'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}], 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}}, {'job_name': 'harbor', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9090}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-harbor-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9090'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'elastic', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9108}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-elastic-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9108'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'elastic-haproxy', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9900}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-elastic-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9900'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'redis', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9121}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-redis-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9121'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'tools-db', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9104}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-db-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9104'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'k8s-nodes', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'node', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}}], 'relabel_configs': [{'action': 'labelmap', 'regex': '__meta_kubernetes_node_label_(.+)'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_node_name'], 'regex': '(.+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/nodes/${1}/proxy/metrics'}]}, {'job_name': 'probes/custom', 'metrics_path': '/probe', 'scrape_interval': '30s', 'scrape_timeout': '30s', 'file_sd_configs': [{'files': ['/srv/prometheus/tools/targets/probes-custom_*.yaml']}], 'relabel_configs': [{'source_labels': ['__address__'], 'regex': '([^@]+)', 'target_label': 'instance'}, {'source_labels': ['__address__'], 'regex': '([^@]+)', 'target_label': '__param_target'}, {'source_labels': ['__address__'], 'regex': '(.+)@(.+)', 'target_label': 'instance', 'replacement': '${1}'}, {'source_labels': ['__address__'], 'regex': '(.+)@(.+)', 'target_label': '__param_target', 'replacement': '${2}'}, {'source_labels': ['module'], 'target_label': '__param_module'}, {'target_label': '__address__', 'replacement': '127.0.0.1:9115'}]}, {'job_name': 'probes/pingthing', 'metrics_path': '/probe', 'params': {'module': ['http_connect_23xx']}, 'file_sd_configs': [{'files': ['/srv/prometheus/tools/targets/probes_pingthing_http_check_urls.yaml']}], 'relabel_configs': [{'source_labels': ['__address__'], 'regex': '(.*)', 'target_label': 'url'}, {'source_labels': ['__address__'], 'regex': '([^@]+)', 'target_label': '__param_target'}, {'source_labels': ['__address__'], 'regex': '(.+)@(.+)', 'target_label': '__param_target', 'replacement': '${2}'}, {'source_labels': ['module'], 'target_label': '__param_module'}, {'target_label': '__address__', 'replacement': '127.0.0.1:9115'}]}, {'job_name': 'pint', 'scheme': 'http', 'static_configs': [{'targets': ['localhost:9123']}]}, {'job_name': 'k8s-cert-manager', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['cert-manager']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'cert-manager-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(cert-manager-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/cert-manager/pods/${1}:9402/proxy/metrics'}]}, {'job_name': 'k8s-cert-manager-reloader', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['cert-manager']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'reloader-reloader-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(reloader-reloader-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/cert-manager/pods/${1}:9090/proxy/metrics'}]}, {'job_name': 'k8s-coredns', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['kube-system']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'coredns-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(coredns-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/kube-system/pods/${1}:9153/proxy/metrics'}]}, {'job_name': 'k8s-ingress-nginx', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['ingress-nginx-gen2']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'ingress-nginx-gen2-controller-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(ingress-nginx-gen2-controller-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/ingress-nginx-gen2/pods/${1}:10254/proxy/metrics'}], 'scrape_interval': '4m', 'scrape_timeout': '60s', 'metric_relabel_configs': [{'action': 'keep', 'source_labels': ['__name__'], 'regex': '.*(requests|process_connections).*'}]}, {'job_name': 'istiod', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['istio-system']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'istiod-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(istiod-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/istio-system/pods/${1}:15014/proxy/metrics'}]}, {'job_name': 'k8s-cadvisor', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['metrics']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'cadvisor-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(cadvisor-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/metrics/pods/${1}:8080/proxy/metrics'}], 'scrape_interval': '4m', 'scrape_timeout': '60s'}, {'job_name': 'k8s-maintain-kubeusers', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['maintain-kubeusers']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'maintain-kubeusers-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(maintain-kubeusers-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/maintain-kubeusers/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'k8s-kube-state-metrics', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['metrics']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'kube-state-metrics(-[a-zA-Z0-9]+)+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(kube-state-metrics(-[a-zA-Z0-9]+)+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/metrics/pods/${1}:8080/proxy/metrics'}]}, {'job_name': 'jobs-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['jobs-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'jobs-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(jobs-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/jobs-api/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'tekton-pipelines-controller', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['tekton-pipelines']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'tekton-pipelines-controller-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(tekton-pipelines-controller-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/tekton-pipelines/pods/${1}:9090/proxy/metrics'}]}, {'job_name': 'builds-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['builds-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'builds-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(builds-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/builds-api/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'envvars-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['envvars-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'envvars-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(envvars-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/envvars-api/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'kyverno', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['kyverno']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'kyverno-.*controller.*', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(kyverno-.*controller.*)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/kyverno/pods/${1}:8000/proxy/metrics'}], 'metric_relabel_configs': [{'action': 'drop', 'source_labels': ['__name__'], 'regex': 'kyverno_policy_execution_duration_seconds_bucket'}]}, {'job_name': 'jobs-emailer', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['jobs-emailer']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'jobs-emailer-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(jobs-emailer-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/jobs-emailer/pods/${1}:8081/proxy/metrics'}]}, {'job_name': 'components-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['components-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'components-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(components-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/components-api/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'loki-tools', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['loki']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'loki-tools(-[a-zA-Z0-9]+)+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(loki-tools(-[a-zA-Z0-9]+)+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/loki/pods/${1}:3100/proxy/metrics'}]}, {'job_name': 'infra-tracing-loki', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['infra-tracing-loki']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'infra-tracing-loki(-[a-zA-Z0-9]+)+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(infra-tracing-loki(-[a-zA-Z0-9]+)+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/infra-tracing-loki/pods/${1}:3100/proxy/metrics'}]}, {'job_name': 'alloy', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['alloy']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'alloy-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(alloy-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/alloy/pods/${1}:12345/proxy/metrics'}]}, {'job_name': 'logs-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['logs-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'logs-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(logs-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/logs-api/pods/${1}:9000/proxy/metrics'}]}]
  +    scrape_configs_extra => [{'job_name': 'node-exporter', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9100}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9100'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'ssh-banner', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 22}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:22'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}, {'source_labels': ['__address__'], 'regex': '(.*)', 'target_label': '__param_target', 'replacement': '${1}'}, {'source_labels': ['__param_target'], 'regex': '(.*)', 'target_label': 'instance', 'replacement': '${1}'}, {'source_labels': [], 'regex': '.*', 'target_label': '__address__', 'replacement': '127.0.0.1:9115'}], 'metrics_path': '/probe', 'params': {'module': ['ssh_banner']}}, {'job_name': 'haproxy', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 8404}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-test-k8s-haproxy-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:8404'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'exim', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 3903}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-mail-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:3903'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'k8s-etcd', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9051}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-test-k8s-etcd-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9051'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'k8s-apiserver', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 6443}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-test-k8s-control-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:6443'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}], 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}}, {'job_name': 'harbor', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9090}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-harbor-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9090'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'elastic', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9108}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-elastic-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9108'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'elastic-haproxy', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9900}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-elastic-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9900'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'redis', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9121}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-redis-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9121'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'tools-db', 'openstack_sd_configs': [{'role': 'instance', 'region': 'eqiad1-r', 'identity_endpoint': 'https://openstack.eqiad1.wikimediacloud.org:25000/v3', 'username': 'novaobserver', 'password': 'Fs6Dq2RtG8KwmM2Z', 'domain_name': 'default', 'project_name': 'toolsbeta', 'all_tenants': False, 'refresh_interval': '5m', 'port': 9104}], 'relabel_configs': [{'source_labels': ['__meta_openstack_instance_name'], 'action': 'keep', 'regex': 'toolsbeta-db-\\d+'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': 'instance'}, {'source_labels': ['__meta_openstack_instance_name'], 'target_label': '__address__', 'regex': '(.*)', 'replacement': '$1:9104'}, {'source_labels': ['__meta_openstack_instance_status'], 'action': 'keep', 'regex': 'ACTIVE'}]}, {'job_name': 'k8s-nodes', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'node', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}}], 'relabel_configs': [{'action': 'labelmap', 'regex': '__meta_kubernetes_node_label_(.+)'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_node_name'], 'regex': '(.+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/nodes/${1}/proxy/metrics'}]}, {'job_name': 'probes/custom', 'metrics_path': '/probe', 'scrape_interval': '30s', 'scrape_timeout': '30s', 'file_sd_configs': [{'files': ['/srv/prometheus/tools/targets/probes-custom_*.yaml']}], 'relabel_configs': [{'source_labels': ['__address__'], 'regex': '([^@]+)', 'target_label': 'instance'}, {'source_labels': ['__address__'], 'regex': '([^@]+)', 'target_label': '__param_target'}, {'source_labels': ['__address__'], 'regex': '(.+)@(.+)', 'target_label': 'instance', 'replacement': '${1}'}, {'source_labels': ['__address__'], 'regex': '(.+)@(.+)', 'target_label': '__param_target', 'replacement': '${2}'}, {'source_labels': ['module'], 'target_label': '__param_module'}, {'target_label': '__address__', 'replacement': '127.0.0.1:9115'}]}, {'job_name': 'probes/pingthing', 'metrics_path': '/probe', 'params': {'module': ['http_connect_23xx']}, 'file_sd_configs': [{'files': ['/srv/prometheus/tools/targets/probes_pingthing_http_check_urls.yaml']}], 'relabel_configs': [{'source_labels': ['__address__'], 'regex': '(.*)', 'target_label': 'url'}, {'source_labels': ['__address__'], 'regex': '([^@]+)', 'target_label': '__param_target'}, {'source_labels': ['__address__'], 'regex': '(.+)@(.+)', 'target_label': '__param_target', 'replacement': '${2}'}, {'source_labels': ['module'], 'target_label': '__param_module'}, {'target_label': '__address__', 'replacement': '127.0.0.1:9115'}]}, {'job_name': 'pint', 'scheme': 'http', 'static_configs': [{'targets': ['localhost:9123']}]}, {'job_name': 'k8s-cert-manager', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['cert-manager']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'cert-manager-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(cert-manager-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/cert-manager/pods/${1}:9402/proxy/metrics'}]}, {'job_name': 'k8s-cert-manager-reloader', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['cert-manager']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'reloader-reloader-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(reloader-reloader-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/cert-manager/pods/${1}:9090/proxy/metrics'}]}, {'job_name': 'k8s-coredns', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['kube-system']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'coredns-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(coredns-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/kube-system/pods/${1}:9153/proxy/metrics'}]}, {'job_name': 'k8s-ingress-nginx', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['ingress-nginx-gen2']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'ingress-nginx-gen2-controller-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(ingress-nginx-gen2-controller-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/ingress-nginx-gen2/pods/${1}:10254/proxy/metrics'}], 'scrape_interval': '4m', 'scrape_timeout': '60s', 'metric_relabel_configs': [{'action': 'keep', 'source_labels': ['__name__'], 'regex': '.*(requests|process_connections).*'}]}, {'job_name': 'istiod', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['istio-system']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'istiod-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(istiod-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/istio-system/pods/${1}:15014/proxy/metrics'}]}, {'job_name': 'k8s-cadvisor', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['metrics']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'cadvisor-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(cadvisor-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/metrics/pods/${1}:8080/proxy/metrics'}], 'scrape_interval': '4m', 'scrape_timeout': '60s'}, {'job_name': 'k8s-maintain-kubeusers', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['maintain-kubeusers']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'maintain-kubeusers-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(maintain-kubeusers-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/maintain-kubeusers/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'k8s-kube-state-metrics', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['metrics']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'kube-state-metrics(-[a-zA-Z0-9]+)+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(kube-state-metrics(-[a-zA-Z0-9]+)+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/metrics/pods/${1}:8080/proxy/metrics'}]}, {'job_name': 'jobs-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['jobs-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'jobs-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(jobs-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/jobs-api/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'tekton-pipelines-controller', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['tekton-pipelines']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'tekton-pipelines-controller-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(tekton-pipelines-controller-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/tekton-pipelines/pods/${1}:9090/proxy/metrics'}]}, {'job_name': 'builds-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['builds-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'builds-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(builds-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/builds-api/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'envvars-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['envvars-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'envvars-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(envvars-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/envvars-api/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'kyverno', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['kyverno']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'kyverno-.*controller.*', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(kyverno-.*controller.*)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/kyverno/pods/${1}:8000/proxy/metrics'}], 'metric_relabel_configs': [{'action': 'drop', 'source_labels': ['__name__'], 'regex': 'kyverno_policy_execution_duration_seconds_bucket'}]}, {'job_name': 'jobs-emailer', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['jobs-emailer']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'jobs-emailer-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(jobs-emailer-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/jobs-emailer/pods/${1}:8081/proxy/metrics'}]}, {'job_name': 'components-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['components-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'components-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(components-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/components-api/pods/${1}:9000/proxy/metrics'}]}, {'job_name': 'loki-tools', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['loki']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'loki-tools(-[a-zA-Z0-9]+)+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(loki-tools(-[a-zA-Z0-9]+)+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/loki/pods/${1}:3100/proxy/metrics'}]}, {'job_name': 'infra-tracing-loki', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['infra-tracing-loki']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'infra-tracing-loki(-[a-zA-Z0-9]+)+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(infra-tracing-loki(-[a-zA-Z0-9]+)+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/infra-tracing-loki/pods/${1}:3100/proxy/metrics'}]}, {'job_name': 'alloy', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['alloy']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'alloy-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(alloy-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/alloy/pods/${1}:12345/proxy/metrics'}]}, {'job_name': 'logs-api', 'scheme': 'https', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'kubernetes_sd_configs': [{'api_server': 'https://k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443', 'role': 'pod', 'tls_config': {'insecure_skip_verify': True, 'cert_file': '/etc/ssl/localcerts/toolsbeta-k8s-prometheus.crt', 'key_file': '/etc/ssl/private/toolsbeta-k8s-prometheus.key'}, 'namespaces': {'names': ['logs-api']}}], 'relabel_configs': [{'action': 'keep', 'regex': 'logs-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+', 'source_labels': ['__meta_kubernetes_pod_name']}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_name', 'replacement': 'pod_name'}, {'action': 'labelmap', 'regex': '__meta_kubernetes_pod_label_(.+)', 'replacement': 'pod_label_$1'}, {'target_label': '__address__', 'replacement': 'k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443'}, {'source_labels': ['__meta_kubernetes_pod_name'], 'regex': '(logs-api-[a-zA-Z0-9]+-[a-zA-Z0-9]+)', 'target_label': '__metrics_path__', 'replacement': '/api/v1/namespaces/logs-api/pods/${1}:9000/proxy/metrics'}]}]
  

• File[/srv/prometheus/tools/prometheus.yml]

  Content differences:

  --- /srv/prometheus/tools/prometheus.yml.orig
  +++ /srv/prometheus/tools/prometheus.yml
  @@ -641,12 +641,6 @@
       regex: cadvisor-[a-zA-Z0-9]+
       source_labels:
       - __meta_kubernetes_pod_name
  -  - action: labelmap
  -    regex: __meta_kubernetes_pod_name
  -    replacement: pod_name
  -  - action: labelmap
  -    regex: __meta_kubernetes_pod_label_(.+)
  -    replacement: pod_label_$1
     - target_label: __address__
       replacement: k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443
     - source_labels:
  @@ -711,12 +705,6 @@
       regex: kube-state-metrics(-[a-zA-Z0-9]+)+
       source_labels:
       - __meta_kubernetes_pod_name
  -  - action: labelmap
  -    regex: __meta_kubernetes_pod_name
  -    replacement: pod_name
  -  - action: labelmap
  -    regex: __meta_kubernetes_pod_label_(.+)
  -    replacement: pod_label_$1
     - target_label: __address__
       replacement: k8s.svc.toolsbeta.eqiad1.wikimedia.cloud:6443
     - source_labels:

Relevant files

• Production catalog
• Change catalog
• Production errors/warnings
• Change errors/warnings
• Core Diff
• Full Diff