{"host": "registry2005.codfw.wmnet", "state": "core_diff", "description": "Differences to core resources", "diff": {"full": {"total": 2933, "only_in_self": ["File[/etc/nginx/registry-nginx-cache.conf]", "File[/var/cache/nginx-docker-registry]"], "only_in_other": [], "resource_diffs": [{"resource": "File[/var/cache/nginx-docker-registry]", "parameters": "--- File[/var/cache/nginx-docker-registry].orig\n+++ File[/var/cache/nginx-docker-registry]\n\n-    group  => www-data\n-    owner  => www-data\n-    ensure => absent\n-    mode   => 0775\n"}, {"resource": "Class[Profile::Docker_registry]", "parameters": "--- Class[Profile::Docker_registry].orig\n+++ Class[Profile::Docker_registry]\n\n-    nginx_blob_cache => False\n"}, {"resource": "File[/etc/nginx/registry-nginx-cache.conf]", "parameters": "--- File[/etc/nginx/registry-nginx-cache.conf].orig\n+++ File[/etc/nginx/registry-nginx-cache.conf]\n\n-    group   => root\n-    source  => puppet:///modules/docker_registry/registry-nginx-cache.conf\n-    owner   => root\n-    ensure  => absent\n-    require => Package[nginx]\n-    mode    => 0444\n"}, {"resource": "Nginx::Site[registry]"}, {"resource": "Class[Docker_registry::Web]", "parameters": "--- Class[Docker_registry::Web].orig\n+++ Class[Docker_registry::Web]\n\n-    nginx_blob_cache => False\n"}, {"resource": "File[/etc/nginx/sites-available/registry]", "content": "--- /etc/nginx/sites-available/registry.orig\n+++ /etc/nginx/sites-available/registry\n@@ -14,6 +14,7 @@\n     default upgrade;\n     '' close;\n }\n+\n \n \n # Define a separate cache for the auth backends to mitigate subrequests for\n@@ -47,6 +48,17 @@\n     HEAD \"none\";\n     OPTIONS \"none\";\n     default $auth_type_maybe;\n+}\n+\n+# Decides whether a request that 404'd on registry-{ml,..} should be retried against\n+# Swift (see @fallback). Layer existence checks (HEAD .../blobs/...) issued\n+# during a push must NOT fall back: a Swift 200 would make the docker client\n+# skip uploading the layer to registry-{ml,..}, and the subsequent manifest PUT\n+# would then fail with MANIFEST_BLOB_UNKNOWN.\n+# Those must see registry-{ml,..}'s real 404 so the layer gets uploaded to S3.\n+map \"$request_method:$uri\" $skip_fallback {\n+    default                0;\n+    \"~^HEAD:/v2/.*/blobs/\" 1;\n }\n \n # To get auth_basic working in the context of an auth_request, the latter\n@@ -799,6 +811,79 @@\n       include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n     }\n \n+    # Special handling location: Machine Learning service images\n+    # These images are different from the base ones since they are built and pushed\n+    # via Blubber/Gitlab, without requiring special permissions.\n+    #\n+    # MIGRATION IN PROGRESS:\n+    # - Write requests are directed towards the ML S3 bucket (registry-ml).\n+    # - Read requests are attempted against registry-ml first, and if a HTTP 404\n+    #   is returned, registry-swift is used as a fallback.\n+    # TODO: remove the fallback (and route writes straight here) once all read\n+    # and write traffic goes towards the ML S3 bucket.\n+    location ~ ^/v2/(wikimedia/machine-learning-.*) {\n+      # Capture the original request path here so we can pass it to\n+      # jwt-authorizer in /auth/jwt (the use of rewrite in /auth means we'd\n+      # otherwise lose track of it).\n+      set $auth_request_path $1;\n+\n+      include /etc/nginx/registry-nginx-common-cache-settings.conf;\n+\n+      # Send all but GET/HEAD requests to @ml_service_write below. This keeps the\n+      # 404 -> Swift fallback (set up at the bottom of this block) on the read\n+      # path only: we never want a write to be replayed against Swift.\n+      # See <https://www.nginx.com/resources/wiki/start/topics/depth/ifisevil/>\n+      # which explains and recommends this.\n+      error_page 418 = @ml_service_write;\n+      recursive_error_pages on;\n+      if ($request_method !~ ^(GET|HEAD)$) {\n+        return 418;\n+      }\n+\n+      # This covers GET/HEAD requests to /v2/wikimedia/machine-learning-*\n+      auth_request /auth;\n+      auth_request_set $auth_status $upstream_status;\n+\n+      proxy_pass http://registry-ml;\n+      include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n+\n+      # If registry-ml returns a 404 for a read, fall back to Swift.\n+      proxy_intercept_errors on;\n+      error_page 404 = @fallback;\n+    }\n+\n+    # This block applies to POST/PUT/DELETE/etc. methods to\n+    # /v2/wikimedia/machine-learning-*. Writes go straight to registry-ml with\n+    # no Swift fallback.\n+    location @ml_service_write {\n+      auth_request /auth;\n+      auth_request_set $auth_status $upstream_status;\n+\n+      proxy_pass http://registry-ml;\n+      include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n+    }\n+\n+    # Special location block: fallback reads to Swift.\n+    # This block is meant to be referenced by other blocks with\n+    # the 'error_page 404' directive. If a read fails on S3, nginx\n+    # falls back to Swift before returning to the client.\n+    location @fallback {\n+      internal;\n+\n+      # Push-time layer existence checks (HEAD .../blobs/...) must see the real\n+      # 404 from registry-ml rather than a Swift hit, otherwise the client skips\n+      # uploading the layer and the manifest PUT to registry-{ml,..} fails. See the\n+      # $skip_fallback map above.\n+      if ($skip_fallback) {\n+        return 404;\n+      }\n+\n+      access_log /var/log/nginx/access_swift_fallback.log upstream_time;\n+\n+      proxy_pass http://registry-swift;\n+      include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n+    }\n+\n     # Catch-all location for all the Docker images.\n     # Authorization can happen via Basic Auth or JWT token,\n     # via the auth_request directives. The Docker images handled\n@@ -843,7 +928,6 @@\n         allow 127.0.0.1/32;\n         deny all;\n       }\n-\n     }\n \n     # Below are a number of internal locations used by auth_request to route"}], "perc_changed": "0.27%"}, "core": {"total": 2933, "only_in_self": ["File[/etc/nginx/registry-nginx-cache.conf]", "File[/var/cache/nginx-docker-registry]"], "only_in_other": [], "resource_diffs": [{"resource": "File[/etc/nginx/sites-available/registry]", "content": "--- /etc/nginx/sites-available/registry.orig\n+++ /etc/nginx/sites-available/registry\n@@ -14,6 +14,7 @@\n     default upgrade;\n     '' close;\n }\n+\n \n \n # Define a separate cache for the auth backends to mitigate subrequests for\n@@ -47,6 +48,17 @@\n     HEAD \"none\";\n     OPTIONS \"none\";\n     default $auth_type_maybe;\n+}\n+\n+# Decides whether a request that 404'd on registry-{ml,..} should be retried against\n+# Swift (see @fallback). Layer existence checks (HEAD .../blobs/...) issued\n+# during a push must NOT fall back: a Swift 200 would make the docker client\n+# skip uploading the layer to registry-{ml,..}, and the subsequent manifest PUT\n+# would then fail with MANIFEST_BLOB_UNKNOWN.\n+# Those must see registry-{ml,..}'s real 404 so the layer gets uploaded to S3.\n+map \"$request_method:$uri\" $skip_fallback {\n+    default                0;\n+    \"~^HEAD:/v2/.*/blobs/\" 1;\n }\n \n # To get auth_basic working in the context of an auth_request, the latter\n@@ -799,6 +811,79 @@\n       include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n     }\n \n+    # Special handling location: Machine Learning service images\n+    # These images are different from the base ones since they are built and pushed\n+    # via Blubber/Gitlab, without requiring special permissions.\n+    #\n+    # MIGRATION IN PROGRESS:\n+    # - Write requests are directed towards the ML S3 bucket (registry-ml).\n+    # - Read requests are attempted against registry-ml first, and if a HTTP 404\n+    #   is returned, registry-swift is used as a fallback.\n+    # TODO: remove the fallback (and route writes straight here) once all read\n+    # and write traffic goes towards the ML S3 bucket.\n+    location ~ ^/v2/(wikimedia/machine-learning-.*) {\n+      # Capture the original request path here so we can pass it to\n+      # jwt-authorizer in /auth/jwt (the use of rewrite in /auth means we'd\n+      # otherwise lose track of it).\n+      set $auth_request_path $1;\n+\n+      include /etc/nginx/registry-nginx-common-cache-settings.conf;\n+\n+      # Send all but GET/HEAD requests to @ml_service_write below. This keeps the\n+      # 404 -> Swift fallback (set up at the bottom of this block) on the read\n+      # path only: we never want a write to be replayed against Swift.\n+      # See <https://www.nginx.com/resources/wiki/start/topics/depth/ifisevil/>\n+      # which explains and recommends this.\n+      error_page 418 = @ml_service_write;\n+      recursive_error_pages on;\n+      if ($request_method !~ ^(GET|HEAD)$) {\n+        return 418;\n+      }\n+\n+      # This covers GET/HEAD requests to /v2/wikimedia/machine-learning-*\n+      auth_request /auth;\n+      auth_request_set $auth_status $upstream_status;\n+\n+      proxy_pass http://registry-ml;\n+      include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n+\n+      # If registry-ml returns a 404 for a read, fall back to Swift.\n+      proxy_intercept_errors on;\n+      error_page 404 = @fallback;\n+    }\n+\n+    # This block applies to POST/PUT/DELETE/etc. methods to\n+    # /v2/wikimedia/machine-learning-*. Writes go straight to registry-ml with\n+    # no Swift fallback.\n+    location @ml_service_write {\n+      auth_request /auth;\n+      auth_request_set $auth_status $upstream_status;\n+\n+      proxy_pass http://registry-ml;\n+      include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n+    }\n+\n+    # Special location block: fallback reads to Swift.\n+    # This block is meant to be referenced by other blocks with\n+    # the 'error_page 404' directive. If a read fails on S3, nginx\n+    # falls back to Swift before returning to the client.\n+    location @fallback {\n+      internal;\n+\n+      # Push-time layer existence checks (HEAD .../blobs/...) must see the real\n+      # 404 from registry-ml rather than a Swift hit, otherwise the client skips\n+      # uploading the layer and the manifest PUT to registry-{ml,..} fails. See the\n+      # $skip_fallback map above.\n+      if ($skip_fallback) {\n+        return 404;\n+      }\n+\n+      access_log /var/log/nginx/access_swift_fallback.log upstream_time;\n+\n+      proxy_pass http://registry-swift;\n+      include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n+    }\n+\n     # Catch-all location for all the Docker images.\n     # Authorization can happen via Basic Auth or JWT token,\n     # via the auth_request directives. The Docker images handled\n@@ -843,7 +928,6 @@\n         allow 127.0.0.1/32;\n         deny all;\n       }\n-\n     }\n \n     # Below are a number of internal locations used by auth_request to route"}], "perc_changed": "0.10%"}, "main": {"total": 2933, "only_in_self": ["File[/etc/nginx/registry-nginx-cache.conf]", "File[/var/cache/nginx-docker-registry]"], "only_in_other": [], "resource_diffs": [{"resource": "Class[Docker_registry::Web]", "parameters": "--- Class[Docker_registry::Web].orig\n+++ Class[Docker_registry::Web]\n\n-    nginx_blob_cache => False\n"}, {"resource": "File[/etc/nginx/sites-available/registry]", "content": "--- /etc/nginx/sites-available/registry.orig\n+++ /etc/nginx/sites-available/registry\n@@ -14,6 +14,7 @@\n     default upgrade;\n     '' close;\n }\n+\n \n \n # Define a separate cache for the auth backends to mitigate subrequests for\n@@ -47,6 +48,17 @@\n     HEAD \"none\";\n     OPTIONS \"none\";\n     default $auth_type_maybe;\n+}\n+\n+# Decides whether a request that 404'd on registry-{ml,..} should be retried against\n+# Swift (see @fallback). Layer existence checks (HEAD .../blobs/...) issued\n+# during a push must NOT fall back: a Swift 200 would make the docker client\n+# skip uploading the layer to registry-{ml,..}, and the subsequent manifest PUT\n+# would then fail with MANIFEST_BLOB_UNKNOWN.\n+# Those must see registry-{ml,..}'s real 404 so the layer gets uploaded to S3.\n+map \"$request_method:$uri\" $skip_fallback {\n+    default                0;\n+    \"~^HEAD:/v2/.*/blobs/\" 1;\n }\n \n # To get auth_basic working in the context of an auth_request, the latter\n@@ -799,6 +811,79 @@\n       include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n     }\n \n+    # Special handling location: Machine Learning service images\n+    # These images are different from the base ones since they are built and pushed\n+    # via Blubber/Gitlab, without requiring special permissions.\n+    #\n+    # MIGRATION IN PROGRESS:\n+    # - Write requests are directed towards the ML S3 bucket (registry-ml).\n+    # - Read requests are attempted against registry-ml first, and if a HTTP 404\n+    #   is returned, registry-swift is used as a fallback.\n+    # TODO: remove the fallback (and route writes straight here) once all read\n+    # and write traffic goes towards the ML S3 bucket.\n+    location ~ ^/v2/(wikimedia/machine-learning-.*) {\n+      # Capture the original request path here so we can pass it to\n+      # jwt-authorizer in /auth/jwt (the use of rewrite in /auth means we'd\n+      # otherwise lose track of it).\n+      set $auth_request_path $1;\n+\n+      include /etc/nginx/registry-nginx-common-cache-settings.conf;\n+\n+      # Send all but GET/HEAD requests to @ml_service_write below. This keeps the\n+      # 404 -> Swift fallback (set up at the bottom of this block) on the read\n+      # path only: we never want a write to be replayed against Swift.\n+      # See <https://www.nginx.com/resources/wiki/start/topics/depth/ifisevil/>\n+      # which explains and recommends this.\n+      error_page 418 = @ml_service_write;\n+      recursive_error_pages on;\n+      if ($request_method !~ ^(GET|HEAD)$) {\n+        return 418;\n+      }\n+\n+      # This covers GET/HEAD requests to /v2/wikimedia/machine-learning-*\n+      auth_request /auth;\n+      auth_request_set $auth_status $upstream_status;\n+\n+      proxy_pass http://registry-ml;\n+      include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n+\n+      # If registry-ml returns a 404 for a read, fall back to Swift.\n+      proxy_intercept_errors on;\n+      error_page 404 = @fallback;\n+    }\n+\n+    # This block applies to POST/PUT/DELETE/etc. methods to\n+    # /v2/wikimedia/machine-learning-*. Writes go straight to registry-ml with\n+    # no Swift fallback.\n+    location @ml_service_write {\n+      auth_request /auth;\n+      auth_request_set $auth_status $upstream_status;\n+\n+      proxy_pass http://registry-ml;\n+      include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n+    }\n+\n+    # Special location block: fallback reads to Swift.\n+    # This block is meant to be referenced by other blocks with\n+    # the 'error_page 404' directive. If a read fails on S3, nginx\n+    # falls back to Swift before returning to the client.\n+    location @fallback {\n+      internal;\n+\n+      # Push-time layer existence checks (HEAD .../blobs/...) must see the real\n+      # 404 from registry-ml rather than a Swift hit, otherwise the client skips\n+      # uploading the layer and the manifest PUT to registry-{ml,..} fails. See the\n+      # $skip_fallback map above.\n+      if ($skip_fallback) {\n+        return 404;\n+      }\n+\n+      access_log /var/log/nginx/access_swift_fallback.log upstream_time;\n+\n+      proxy_pass http://registry-swift;\n+      include /etc/nginx/registry-nginx-common-proxy-settings.conf;\n+    }\n+\n     # Catch-all location for all the Docker images.\n     # Authorization can happen via Basic Auth or JWT token,\n     # via the auth_request directives. The Docker images handled\n@@ -843,7 +928,6 @@\n         allow 127.0.0.1/32;\n         deny all;\n       }\n-\n     }\n \n     # Below are a number of internal locations used by auth_request to route"}, {"resource": "Class[Profile::Docker_registry]", "parameters": "--- Class[Profile::Docker_registry].orig\n+++ Class[Profile::Docker_registry]\n\n-    nginx_blob_cache => False\n"}, {"resource": "Nginx::Site[registry]"}], "perc_changed": "0.20%"}}}