coreci
da15bda6d2
---ci--- phase: 3 milestone: v0.2 status: complete requirements: covered: [REQ-21, REQ-22, REQ-23] partial: [] ---/ci---
844 lines
36 KiB
Python
844 lines
36 KiB
Python
#!/usr/bin/env python3
|
|
"""Daemonless OCI image publish client (D-019; stdlib only).
|
|
|
|
Usage:
|
|
python3 scripts/publish_image.py <tag> [--smoke]
|
|
[--artifact PATH] [--hub-registry URL] [--hub-auth URL]
|
|
[--gitea-registry URL] [--env-root DIR] [--owner O] [--repo R]
|
|
[--base-manifest DIGEST] [--timeout SECS]
|
|
|
|
Flow (REQ-23, RESEARCH §3): pull the pinned ``python:3.11-slim`` base
|
|
from the public registry anonymously (token -> OCI index -> resolve the
|
|
linux/amd64 child manifest by digest -> config + layer blobs; digests
|
|
recorded for provenance; ``--base-manifest`` overrides the tag
|
|
resolution for exact re-publishes) -> authenticate to the gitea
|
|
container registry (``GET /v2/token`` with Basic credentials — the
|
|
gitea token comes only from the sanctioned env files via
|
|
``scripts/_env_secrets.py``; the endpoint answers 200 even anonymously
|
|
with an empty-scope JWT, so the issued token must be non-empty and
|
|
failures must be loud — never treat a 200 alone as authorization) ->
|
|
push: HEAD each blob first (200 -> skip — retries resume cheap; the
|
|
~45.7 MB base layers are re-uploaded once per milestone because the
|
|
registries cannot cross-mount) -> monolithic blob upload
|
|
``POST /v2/<name>/blobs/uploads/?digest=`` (202 + ``Location`` ->
|
|
two-step PUT) with explicit Location-following on 307/308 (stdlib
|
|
urllib does NOT method-preserve on PUT/POST redirects) -> deterministic
|
|
broker layer (sorted tar entries, fixed metadata, member at the stable
|
|
path ``/app/coreci-broker.pyz``, ``gzip.compress(..., mtime=0)``) ->
|
|
new config blob (base config + appended diff_id; Entrypoint
|
|
``["python3", "/app/coreci-broker.pyz"]``, ``Cmd []``, ``WorkingDir
|
|
"/state"``, version label; serialized with sort_keys) -> OCI manifest
|
|
PUT with the base layer descriptors reused verbatim under ``<tag>``
|
|
FIRST, then ``latest`` (D-021).
|
|
|
|
Exit codes: 0 only after both manifest PUTs confirm 201; 1 after a
|
|
remote failure — bounded retries (3 attempts, 2s/4s backoff) exhausted
|
|
OR a permanent registry answer — printing the machine-readable
|
|
``IMAGE_PENDING <image>:<tag>`` marker for the ship escalation (never
|
|
blocks tag+merge); 2 for local precondition errors (missing artifact,
|
|
token, or config — nothing attempted remotely, no marker). Token values
|
|
are never printed.
|
|
|
|
Smoke mode (``--smoke``; G-013) — read-only pre-ship verify. NEVER part
|
|
of the default publish flow (the flag dispatches an early return; the
|
|
publish path above never consults it): four GET-only checks against
|
|
the live endpoints, run in order and stopping at the first failure —
|
|
nothing is uploaded, no manifest is PUT, no remote state is mutated.
|
|
|
|
1. ``GET {gitea}/v2/`` must answer 401 with a Bearer challenge whose
|
|
realm is the gitea token endpoint (the realm is recorded for 2).
|
|
2. Token->JWT exchange at that recorded realm
|
|
(``?service=container_registry&scope=repository:{o}/{r}:pull,push``)
|
|
with Basic auth — the gitea token resolves ONLY from the sanctioned
|
|
env files via ``scripts/_env_secrets.py`` (never the shell
|
|
environment); an empty token in the 200 answer is the anonymous-200
|
|
trap and fails loudly.
|
|
3. ``GET {gitea}/api/v1/repos/{o}/{r}/releases/tags/<tag>`` — 200
|
|
(exists) and 404 (not yet created) are BOTH acceptable pre-ship and
|
|
the line says which; the release's asset names print as a fact, and
|
|
a hard assertion binds ONLY when the tag is exactly ``v0.2.0``
|
|
(G-021 forge-side confirmation: the phase-0 release predates the
|
|
CLI and must carry NO ``.pyz`` asset, so any ``.pyz`` in its asset
|
|
list fails the smoke — the structural half of the guarantee,
|
|
release.py taking the tag as a positional argument with no v0.2.0
|
|
code path, is already machine-checked in the suite).
|
|
4. Docker Hub anonymous index resolve of ``python:3.11-slim`` with the
|
|
full Accept set must yield an index whose linux/amd64 child is
|
|
resolved; the amd64 child digest is printed — the base-drift fact
|
|
the P3-05 ship compares against expectations.
|
|
|
|
Smoke exit codes mirror the publish flow: 0 all four PASS; 1 with one
|
|
clear FAILED line per failure (checks after the failed one never run);
|
|
2 for local preconditions (missing token — nothing attempted
|
|
remotely). Smoke never prints token values and never emits
|
|
``IMAGE_PENDING`` (nothing was attempted, so nothing is pending).
|
|
|
|
Live gating (G-013): the LIVE smoke executes only at the P3-05 ship
|
|
step, which sets ``PUBLISH_SMOKE=1`` in the environment before invoking
|
|
this mode — this script neither reads nor requires that variable (the
|
|
SHIP GATE owns the gating). The smoke's LOGIC is what the default
|
|
unittest suite exercises (stub-tested, zero network access); the live
|
|
smoke itself never runs in the default suite.
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
import gzip
|
|
import hashlib
|
|
import http.client
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import tarfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
_SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
if _SCRIPTS_DIR not in sys.path:
|
|
sys.path.insert(0, _SCRIPTS_DIR)
|
|
|
|
import _env_secrets # sibling import (scripts/ dir on sys.path)
|
|
|
|
METADATA_TIMEOUT_SECONDS = 30
|
|
BLOB_TIMEOUT_SECONDS = 120
|
|
MAX_ATTEMPTS = 3
|
|
BACKOFF_SECONDS = (2, 4)
|
|
MAX_REDIRECTS = 5
|
|
BASE_REPO = "library/python"
|
|
BASE_TAG = "3.11-slim"
|
|
HUB_AUTH_SERVICE = "registry.docker.io"
|
|
GITEA_AUTH_SERVICE = "container_registry"
|
|
# The smoke-mode G-021 forge-side no-.pyz assertion binds ONLY this
|
|
# tag: the phase-0 release predates the CLI (D-020 starts at v0.2.1).
|
|
SMOKE_G021_TAG = "v0.2.0"
|
|
|
|
OCI_INDEX = "application/vnd.oci.image.index.v1+json"
|
|
OCI_MANIFEST = "application/vnd.oci.image.manifest.v1+json"
|
|
OCI_CONFIG = "application/vnd.oci.image.config.v1+json"
|
|
OCI_LAYER = "application/vnd.oci.image.layer.v1.tar+gzip"
|
|
DOCKER_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
|
|
DOCKER_MANIFEST = "application/vnd.docker.distribution.manifest.v2+json"
|
|
ACCEPT_MANIFESTS = ", ".join(
|
|
(OCI_INDEX, DOCKER_LIST, OCI_MANIFEST, DOCKER_MANIFEST))
|
|
|
|
|
|
class PublishError(Exception):
|
|
"""Image-publisher failure base."""
|
|
|
|
|
|
class TransientFailure(PublishError):
|
|
"""Connection error or 5xx persisting past the bounded retries."""
|
|
|
|
|
|
class RegistryApiError(PublishError):
|
|
"""Permanent registry answer carrying the remote detail."""
|
|
|
|
|
|
class LocalError(PublishError):
|
|
"""Local precondition failure (nothing was attempted remotely)."""
|
|
|
|
|
|
def repo_root():
|
|
return os.path.dirname(_SCRIPTS_DIR)
|
|
|
|
|
|
def load_release_defaults(root=None):
|
|
"""Read the release block from .ciagent/config.json ({} if absent)."""
|
|
path = os.path.join(root or repo_root(), ".ciagent", "config.json")
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
config = json.load(handle)
|
|
except (OSError, ValueError):
|
|
return {}
|
|
block = config.get("release")
|
|
return block if isinstance(block, dict) else {}
|
|
|
|
|
|
def _error_detail(body):
|
|
try:
|
|
payload = json.loads(body.decode("utf-8", "replace"))
|
|
except ValueError:
|
|
text = body.decode("utf-8", "replace").strip()
|
|
return text[:200] if text else "<empty body>"
|
|
if isinstance(payload, dict):
|
|
for key in ("errors", "message"):
|
|
if payload.get(key):
|
|
return json.dumps(payload[key])[:200]
|
|
return json.dumps(payload)[:200] if not isinstance(payload, str) else payload[:200]
|
|
|
|
|
|
def _header(headers, name):
|
|
"""Case-insensitive response-header lookup — ``_attempt_once``
|
|
returns a plain dict whose keys keep the wire casing, and registry
|
|
header casing is not guaranteed (e.g. WWW-Authenticate)."""
|
|
for key, value in headers.items():
|
|
if key.lower() == name.lower():
|
|
return value
|
|
return None
|
|
|
|
|
|
def _parse_json(body, context, status):
|
|
"""Parse a remote JSON body (token / manifest / config answers).
|
|
|
|
Malformed bodies raise RegistryApiError — never an uncaught
|
|
ValueError — so main()'s handler keeps the exit-1 + IMAGE_PENDING
|
|
marker contract for this failure class (CWE-754).
|
|
"""
|
|
try:
|
|
return json.loads(body.decode("utf-8", "replace"))
|
|
except ValueError as error:
|
|
raise RegistryApiError(
|
|
"{} answered malformed JSON ({})".format(context, error),
|
|
status, body)
|
|
|
|
|
|
def _attempt_once(method, url, headers, data, timeout):
|
|
"""One HTTP exchange; HTTPError becomes a completed answer."""
|
|
request = urllib.request.Request(url, data=data, method=method)
|
|
for key, value in headers.items():
|
|
request.add_header(key, value)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
body = b"" if method == "HEAD" else response.read()
|
|
return response.status, body, dict(response.headers.items())
|
|
except urllib.error.HTTPError as error:
|
|
body = b"" if method == "HEAD" else error.read()
|
|
return error.code, body, dict((error.headers or {}).items())
|
|
|
|
|
|
def _origin(url):
|
|
"""(scheme, netloc) of a URL — two URLs share an origin iff both match."""
|
|
parts = urllib.parse.urlsplit(url)
|
|
return (parts.scheme, parts.netloc)
|
|
|
|
|
|
def _attempt_following(method, url, headers, data, timeout):
|
|
"""One attempt, following 307/308 redirects explicitly (urllib does
|
|
NOT method-preserve on PUT/POST — the Location may be relative).
|
|
|
|
Cross-origin discipline (CWE-200): when a followed Location resolves
|
|
to a different origin than the ORIGINAL request's, the Authorization
|
|
header is stripped from the re-issued request — the registry JWT
|
|
must never reach another origin (blob-upload Locations are commonly
|
|
presigned external storage). Same-origin redirects keep it; once
|
|
stripped it is never re-attached."""
|
|
current_url = url
|
|
current_headers = dict(headers)
|
|
for _ in range(MAX_REDIRECTS + 1):
|
|
status, body, resp_headers = _attempt_once(
|
|
method, current_url, current_headers, data, timeout)
|
|
if status in (307, 308) and resp_headers.get("Location"):
|
|
next_url = urllib.parse.urljoin(
|
|
current_url, resp_headers["Location"])
|
|
if _origin(next_url) != _origin(url):
|
|
current_headers.pop("Authorization", None)
|
|
current_url = next_url
|
|
continue
|
|
return status, body, resp_headers
|
|
raise RegistryApiError(
|
|
"too many redirects following {} {}".format(method, url), 500, b"")
|
|
|
|
|
|
def _request(method, url, headers, data, timeout,
|
|
max_attempts=None, backoff=None):
|
|
"""Bounded-retry HTTP: retries connection errors and 5xx answers.
|
|
|
|
Returns (status, body, response_headers) for any answer below 500;
|
|
a 4xx returns immediately (permanent). Raises TransientFailure once
|
|
the retry budget is exhausted.
|
|
"""
|
|
attempts = MAX_ATTEMPTS if max_attempts is None else int(max_attempts)
|
|
delays = BACKOFF_SECONDS if backoff is None else tuple(backoff)
|
|
failure = "unknown failure"
|
|
for attempt in range(1, attempts + 1):
|
|
try:
|
|
status, body, resp_headers = _attempt_following(
|
|
method, url, headers, data, timeout)
|
|
except (OSError, http.client.HTTPException) as error:
|
|
status = None
|
|
failure = "connection error: {}: {}".format(
|
|
type(error).__name__, error)
|
|
else:
|
|
if status < 500:
|
|
return status, body, resp_headers
|
|
failure = "HTTP {}: {}".format(status, _error_detail(body))
|
|
if attempt < attempts and delays:
|
|
time.sleep(delays[min(attempt - 1, len(delays) - 1)])
|
|
raise TransientFailure(
|
|
"{} {} failed after {} attempt(s): {}".format(
|
|
method, url, attempts, failure))
|
|
|
|
|
|
def _hub_token(hub_auth, scope):
|
|
query = urllib.parse.urlencode(
|
|
{"service": HUB_AUTH_SERVICE, "scope": scope})
|
|
status, body, _ = _request(
|
|
"GET", "{}?{}".format(hub_auth, query), {}, None,
|
|
METADATA_TIMEOUT_SECONDS)
|
|
if status != 200:
|
|
raise RegistryApiError("hub token request failed", status, body)
|
|
payload = _parse_json(body, "hub token endpoint", status)
|
|
token = ""
|
|
if isinstance(payload, dict):
|
|
token = payload.get("token") or payload.get("access_token") or ""
|
|
if not token:
|
|
raise RegistryApiError("hub token endpoint returned no token",
|
|
status, body)
|
|
return token
|
|
|
|
|
|
def _fetch_manifest(registry, repository, ref, bearer, accept=ACCEPT_MANIFESTS):
|
|
url = "{}/v2/{}/manifests/{}".format(registry.rstrip("/"), repository, ref)
|
|
status, body, _ = _request(
|
|
"GET", url, {"Authorization": "Bearer {}".format(bearer),
|
|
"Accept": accept}, None, METADATA_TIMEOUT_SECONDS)
|
|
if status != 200:
|
|
raise RegistryApiError(
|
|
"manifest fetch {} {}".format(repository, ref), status, body)
|
|
return _parse_json(
|
|
body, "manifest fetch {} {}".format(repository, ref), status)
|
|
|
|
|
|
def _resolve_amd64_child(index):
|
|
"""linux/amd64 child digest from a base index; malformed index
|
|
shapes degrade to the not-found RegistryApiError, never a crash."""
|
|
manifests = index.get("manifests") if isinstance(index, dict) else None
|
|
if not isinstance(manifests, list):
|
|
manifests = []
|
|
for descriptor in manifests:
|
|
if not isinstance(descriptor, dict):
|
|
continue
|
|
platform = descriptor.get("platform")
|
|
if not isinstance(platform, dict):
|
|
continue
|
|
if platform.get("os") == "linux" and platform.get(
|
|
"architecture") == "amd64":
|
|
return descriptor.get("digest")
|
|
raise RegistryApiError(
|
|
"no linux/amd64 child manifest in the base index", 404,
|
|
json.dumps(manifests).encode("utf-8", "replace"))
|
|
|
|
|
|
def _fetch_blob(registry, repository, digest, bearer, timeout):
|
|
url = "{}/v2/{}/blobs/{}".format(
|
|
registry.rstrip("/"), repository, digest)
|
|
status, body, _ = _request(
|
|
"GET", url, {"Authorization": "Bearer {}".format(bearer)}, None,
|
|
timeout)
|
|
if status != 200:
|
|
raise RegistryApiError("blob fetch {}".format(digest), status, body)
|
|
if "sha256:" + hashlib.sha256(body).hexdigest() != digest:
|
|
raise RegistryApiError("blob digest mismatch {}".format(digest),
|
|
500, b"")
|
|
return body
|
|
|
|
|
|
def _child_parts(child):
|
|
"""Validated (config descriptor, layer descriptors) of a base child
|
|
manifest — missing or malformed keys raise a clear RegistryApiError
|
|
(never an uncaught KeyError), keeping the marker contract for this
|
|
failure class (CWE-754)."""
|
|
def malformed(reason):
|
|
return RegistryApiError(
|
|
"malformed base child manifest: {}".format(reason), 500,
|
|
json.dumps(child).encode("utf-8", "replace"))
|
|
|
|
if not isinstance(child, dict):
|
|
raise malformed("not a JSON object")
|
|
config = child.get("config")
|
|
if not isinstance(config, dict) or not config.get("digest"):
|
|
raise malformed("missing or empty config.digest")
|
|
layers = child.get("layers")
|
|
if not isinstance(layers, list):
|
|
raise malformed("missing or non-list layers")
|
|
for descriptor in layers:
|
|
if not isinstance(descriptor, dict) or not descriptor.get("digest"):
|
|
raise malformed("layer descriptor without a digest")
|
|
return config, layers
|
|
|
|
|
|
def _gitea_jwt(token_url, owner, repo, token, timeout):
|
|
"""Token->JWT exchange at a token endpoint URL (Basic auth).
|
|
|
|
``token_url`` is the FULL endpoint: the registry's ``/v2/token``
|
|
for the publish flow, or the realm recorded from a ``/v2/``
|
|
challenge for the smoke (the standard token-auth flow).
|
|
"""
|
|
query = urllib.parse.urlencode({
|
|
"service": GITEA_AUTH_SERVICE,
|
|
"scope": "repository:{}/{}:pull,push".format(owner, repo),
|
|
})
|
|
basic = base64.b64encode(
|
|
"publisher:{}".format(token).encode("utf-8")).decode("ascii")
|
|
status, body, _ = _request(
|
|
"GET", "{}?{}".format(token_url, query),
|
|
{"Authorization": "Basic {}".format(basic)}, None, timeout)
|
|
if status != 200:
|
|
raise RegistryApiError("gitea registry token request failed",
|
|
status, body)
|
|
payload = _parse_json(body, "gitea registry token endpoint", status)
|
|
jwt = (payload.get("token") or "") if isinstance(payload, dict) else ""
|
|
if not jwt:
|
|
# The endpoint answers 200 even anonymously with an unusable
|
|
# (empty-scope) JWT — a 200 proves nothing about push rights.
|
|
raise RegistryApiError(
|
|
"gitea registry token endpoint returned an empty token "
|
|
"(anonymous-200 trap refused; never treat 200 as authorization)",
|
|
status, body)
|
|
return jwt
|
|
|
|
|
|
def _bearer_challenge(www_authenticate):
|
|
"""Realm recorded from a Bearer WWW-Authenticate challenge.
|
|
|
|
Returns None when the header is absent, is not a Bearer challenge,
|
|
or carries no realm — a challenge without a realm cannot be
|
|
followed to the token endpoint, so the smoke treats that as no
|
|
challenge at all.
|
|
"""
|
|
if not www_authenticate:
|
|
return None
|
|
parts = www_authenticate.strip().split(None, 1)
|
|
if len(parts) < 2 or parts[0].lower() != "bearer":
|
|
return None
|
|
match = re.search(r'\brealm\s*=\s*"([^"]*)"', parts[1])
|
|
return match.group(1) if match else None
|
|
|
|
|
|
def _asset_names(release):
|
|
"""Asset display names from a gitea release payload — odd shapes
|
|
degrade to [] (the read-only smoke never crashes on them)."""
|
|
if not isinstance(release, dict):
|
|
return []
|
|
assets = release.get("assets")
|
|
if not isinstance(assets, list):
|
|
return []
|
|
names = []
|
|
for asset in assets:
|
|
if isinstance(asset, dict) and isinstance(asset.get("name"), str):
|
|
names.append(asset["name"])
|
|
return names
|
|
|
|
|
|
def _blob_exists(gitea_registry, name, digest, jwt, timeout):
|
|
url = "{}/v2/{}/blobs/{}".format(
|
|
gitea_registry.rstrip("/"), name, digest)
|
|
status, _, _ = _request(
|
|
"HEAD", url, {"Authorization": "Bearer {}".format(jwt)}, None,
|
|
timeout)
|
|
return status == 200
|
|
|
|
|
|
def _upload_blob(gitea_registry, name, digest, blob, jwt, timeout):
|
|
base_url = gitea_registry.rstrip("/")
|
|
url = "{}/v2/{}/blobs/uploads/?digest={}".format(base_url, name, digest)
|
|
headers = {"Authorization": "Bearer {}".format(jwt),
|
|
"Content-Type": "application/octet-stream"}
|
|
status, body, resp_headers = _request(
|
|
"POST", url, headers, blob, timeout)
|
|
if status in (200, 201):
|
|
return
|
|
if status == 202 and resp_headers.get("Location"):
|
|
location = urllib.parse.urljoin(url, resp_headers["Location"])
|
|
if "digest=" not in location:
|
|
separator = "&" if "?" in location else "?"
|
|
location = "{}{}digest={}".format(location, separator, digest)
|
|
status, body, _ = _request(
|
|
"PUT", location, headers, blob, timeout)
|
|
if status in (200, 201):
|
|
return
|
|
raise RegistryApiError("blob upload {}".format(digest), status, body)
|
|
|
|
|
|
def _push_blob_if_missing(gitea_registry, name, digest, blob, jwt, timeout):
|
|
"""HEAD-skip then upload — retries resume cheap."""
|
|
if _blob_exists(gitea_registry, name, digest, jwt, timeout):
|
|
return False
|
|
_upload_blob(gitea_registry, name, digest, blob, jwt, timeout)
|
|
return True
|
|
|
|
|
|
def build_broker_layer(artifact_bytes):
|
|
"""Deterministic layer carrying /app/coreci-broker.pyz.
|
|
|
|
Returns (gzip_blob, layer_digest, diff_id): sorted single entry,
|
|
fixed TarInfo metadata, gzip with mtime=0 — byte-stable rebuilds.
|
|
"""
|
|
info = tarfile.TarInfo("app/coreci-broker.pyz")
|
|
info.size = len(artifact_bytes)
|
|
info.mode = 0o644
|
|
info.uid = 0
|
|
info.gid = 0
|
|
info.uname = ""
|
|
info.gname = ""
|
|
info.mtime = 0
|
|
buffer = io.BytesIO()
|
|
with tarfile.open(fileobj=buffer, mode="w",
|
|
format=tarfile.GNU_FORMAT) as archive:
|
|
archive.addfile(info, io.BytesIO(artifact_bytes))
|
|
raw = buffer.getvalue()
|
|
compressed = gzip.compress(raw, compresslevel=9, mtime=0)
|
|
layer_digest = "sha256:" + hashlib.sha256(compressed).hexdigest()
|
|
diff_id = "sha256:" + hashlib.sha256(raw).hexdigest()
|
|
return compressed, layer_digest, diff_id
|
|
|
|
|
|
def build_config_blob(base_config, diff_id, tag):
|
|
"""Base config + appended diff_id + CLI entrypoint/WorkingDir/label.
|
|
|
|
Serialized with sort_keys — deterministic config digest.
|
|
"""
|
|
config = dict(base_config)
|
|
rootfs = dict(base_config.get("rootfs") or {})
|
|
rootfs["diff_ids"] = list(rootfs.get("diff_ids") or []) + [diff_id]
|
|
if "type" not in rootfs:
|
|
rootfs["type"] = "layers"
|
|
config["rootfs"] = rootfs
|
|
inner = dict(base_config.get("config") or {})
|
|
inner["Entrypoint"] = ["python3", "/app/coreci-broker.pyz"]
|
|
inner["Cmd"] = []
|
|
inner["WorkingDir"] = "/state"
|
|
inner["Labels"] = {"org.opencontainers.image.version": tag}
|
|
config["config"] = inner
|
|
return json.dumps(config, sort_keys=True,
|
|
separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def build_manifest_blob(base_layer_descriptors, broker_layer_descriptor,
|
|
config_descriptor):
|
|
manifest = {
|
|
"schemaVersion": 2,
|
|
"mediaType": OCI_MANIFEST,
|
|
"config": config_descriptor,
|
|
"layers": list(base_layer_descriptors) + [broker_layer_descriptor],
|
|
}
|
|
return json.dumps(manifest, sort_keys=True,
|
|
separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def _descriptor(digest, size, media_type):
|
|
return {"mediaType": media_type, "digest": digest, "size": size}
|
|
|
|
|
|
def _put_manifest(gitea_registry, name, tag, manifest_blob, jwt, timeout):
|
|
url = "{}/v2/{}/manifests/{}".format(
|
|
gitea_registry.rstrip("/"), name, tag)
|
|
status, body, _ = _request(
|
|
"PUT", url, {"Authorization": "Bearer {}".format(jwt),
|
|
"Content-Type": OCI_MANIFEST}, manifest_blob, timeout)
|
|
if status not in (200, 201):
|
|
raise RegistryApiError("manifest PUT {}".format(tag), status, body)
|
|
return "sha256:" + hashlib.sha256(manifest_blob).hexdigest()
|
|
|
|
|
|
def _load_artifact(path):
|
|
if not os.path.isfile(path):
|
|
raise LocalError("artifact not found: {}".format(path))
|
|
with open(path, "rb") as handle:
|
|
return handle.read()
|
|
|
|
|
|
def smoke(tag, gitea_registry, hub_registry, hub_auth, owner, repo,
|
|
timeout, env_root):
|
|
"""G-013 read-only live smoke: the four GET-only pre-ship checks.
|
|
|
|
Never part of the default publish flow; never mutates remote state
|
|
(no uploads, no manifest PUTs). The LIVE invocation happens only
|
|
at the P3-05 ship step, which gates it by setting PUBLISH_SMOKE=1
|
|
in the environment — this script neither reads nor requires that
|
|
variable (the SHIP GATE owns the gating). The default unittest
|
|
suite exercises this LOGIC against stubs (zero network).
|
|
|
|
Checks, in order, stopping at the first failure (see the module
|
|
docstring for the full G-021 tie-in): [1/4] the gitea /v2/ 401
|
|
Bearer challenge (realm recorded); [2/4] the token->JWT exchange at
|
|
the recorded realm — the gitea token resolves only from the
|
|
sanctioned env files via ``_env_secrets.get_token`` (never the
|
|
shell environment), and an empty token is the anonymous-200 trap
|
|
refused loudly; [3/4] the gitea API release-by-tag GET — 200
|
|
(exists) and 404 (not yet created) are both acceptable pre-ship,
|
|
asset names print as a fact, and the hard no-.pyz assertion binds
|
|
ONLY the tag v0.2.0 exactly (G-021 forge-side confirmation);
|
|
[4/4] the Docker Hub anonymous index resolve of python:3.11-slim —
|
|
the linux/amd64 child digest prints (the drift fact P3-05
|
|
compares against expectations).
|
|
|
|
Returns 0 (all PASS), or 1 after one clear FAILED line (token
|
|
values are never printed; IMAGE_PENDING is never emitted — nothing
|
|
was attempted, so nothing is pending). Local preconditions (a
|
|
missing token) raise ``_env_secrets.SecretsError`` BEFORE any
|
|
request; main() maps that to exit 2.
|
|
"""
|
|
gitea_token = _env_secrets.get_token(root=env_root)
|
|
metadata_timeout = min(METADATA_TIMEOUT_SECONDS, timeout)
|
|
gitea_base = gitea_registry.rstrip("/")
|
|
|
|
# [1/4] gitea /v2/ auth challenge (anonymous; realm recorded).
|
|
try:
|
|
status, _, resp_headers = _request(
|
|
"GET", "{}/v2/".format(gitea_base), {}, None, metadata_timeout)
|
|
except (TransientFailure, RegistryApiError) as error:
|
|
print("FAILED [1/4] gitea /v2/ auth challenge: {}".format(error))
|
|
return 1
|
|
challenge = _header(resp_headers, "WWW-Authenticate")
|
|
realm = _bearer_challenge(challenge)
|
|
if status != 401 or not realm:
|
|
print("FAILED [1/4] gitea /v2/ auth challenge: expected 401 with "
|
|
"a Bearer challenge, got HTTP {} (WWW-Authenticate: {!r})"
|
|
.format(status, challenge))
|
|
return 1
|
|
print("PASS [1/4] gitea /v2/ auth challenge: 401 + Bearer challenge "
|
|
"(realm {})".format(realm))
|
|
|
|
# [2/4] token->JWT exchange at the recorded realm (Basic auth).
|
|
try:
|
|
_gitea_jwt(realm, owner, repo, gitea_token, metadata_timeout)
|
|
except (TransientFailure, RegistryApiError) as error:
|
|
print("FAILED [2/4] gitea token->JWT exchange at {}: {}".format(
|
|
realm, error))
|
|
return 1
|
|
print("PASS [2/4] gitea token->JWT exchange: 200 + non-empty token "
|
|
"(service {}, scope repository:{}/{}:pull,push)".format(
|
|
GITEA_AUTH_SERVICE, owner, repo))
|
|
|
|
# [3/4] release-by-tag GET (gitea API; 200 exists, 404 pre-ship ok).
|
|
release_url = "{}/api/v1/repos/{}/{}/releases/tags/{}".format(
|
|
gitea_base, owner, repo, urllib.parse.quote(tag, safe=""))
|
|
try:
|
|
status, body, _ = _request(
|
|
"GET", release_url,
|
|
{"Authorization": "token {}".format(gitea_token)}, None,
|
|
metadata_timeout)
|
|
except (TransientFailure, RegistryApiError) as error:
|
|
print("FAILED [3/4] release-by-tag {}: {}".format(tag, error))
|
|
return 1
|
|
if status == 404:
|
|
print("PASS [3/4] release-by-tag {}: not yet created (404) — "
|
|
"acceptable pre-ship".format(tag))
|
|
elif status != 200:
|
|
print("FAILED [3/4] release-by-tag {}: expected 200 or 404, got "
|
|
"HTTP {}".format(tag, status))
|
|
return 1
|
|
else:
|
|
try:
|
|
release = _parse_json(
|
|
body, "release-by-tag {}".format(tag), status)
|
|
except RegistryApiError as error:
|
|
print("FAILED [3/4] release-by-tag {}: {}".format(tag, error))
|
|
return 1
|
|
asset_names = _asset_names(release)
|
|
pyz_assets = [name for name in asset_names
|
|
if name.endswith(".pyz")]
|
|
if tag == SMOKE_G021_TAG and pyz_assets:
|
|
print("FAILED [3/4] release-by-tag {}: G-021 violation — the "
|
|
"phase-0 release must carry no .pyz asset, found: {}"
|
|
.format(tag, ", ".join(pyz_assets)))
|
|
return 1
|
|
listing = ", ".join(asset_names) if asset_names else "(none)"
|
|
note = (" — G-021 confirmed: no .pyz asset"
|
|
if tag == SMOKE_G021_TAG else "")
|
|
print("PASS [3/4] release-by-tag {}: exists (assets: {}){}".format(
|
|
tag, listing, note))
|
|
|
|
# [4/4] Docker Hub index resolve (anonymous; amd64 drift print).
|
|
try:
|
|
hub_bearer = _hub_token(
|
|
hub_auth, "repository:{}:pull".format(BASE_REPO))
|
|
index = _fetch_manifest(hub_registry, BASE_REPO, BASE_TAG,
|
|
hub_bearer)
|
|
child_digest = _resolve_amd64_child(index)
|
|
if not child_digest:
|
|
raise RegistryApiError(
|
|
"amd64 child descriptor carries no digest", 500, b"")
|
|
except (TransientFailure, RegistryApiError) as error:
|
|
print("FAILED [4/4] hub index resolve {}:{}: {}".format(
|
|
BASE_REPO, BASE_TAG, error))
|
|
return 1
|
|
print("PASS [4/4] hub index resolve {}:{} -> linux/amd64 child {}"
|
|
.format(BASE_REPO, BASE_TAG, child_digest))
|
|
return 0
|
|
|
|
|
|
def publish(tag, artifact_path, hub_registry, hub_auth, gitea_registry,
|
|
owner, repo, base_manifest, timeout, env_root):
|
|
"""Run the full publish flow; returns the image provenance dict."""
|
|
artifact_bytes = _load_artifact(artifact_path)
|
|
gitea_token = _env_secrets.get_token(root=env_root)
|
|
name = "{}/{}".format(owner, repo)
|
|
metadata_timeout = min(METADATA_TIMEOUT_SECONDS, timeout)
|
|
|
|
hub_bearer = _hub_token(hub_auth, "repository:{}:pull".format(BASE_REPO))
|
|
if base_manifest:
|
|
child_digest = base_manifest
|
|
child = _fetch_manifest(hub_registry, BASE_REPO, base_manifest,
|
|
hub_bearer)
|
|
index_digest = None
|
|
else:
|
|
base = _fetch_manifest(hub_registry, BASE_REPO, BASE_TAG, hub_bearer)
|
|
index_digest = "sha256:" + hashlib.sha256(
|
|
json.dumps(base, sort_keys=True,
|
|
separators=(",", ":")).encode("utf-8")).hexdigest()
|
|
child_digest = (_resolve_amd64_child(base)
|
|
if "manifests" in base else BASE_TAG)
|
|
child = _fetch_manifest(hub_registry, BASE_REPO, child_digest,
|
|
hub_bearer)
|
|
child_config, base_layer_descriptors = _child_parts(child)
|
|
base_config_bytes = _fetch_blob(
|
|
hub_registry, BASE_REPO, child_config["digest"], hub_bearer,
|
|
timeout)
|
|
base_config = _parse_json(base_config_bytes, "base config blob", 200)
|
|
if not isinstance(base_config, dict):
|
|
raise RegistryApiError("base config blob is not a JSON object",
|
|
500, base_config_bytes)
|
|
|
|
jwt = _gitea_jwt("{}/v2/token".format(gitea_registry.rstrip("/")),
|
|
owner, repo, gitea_token, metadata_timeout)
|
|
|
|
pushed = []
|
|
pushed.append(_push_blob_if_missing(
|
|
gitea_registry, name, child_config["digest"], base_config_bytes,
|
|
jwt, timeout))
|
|
for descriptor in base_layer_descriptors:
|
|
if _blob_exists(gitea_registry, name, descriptor["digest"], jwt,
|
|
timeout):
|
|
continue
|
|
blob = _fetch_blob(hub_registry, BASE_REPO, descriptor["digest"],
|
|
hub_bearer, timeout)
|
|
_upload_blob(gitea_registry, name, descriptor["digest"], blob, jwt,
|
|
timeout)
|
|
pushed.append(True)
|
|
|
|
layer_blob, layer_digest, diff_id = build_broker_layer(artifact_bytes)
|
|
config_blob = build_config_blob(base_config, diff_id, tag)
|
|
config_digest = "sha256:" + hashlib.sha256(config_blob).hexdigest()
|
|
_push_blob_if_missing(gitea_registry, name, layer_digest, layer_blob,
|
|
jwt, timeout)
|
|
_push_blob_if_missing(gitea_registry, name, config_digest, config_blob,
|
|
jwt, timeout)
|
|
|
|
manifest_blob = build_manifest_blob(
|
|
base_layer_descriptors,
|
|
_descriptor(layer_digest, len(layer_blob), OCI_LAYER),
|
|
_descriptor(config_digest, len(config_blob), OCI_CONFIG))
|
|
manifest_digest = _put_manifest(
|
|
gitea_registry, name, tag, manifest_blob, jwt, timeout)
|
|
_put_manifest(gitea_registry, name, "latest", manifest_blob, jwt,
|
|
timeout)
|
|
|
|
return {
|
|
"base_index_digest": index_digest,
|
|
"base_child_digest": child_digest if child_digest != BASE_TAG
|
|
else "tag-resolved",
|
|
"base_config_digest": child_config["digest"],
|
|
"base_layers": [descriptor["digest"]
|
|
for descriptor in base_layer_descriptors],
|
|
"broker_layer_digest": layer_digest,
|
|
"broker_layer_diff_id": diff_id,
|
|
"config_digest": config_digest,
|
|
"manifest_digest": manifest_digest,
|
|
"blobs_uploaded": sum(1 for flag in pushed if flag),
|
|
}
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(
|
|
description="Daemonless OCI image publish (REQ-23, D-019)."
|
|
)
|
|
parser.add_argument("tag", help="image tag, e.g. v0.2.3")
|
|
parser.add_argument("--artifact",
|
|
help="path to the built .pyz artifact "
|
|
"(default: dist/coreci-broker-v<tag>.pyz)")
|
|
parser.add_argument("--hub-registry",
|
|
default="https://registry-1.docker.io",
|
|
help="public base-image registry")
|
|
parser.add_argument("--hub-auth", default="https://auth.docker.io/token",
|
|
help="public registry token service")
|
|
parser.add_argument("--gitea-registry", default="https://git.coreci.dev",
|
|
help="gitea registry base URL")
|
|
parser.add_argument("--owner", help="gitea owner (default: release.owner)")
|
|
parser.add_argument("--repo", help="gitea repo (default: release.repo)")
|
|
parser.add_argument("--base-manifest",
|
|
help="base child-manifest digest for exact "
|
|
"re-publishes (skips tag resolution)")
|
|
parser.add_argument("--smoke", action="store_true",
|
|
help="G-013 read-only live smoke: four GET-only "
|
|
"pre-ship checks, NO publish (see the "
|
|
"module docstring)")
|
|
parser.add_argument("--timeout", type=float, default=BLOB_TIMEOUT_SECONDS,
|
|
help="per-request timeout for blob operations "
|
|
"(seconds; metadata uses the smaller of this "
|
|
"and 30)")
|
|
parser.add_argument("--env-root",
|
|
help="root dir for the sanctioned .env* files "
|
|
"(default: repo root)")
|
|
args = parser.parse_args(argv)
|
|
|
|
defaults = load_release_defaults()
|
|
owner = args.owner or defaults.get("owner")
|
|
repo = args.repo or defaults.get("repo")
|
|
if not owner or not repo:
|
|
print("ERROR: missing gitea target (owner/repo); pass the flags or"
|
|
" extend the .ciagent/config.json release block",
|
|
file=sys.stderr)
|
|
return 2
|
|
blob_timeout = max(1.0, float(args.timeout))
|
|
if args.smoke:
|
|
# G-013 read-only live smoke: dispatched BEFORE any
|
|
# publish-side work and returning directly — the default
|
|
# publish flow below can never trigger it.
|
|
try:
|
|
return smoke(
|
|
tag=args.tag, gitea_registry=args.gitea_registry,
|
|
hub_registry=args.hub_registry, hub_auth=args.hub_auth,
|
|
owner=owner, repo=repo, timeout=blob_timeout,
|
|
env_root=args.env_root)
|
|
except _env_secrets.SecretsError as error:
|
|
print("ERROR: {}".format(error), file=sys.stderr)
|
|
return 2
|
|
artifact_path = args.artifact or os.path.join(
|
|
"dist", "coreci-broker-{}.pyz".format(args.tag))
|
|
# Reuse the smaller metadata timeout for the registry auth exchange so
|
|
# the timeout-abort path stays bounded for every request class.
|
|
try:
|
|
provenance = publish(
|
|
tag=args.tag, artifact_path=artifact_path,
|
|
hub_registry=args.hub_registry, hub_auth=args.hub_auth,
|
|
gitea_registry=args.gitea_registry, owner=owner, repo=repo,
|
|
base_manifest=args.base_manifest, timeout=blob_timeout,
|
|
env_root=args.env_root)
|
|
except _env_secrets.SecretsError as error:
|
|
print("ERROR: {}".format(error), file=sys.stderr)
|
|
return 2
|
|
except LocalError as error:
|
|
print("ERROR: {}".format(error), file=sys.stderr)
|
|
return 2
|
|
except (TransientFailure, RegistryApiError) as error:
|
|
print("ERROR: {}".format(error), file=sys.stderr)
|
|
registry_host = urllib.parse.urlsplit(
|
|
args.gitea_registry).netloc or args.gitea_registry
|
|
print("IMAGE_PENDING {}/{}:{}".format(registry_host, owner, repo,
|
|
args.tag))
|
|
return 1
|
|
|
|
print("image: {}/{}:{} (+ latest)".format(
|
|
args.gitea_registry.rstrip("/"), owner, repo, args.tag))
|
|
print("base index: {}".format(
|
|
provenance["base_index_digest"] or "direct (--base-manifest)"))
|
|
print("base amd64 manifest: {}".format(provenance["base_child_digest"]))
|
|
print("base config: {}".format(provenance["base_config_digest"]))
|
|
print("base layers: {}".format(", ".join(provenance["base_layers"])))
|
|
print("broker layer: {} (diff_id {})".format(
|
|
provenance["broker_layer_digest"],
|
|
provenance["broker_layer_diff_id"]))
|
|
print("config: {}".format(provenance["config_digest"]))
|
|
print("manifest: {}".format(provenance["manifest_digest"]))
|
|
print("blobs uploaded this run: {}".format(provenance["blobs_uploaded"]))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main()) |