Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 20 additions & 11 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ on:

permissions:
contents: write
packages: write

concurrency:
group: release-${{ github.ref }}
Expand Down Expand Up @@ -45,6 +46,12 @@ jobs:
echo "RELEASE_TAG=$TAG" >> "$GITHUB_ENV"
echo "RELEASE_VERSION=${TAG#v}" >> "$GITHUB_ENV"

# Fail before publishing an image for a tag that has no Release to attach assets to.
- name: Require an existing GitHub Release
env:
GH_TOKEN: ${{ github.token }}
run: gh release view "$RELEASE_TAG" >/dev/null

- name: Install protobuf compiler
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler

Expand All @@ -66,16 +73,21 @@ jobs:
cargo build --locked --release \
-p fleet-scheduler -p fleetctl -p fleet-mcp -p fleet-node -p fleet-proof-app

- name: Build scheduler Docker image
- name: Publish scheduler image to GHCR
shell: bash
env:
REGISTRY_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
IMAGE="ghcr.io/${GITHUB_REPOSITORY,,}-scheduler"
echo "$REGISTRY_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
docker build \
--tag "zkm-fleet-scheduler:${RELEASE_VERSION}" \
--tag zkm-fleet-scheduler:local \
--label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
--label "org.opencontainers.image.version=${RELEASE_VERSION}" \
--label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
--tag "$IMAGE:${RELEASE_VERSION}" \
-f deploy/Dockerfile.scheduler .
mkdir -p dist
docker save zkm-fleet-scheduler:local | gzip > "dist/zkm-fleet-scheduler-${RELEASE_VERSION}-linux-x86_64.tar.gz"
cd dist
sha256sum "zkm-fleet-scheduler-${RELEASE_VERSION}-linux-x86_64.tar.gz" > "zkm-fleet-scheduler-${RELEASE_VERSION}-linux-x86_64.tar.gz.sha256"
docker push "$IMAGE:${RELEASE_VERSION}"

- name: Build release package
shell: bash
Expand All @@ -88,7 +100,7 @@ jobs:
install -m 755 target/release/fleet-mcp "$DIST/bin/"
install -m 755 target/release/fleet-node "$DIST/bin/"
install -m 755 target/release/fleet-proof-app "$DIST/bin/"
cp -a deploy/docker-compose.yml deploy/gateway.yaml.in deploy/Dockerfile.scheduler \
cp -a deploy/docker-compose.yml deploy/gateway.yaml.in \
deploy/install.sh deploy/setup.sh deploy/fleet-watchdog.sh deploy/ray-start.sh deploy/node-start.sh deploy/fleet-ctl.sh "$DIST/deploy/"
cp -a bridge/pyproject.toml bridge/fleet_bridge "$DIST/bridge/"
cp README.md docs/installation.md docs/maintenance.md docs/reference.md "$DIST/"
Expand All @@ -103,10 +115,7 @@ jobs:
shell: bash
run: |
set -euo pipefail
gh release view "$RELEASE_TAG" >/dev/null
gh release upload "$RELEASE_TAG" \
"dist/zkm-fleet-${RELEASE_VERSION}-linux-x86_64.tar.gz" \
"dist/zkm-fleet-${RELEASE_VERSION}-linux-x86_64.tar.gz.sha256" \
"dist/zkm-fleet-scheduler-${RELEASE_VERSION}-linux-x86_64.tar.gz" \
"dist/zkm-fleet-scheduler-${RELEASE_VERSION}-linux-x86_64.tar.gz.sha256" \
--clobber
14 changes: 7 additions & 7 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ default-members = ["crates/fleet-proto", "crates/fleet-scheduler", "crates/fleet
resolver = "2"

[workspace.package]
version = "0.1.1"
version = "0.1.2"
edition = "2021"
publish = false

Expand Down
7 changes: 2 additions & 5 deletions deploy/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@ services:
retries: 20

scheduler:
build:
context: ..
dockerfile: deploy/Dockerfile.scheduler
image: ${FLEET_SCHEDULER_IMAGE:-zkm-fleet-scheduler:local}
image: ${FLEET_SCHEDULER_IMAGE:?set FLEET_SCHEDULER_IMAGE in compose.env}
restart: unless-stopped
# The user that owns ${FLEET_ETC}: the secrets there are mode 600.
user: "${FLEET_UID:-10001}:${FLEET_GID:-10001}"
Expand All @@ -39,7 +36,7 @@ services:

mcp:
profiles: [mcp]
image: ${FLEET_SCHEDULER_IMAGE:-zkm-fleet-scheduler:local}
image: ${FLEET_SCHEDULER_IMAGE:?set FLEET_SCHEDULER_IMAGE in compose.env}
restart: unless-stopped
user: "${FLEET_UID:-10001}:${FLEET_GID:-10001}"
depends_on: [scheduler]
Expand Down
33 changes: 15 additions & 18 deletions deploy/fleet-ctl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# Upgrade, roll back and inspect Fleet: fleet-ctl.sh [-n] COMMAND
#
# status control services, health, nodes and apps (and the node, if configured)
# scheduler upgrade [TAG] load the release's scheduler image, recreate it with Docker Compose
# and install its fleetctl (default: latest release)
# scheduler rollback restore the image, fleetctl and compose files the last upgrade replaced
# scheduler upgrade [TAG] pull the release's scheduler image from GHCR, recreate it with Docker
# Compose and install its fleetctl (default: latest release)
# scheduler rollback restore the image setting, fleetctl and compose files the last upgrade replaced
# node upgrade [TAG] install a release's fleet-node, bridge and fleet-proof-app
# node upgrade FLEET_NODE BRIDGE_DIR [FLEET_PROOF_APP] the same from local files
# node rollback restore what the last node upgrade replaced
Expand Down Expand Up @@ -92,38 +92,35 @@ scheduler_healthy() {
for _ in $(seq 30); do curl -fs -o /dev/null http://127.0.0.1:9191/healthz && return 0; sleep 2; done
return 1
}
set_scheduler_image() { # compose.env names the image both services run
[ "$DRY" = 1 ] && { echo "+ set FLEET_SCHEDULER_IMAGE=$1 in $PREFIX/compose.env"; return; }
{ grep -v '^FLEET_SCHEDULER_IMAGE=' "$PREFIX/compose.env" || true; echo "FLEET_SCHEDULER_IMAGE=$1"; } > "$PREFIX/compose.env.new"
mv "$PREFIX/compose.env.new" "$PREFIX/compose.env"
}
# The scheduler and MCP run from the image in Docker Compose; only fleetctl is a host binary.
scheduler_upgrade() {
local source=${1:-$RELEASE_VERSION} b; b=$PREFIX/backup/control-$(stamp)
local stage; stage=$(mktemp -d)
prepare_release "$source" "$stage"
local image="$stage/zkm-fleet-scheduler-${RELEASE_VERSION_NUMBER}-linux-x86_64.tar.gz"
local base="https://github.com/$RELEASE_REPO/releases/download/$RELEASE_TAG"
if [ "$DRY" = 1 ]; then
echo "+ curl --fail --location --retry 3 --output $image $base/$(basename "$image")"
else
curl -fsSL --retry 3 -o "$image" "$base/$(basename "$image")" || die "release has no scheduler image"
curl -fsSL --retry 3 -o "$image.sha256" "$base/$(basename "$image").sha256" || die "failed to download scheduler image checksum"
sed 's# dist/# #g' "$image.sha256" > "$image.sha256.normalized"
(cd "$stage" && sha256sum -c "$(basename "$image").sha256.normalized") || die "scheduler image checksum mismatch"
[ -x "$RELEASE_ROOT/bin/fleetctl" ] || die "release package has no fleetctl"
fi
local image="ghcr.io/${RELEASE_REPO,,}-scheduler:${RELEASE_VERSION_NUMBER}"
[ "$DRY" = 1 ] || [ -x "$RELEASE_ROOT/bin/fleetctl" ] || die "release package has no fleetctl"
# Pull first so a missing image changes nothing.
run docker pull "$image"
run mkdir -p "$b"
run cp -a "$PREFIX/bin/fleetctl" "$PREFIX/deploy/docker-compose.yml" "$PREFIX/deploy/gateway.yaml.in" "$b/"
run docker tag zkm-fleet-scheduler:local zkm-fleet-scheduler:rollback
run docker load -i "$image"
run cp -a "$PREFIX/bin/fleetctl" "$PREFIX/deploy/docker-compose.yml" "$PREFIX/deploy/gateway.yaml.in" "$PREFIX/compose.env" "$b/"
run install -m 755 "$RELEASE_ROOT/bin/fleetctl" "$PREFIX/bin/fleetctl"
run cp "$RELEASE_ROOT/deploy/docker-compose.yml" "$RELEASE_ROOT/deploy/gateway.yaml.in" "$PREFIX/deploy/"
set_scheduler_image "$image"
# The scheduler finishes in-flight calls before it exits, so recreating can take ~90s.
compose up -d --no-deps --force-recreate $(control_services)
scheduler_healthy || die "scheduler is not healthy; run: fleet-ctl.sh scheduler rollback"
echo "control plane upgraded to $RELEASE_TAG; previous fleetctl and compose files in $b"
}
scheduler_rollback() {
local b; b=$(latest_backup control); [ -n "$b" ] || die "no backup in $PREFIX/backup"
run docker tag zkm-fleet-scheduler:rollback zkm-fleet-scheduler:local
run install -m 755 "$b/fleetctl" "$PREFIX/bin/fleetctl"
run cp "$b/docker-compose.yml" "$b/gateway.yaml.in" "$PREFIX/deploy/"
run cp "$b/compose.env" "$PREFIX/compose.env"
compose up -d --no-deps --force-recreate $(control_services)
scheduler_healthy || die "scheduler is not healthy after rollback"
echo "restored $b"
Expand Down
13 changes: 9 additions & 4 deletions deploy/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ fi
# Copy execution code only to execution machines. The control plane has no Ray dependency.
mkdir -p "$PREFIX/deploy"
if [ "$ROLE" = control ]; then
cp "$SRC"/deploy/{docker-compose.yml,gateway.yaml.in,Dockerfile.scheduler} "$PREFIX/deploy/"
cp "$SRC"/deploy/{docker-compose.yml,gateway.yaml.in} "$PREFIX/deploy/"
FLEETCTL="$PREFIX/bin/fleetctl"
[ -f "$ETC/cluster.token" ] || "$FLEETCTL" init token --out "$ETC/cluster.token" >/dev/null
[ -f "$ETC/pg_password" ] || (umask 077; openssl rand -hex 24 > "$ETC/pg_password")
Expand All @@ -81,6 +81,7 @@ EOF
mkdir -p "$PREFIX/data" "$HOME/.config/zkm-fleet" "$HOME/.local/bin"
ln -sf "$ETC/admin.token" "$HOME/.config/zkm-fleet/admin.token"
ln -sf "$FLEETCTL" "$HOME/.local/bin/fleetctl"
IMG=${FLEET_SCHEDULER_IMAGE:-zkm-fleet-scheduler:local}
cat > "$PREFIX/compose.env" <<EOF
FLEET_ETC=$ETC
FLEET_DATA=$PREFIX/data
Expand All @@ -90,11 +91,15 @@ FLEET_SUBNET=$SUBNET
FLEET_SCHEDULER_IP=$SCHED_IP
FLEET_MCP_ALLOWED_HOSTS=$MCP_AUTHORITY,localhost:8906,127.0.0.1:8906
COMPOSE_PROFILES=$([ "$MCP" = 1 ] && echo mcp || true)
FLEET_SCHEDULER_IMAGE=$IMG
EOF
IMG=${FLEET_SCHEDULER_IMAGE:-zkm-fleet-scheduler:local}
if ! docker image inspect "$IMG" >/dev/null 2>&1; then
[ -d "$SRC/crates" ] || die "load the scheduler image or install from a full checkout"
docker build -f "$SRC/deploy/Dockerfile.scheduler" -t "$IMG" "$SRC"
if [ -n "${FLEET_SCHEDULER_IMAGE:-}" ]; then
docker pull "$IMG" || die "cannot pull $IMG"
else # a source checkout builds its own image
[ -d "$SRC/crates" ] || die "set FLEET_SCHEDULER_IMAGE or install from a full checkout"
docker build -f "$SRC/deploy/Dockerfile.scheduler" -t "$IMG" "$SRC"
fi
fi
docker compose --env-file "$PREFIX/compose.env" -p zkm-fleet -f "$PREFIX/deploy/docker-compose.yml" up -d
for _ in $(seq 1 30); do curl -sf http://127.0.0.1:9191/healthz >/dev/null && break; sleep 2; done
Expand Down
13 changes: 4 additions & 9 deletions deploy/setup.sh
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/bin/bash
# Bootstrap Fleet from a published GitHub Release package.
#
# setup.sh --version v0.1.0 node --name gpu-1 \
# setup.sh --version v0.1.2 node --name gpu-1 \
# --scheduler https://fleet.example:8905 --cluster-token FILE
# setup.sh --version v0.1.0 control --public-url https://fleet.example:8905
# setup.sh --version v0.1.2 control --public-url https://fleet.example:8905
set -euo pipefail

REPO=${FLEET_RELEASE_REPO:-ProjectZKM/zkm-fleet}
Expand Down Expand Up @@ -68,14 +68,9 @@ PACKAGE_ROOT=$(find "$STAGE/root" -mindepth 1 -maxdepth 1 -type d | head -1)
[ -x "$PACKAGE_ROOT/deploy/install.sh" ] || { echo "setup.sh: package is missing deploy/install.sh" >&2; exit 1; }

if [ "$ROLE" = control ]; then
IMAGE="$STAGE/zkm-fleet-scheduler-${VERSION_NUMBER}-linux-x86_64.tar.gz"
IMAGE_CHECKSUM="$IMAGE.sha256"
curl -fsSL --retry 3 "$BASE/zkm-fleet-scheduler-${VERSION_NUMBER}-linux-x86_64.tar.gz" -o "$IMAGE"
curl -fsSL --retry 3 "$BASE/zkm-fleet-scheduler-${VERSION_NUMBER}-linux-x86_64.tar.gz.sha256" -o "$IMAGE_CHECKSUM"
sed 's# dist/# #g' "$IMAGE_CHECKSUM" > "$STAGE/image.sha256"
(cd "$STAGE" && sha256sum -c image.sha256) || { echo "setup.sh: scheduler image checksum mismatch" >&2; exit 1; }
command -v docker >/dev/null || { echo "setup.sh: docker is required for control installation" >&2; exit 1; }
docker load -i "$IMAGE" >/dev/null
export FLEET_SCHEDULER_IMAGE="ghcr.io/${REPO,,}-scheduler:${VERSION_NUMBER}"
docker pull "$FLEET_SCHEDULER_IMAGE" >/dev/null || { echo "setup.sh: cannot pull $FLEET_SCHEDULER_IMAGE" >&2; exit 1; }
fi

FORWARD=("${ARGS[@]:1}" "--bin" "$PACKAGE_ROOT/bin")
Expand Down
6 changes: 3 additions & 3 deletions docs/installation.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ checksums and runs `deploy/install.sh` with the packaged binaries.

```bash
curl -fsSL https://raw.githubusercontent.com/ProjectZKM/zkm-fleet/main/deploy/setup.sh | bash -s -- \
--version v0.1.0 control --public-url https://fleet.example:8905 \
--version v0.1.2 control --public-url https://fleet.example:8905 \
--tls-names stage,fleet.example --prefix "$HOME/app/fleet" --etc "$HOME/app/fleet/etc"
```

This loads the scheduler image and starts the scheduler, Postgres, the HTTPS
This pulls the release's scheduler image from GHCR and starts the scheduler, Postgres, the HTTPS
gateway and MCP with Docker Compose. Add `--no-mcp` to leave MCP out. The
installer creates the cluster token, a private TLS CA and the first
administrator token in `etc/admin.token`, and puts `fleetctl` on the PATH.
Expand All @@ -31,7 +31,7 @@ Copy the control host's `etc/cluster.token` to the node (mode 600), and its

```bash
curl -fsSL https://raw.githubusercontent.com/ProjectZKM/zkm-fleet/main/deploy/setup.sh | bash -s -- \
--version v0.1.0 node --name gpu-1 --scheduler https://fleet.example:8905 \
--version v0.1.2 node --name gpu-1 --scheduler https://fleet.example:8905 \
--cluster-token "$HOME/cluster.token" --scheduler-ca "$HOME/ca.pem" \
--prefix "$HOME/app/fleet" --etc "$HOME/app/fleet/etc" --cuda 0,1,2,3 --cpus 32
```
Expand Down
42 changes: 12 additions & 30 deletions docs/maintenance.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,34 +127,17 @@ redistribute scoped credentials before resuming clients.

### Control host

For the default Docker Compose installation, rebuild the image from the checkout
and recreate the scheduler. Set `FLEET_PREFIX` to the installed directory:
The scheduler and MCP run from the release's GHCR image
(`ghcr.io/projectzkm/zkm-fleet-scheduler:VERSION`) in Docker Compose; only
`fleetctl` is a host binary. Upgrade with:

```bash
FLEET_PREFIX="$HOME/app/fleet"
docker build -f deploy/Dockerfile.scheduler -t zkm-fleet-scheduler:local .
install -m 755 target/release/fleetctl "$FLEET_PREFIX/bin/fleetctl"
docker compose --env-file "$FLEET_PREFIX/compose.env" -p zkm-fleet \
-f "$FLEET_PREFIX/deploy/docker-compose.yml" up -d --no-deps --force-recreate scheduler
FLEET_PREFIX="$HOME/app/fleet" deploy/fleet-ctl.sh scheduler upgrade v0.1.2
```

If MCP is enabled, repeat the Compose command with `mcp` instead of `scheduler`.
Use the image tag already configured by the installation if it differs from
`zkm-fleet-scheduler:local`. Replace installed Compose/gateway templates when
their configuration changes; rebuilding an image alone does not replace them.

Existing native systemd installations keep their supervisor. For the deployed
control host, run from the build checkout and select the installed runtime path
explicitly; a separate worktree does not contain that installation:

```bash
FLEET_PREFIX="/data/stephen/zkm-fleet/.fleet"
FLEET_BIN_DIR="${CARGO_TARGET_DIR:-target}/release"
sudo systemctl stop zkm-fleet-mcp.service zkm-scheduler.service
install -m 755 "$FLEET_BIN_DIR/fleet-scheduler" "$FLEET_BIN_DIR/fleetctl" \
"$FLEET_BIN_DIR/fleet-mcp" "$FLEET_PREFIX/bin/"
sudo systemctl start zkm-scheduler.service zkm-fleet-mcp.service
```
It pulls the image, backs up `fleetctl`, the Compose files and `compose.env`,
records the new image in `compose.env`, recreates the services and waits for
`/healthz`. `scheduler rollback` restores that backup and recreates them.

Verify the gateway's `/healthz`, `fleetctl namespace` and `fleetctl node`.
When enabled, also check MCP initialization and an authenticated tool call.
Expand Down Expand Up @@ -197,19 +180,18 @@ before following [verify and resume](#verify-and-resume).
export FLEET_PREFIX=/path/to/control/install FLEET_NODE_SSH=gpu-box # or ~/.config/zkm-fleet/ctl.env
deploy/fleet-ctl.sh status # control services, health, nodes, apps and the node
deploy/fleet-ctl.sh -n scheduler upgrade # -n prints every step and changes nothing
deploy/fleet-ctl.sh scheduler upgrade v0.1.0 # download and verify the GitHub Release package
deploy/fleet-ctl.sh node upgrade v0.1.0 # download and verify the node package
deploy/fleet-ctl.sh scheduler upgrade target/release # legacy local scheduler binaries
deploy/fleet-ctl.sh node upgrade target/release/fleet-node bridge # legacy local node files over ssh
deploy/fleet-ctl.sh scheduler upgrade v0.1.2 # pull the GHCR image, install the release's fleetctl
deploy/fleet-ctl.sh node upgrade v0.1.2 # download and verify the node package
deploy/fleet-ctl.sh node upgrade target/release/fleet-node bridge # local node files over ssh
deploy/fleet-ctl.sh node upgrade target/release/fleet-node bridge target/release/fleet-proof-app
deploy/fleet-ctl.sh scheduler rollback # or node rollback: restore the last upgrade's backup
deploy/fleet-ctl.sh app cleanup SERVICE-0 # leftovers of a failed app start
```

An upgrade copies what it replaces to `backup/` first, then waits for the scheduler's
health check or the node's reconnection. Upgrade the scheduler and nodes together when a
release changes the protocol. The script uses `sudo systemctl` with the unit names in this
guide; set `FLEET_SCHEDULER_UNIT`, `FLEET_MCP_UNIT` or `FLEET_NODE_UNIT` for others.
release changes the protocol. The scheduler and MCP are managed with Docker Compose; the
node uses `sudo systemctl`, and `FLEET_NODE_UNIT` names a non-default unit.
A third `node upgrade` argument also installs `fleet-proof-app` and restarts the app
endpoint units listed in `FLEET_PROOF_APP_UNITS`.

Expand Down
Loading