#!/usr/bin/env bash
# Dev post-deploy LLM smoke check (M51-1): after the k3s deploy rolls out, prove that the
# freshly-restarted API pod can actually reach the OpenClaw po-gateway and get a 200 - not the
# silent 401 token drift that made every project chat narrate "briefly overloaded" while the
# deploy stayed green (issue #942).
#
# It runs bloom's exact outbound request from INSIDE the pod, so it exercises the same network
# path and the same BLOOM_OPENCLAW_* env the app itself uses: `kubectl exec` a tiny Node probe
# that POSTs /v1/chat/completions to $BLOOM_OPENCLAW_BASE_URL with the pod's own
# $BLOOM_OPENCLAW_API_KEY as `Authorization: Bearer`, mirroring OpenClawGatewayClient's request
# shape (model + messages + max_completion_tokens).
#
# Verdict matrix (#942, #989):
#   200           pass.
#   429           degraded pass: exit 0 with a ::warning:: annotation and a step summary. A
#                 provider rate limit proves the opposite of a drift - the pod reached the
#                 gateway, the gateway accepted the pod's token (a drifted token is a 401/403),
#                 and the gateway reached its provider with valid credentials. The gateway's
#                 whole model chain shares one Claude subscription, whose quota 429s carry a
#                 Retry-After of hours to days, so failing the deploy would paint a correct
#                 release red for days while a re-run only spends the same exhausted quota. The
#                 probe retries a 429 only while its Retry-After is short (knobs below).
#   401 / 403     fail, with the token-drift reconciliation guidance.
#   anything else fail (5xx overload; status=000 for unreachable/unconfigured).
#
# The token never leaves the pod and never reaches the log: the probe reads it from the pod env
# and prints only the upstream status and a bounded snippet of the gateway's RESPONSE body (its
# own error text, e.g. {"detail":"..."} - never the request).
#
# Requires: DEPLOY_HOST, DEPLOY_SSH_KEY (the dev deploy credentials, as helm-dev.sh). Optional:
# DEPLOY_USER (exedev), BLOOM_NAMESPACE (bloom), HELM_RELEASE (bloom), BLOOM_API_DEPLOYMENT
# (<release>-server, i.e. the chart's server Deployment), BLOOM_LLM_MODEL (openclaw/default),
# SMOKE_LLM_TIMEOUT_MS (30000), SMOKE_LLM_RETRIES (2: extra attempts after a 429),
# SMOKE_LLM_RETRY_MAX_WAIT_S (60: a 429 asking for a longer wait is not retried),
# SMOKE_LLM_RETRY_BACKOFF_S (5: first wait when a 429 carries no Retry-After; doubles per retry),
# SMOKE_LLM_SUMMARY_TZ (Asia/Manila: time zone of the quota-reset time in the step summary).
# Run from the repo root.
#
# Offline tests (stubbed ssh, plus a local fake gateway that exercises the probe's retry loop):
# scripts/deploy/tests/test_smoke_llm_dev.sh.
set -euo pipefail

namespace="${BLOOM_NAMESPACE:-bloom}"
release="${HELM_RELEASE:-bloom}"
# The Deployment the probe execs into. Defaults to the chart's server Deployment
# (bloom.fullname + "-server" = "bloom-server" for the dev release); the retired Python "-api"
# Deployment is gone (#949). Overridable for a non-default release name or a future rename.
deployment="${BLOOM_API_DEPLOYMENT:-${release}-server}"
model="${BLOOM_LLM_MODEL:-openclaw/default}"
timeout_ms="${SMOKE_LLM_TIMEOUT_MS:-30000}"
retries="${SMOKE_LLM_RETRIES:-2}"
retry_max_wait_s="${SMOKE_LLM_RETRY_MAX_WAIT_S:-60}"
retry_backoff_s="${SMOKE_LLM_RETRY_BACKOFF_S:-5}"

# shellcheck source=scripts/deploy/sshlib.sh
source "$(dirname "${BASH_SOURCE[0]}")/sshlib.sh"
deploy_ssh_setup

# The in-pod probe. Reads config from the pod env (so it uses the exact token the app uses),
# prints `status=<code>`, `attempts=<n>`, `retry_after_s=<s>` / `rate_limited_models=<a,b>` when
# the gateway reported them, and `body=<bounded snippet>`, and exits non-zero on anything but
# 200. Node ships in the API image (the pod runs `node dist/index.cjs`), and Node 22's global
# fetch mirrors createFetchGatewayTransport.
read -r -d '' probe <<'NODE' || true
const base = (process.env.BLOOM_OPENCLAW_BASE_URL || "").replace(/\/+$/, "");
const key = process.env.BLOOM_OPENCLAW_API_KEY;
const model = process.env.SMOKE_LLM_MODEL || "openclaw/default";
const timeoutMs = Number(process.env.SMOKE_LLM_TIMEOUT_MS || "30000");
const retries = Number(process.env.SMOKE_LLM_RETRIES || "2");
const retryMaxWaitS = Number(process.env.SMOKE_LLM_RETRY_MAX_WAIT_S || "60");
const backoffS = Number(process.env.SMOKE_LLM_RETRY_BACKOFF_S || "5");
if (!base) {
  console.log("status=000");
  console.log("body=BLOOM_OPENCLAW_BASE_URL is not set in the pod - openclaw provider unconfigured");
  process.exit(1);
}
const headers = { "content-type": "application/json" };
if (key) headers["authorization"] = "Bearer " + key;
const body = JSON.stringify({
  model,
  messages: [{ role: "user", content: "ping" }],
  max_completion_tokens: 1,
});
// How long the provider asked us to wait. The gateway's OpenAI-compatible error writer sets no
// Retry-After header on its 429 passthrough; it embeds the upstream value in the failover
// message ("...; Retry-After: 226979 seconds"). Honor a header if one ever appears, else parse
// the text (the largest value wins when every model in the chain reports one).
const retryAfterSeconds = (res, text) => {
  const header = res.headers.get("retry-after");
  if (header) {
    const seconds = Number(header);
    if (Number.isFinite(seconds)) return Math.max(0, Math.round(seconds));
    const at = Date.parse(header);
    if (!Number.isNaN(at)) return Math.max(0, Math.round((at - Date.now()) / 1000));
  }
  const found = [...text.matchAll(/Retry-After:\s*(\d+)\s*seconds/gi)].map((m) => Number(m[1]));
  return found.length ? Math.max(...found) : null;
};
const sleep = (seconds) => new Promise((resolve) => setTimeout(resolve, seconds * 1000));
const attempt = async () => {
  const res = await fetch(base + "/v1/chat/completions", {
    method: "POST",
    headers,
    body,
    signal: AbortSignal.timeout(timeoutMs),
  });
  const text = await res.text();
  return { status: res.status, text, retryAfter: retryAfterSeconds(res, text) };
};
(async () => {
  let last;
  let attempts = 0;
  for (;;) {
    attempts += 1;
    last = await attempt();
    if (last.status !== 429 || attempts > retries) break;
    const waitS = last.retryAfter ?? backoffS * 2 ** (attempts - 1);
    if (waitS > retryMaxWaitS) {
      console.log("retry=skipped wait_s=" + waitS);
      break;
    }
    console.log("retry=" + attempts + " wait_s=" + waitS);
    await sleep(waitS);
  }
  console.log("status=" + last.status);
  console.log("attempts=" + attempts);
  if (last.retryAfter != null) console.log("retry_after_s=" + last.retryAfter);
  // The models the gateway's failover message reports as rate-limited ("<provider>/<model>:
  // HTTP 429"), so the summary can name them.
  const limited = [...last.text.matchAll(/([\w.-]+\/[\w.:-]+): HTTP 429/g)].map((m) => m[1]);
  if (limited.length) console.log("rate_limited_models=" + limited.join(","));
  console.log("body=" + last.text.slice(0, 200).replace(/\s+/g, " "));
  process.exit(last.status === 200 ? 0 : 1);
})().catch((e) => {
  console.log("status=000");
  console.log("body=network error: " + (e && e.name ? e.name : "Error"));
  process.exit(1);
});
NODE

# SMOKE_LLM_* are injected into the pod process via `env` (kubectl exec does not forward the
# runner's environment); KUBECONFIG is for kubectl itself on the VM.
remote="KUBECONFIG=\"\$HOME/.kube/config\" \
  kubectl -n '$namespace' exec -i 'deploy/${deployment}' -- \
  env SMOKE_LLM_MODEL='$model' SMOKE_LLM_TIMEOUT_MS='$timeout_ms' \
  SMOKE_LLM_RETRIES='$retries' SMOKE_LLM_RETRY_MAX_WAIT_S='$retry_max_wait_s' \
  SMOKE_LLM_RETRY_BACKOFF_S='$retry_backoff_s' node -"

echo "==> smoke (llm, dev): exercising OpenClaw /v1/chat/completions from inside ${deployment} (ns=$namespace)"
output="$(printf '%s\n' "$probe" | "${ssh_cmd[@]}" "$remote" 2>&1)" || true

probe_field() { printf '%s\n' "$output" | sed -n "s/^$1=//p" | tail -1; }
status="$(probe_field status)"
body="$(probe_field body)"
attempts="$(probe_field attempts)"
retry_after="$(probe_field retry_after_s)"
limited_models="$(probe_field rate_limited_models)"
: "${status:=000}"
case "$attempts" in *[!0-9]* | '') attempts=1 ;; esac
case "$retry_after" in *[!0-9]*) retry_after="" ;; esac

summary() { [ -z "${GITHUB_STEP_SUMMARY:-}" ] || echo "$1" >> "$GITHUB_STEP_SUMMARY"; }
summary "## Smoke (LLM, dev): OpenClaw /v1/chat/completions"
summary ""

attempts_note=""
[ "$attempts" -le 1 ] || attempts_note=" after $attempts attempts"

if [ "$status" = "200" ]; then
  echo "  ok: OpenClaw gateway returned 200 from inside the pod${attempts_note}"
  summary "✅ The API pod reached the OpenClaw po-gateway and got **200**${attempts_note}."
  echo "==> smoke (llm, dev): passed"
  exit 0
fi

if [ "$status" = "429" ]; then
  # A provider rate limit (#989): the gateway is up and accepted the pod's token, which is what
  # this check exists to prove. Pass, but loudly - dev chat is degraded until the quota resets.
  reset_note="the provider did not say when its quota resets"
  if [ -n "$retry_after" ]; then
    if [ "$retry_after" -ge 3600 ]; then
      wait_note="about $(((retry_after + 1800) / 3600)) h"
    else
      wait_note="about $(((retry_after + 30) / 60)) min"
    fi
    reset_at="$(TZ="${SMOKE_LLM_SUMMARY_TZ:-Asia/Manila}" \
      date -d "@$(($(date +%s) + retry_after))" '+%Y-%m-%d %H:%M %Z' 2>/dev/null || true)"
    reset_note="the quota resets in ${wait_note} (Retry-After ${retry_after} s${reset_at:+, around $reset_at})"
  fi
  models_note="${limited_models:+ (${limited_models//,/, })}"
  echo "::warning::smoke (llm, dev): OpenClaw /v1/chat/completions returned HTTP 429 - provider rate limit${models_note}. The gateway is up and accepted the pod's token, so the deploy stands, but dev chat is degraded until ${reset_note}."
  [ -z "$body" ] || echo "  gateway body snippet: $body"
  summary "### ⚠️ OpenClaw chat completions returned HTTP \`429\` (provider rate limit) - deploy accepted"
  summary ""
  summary "The API pod reached the po-gateway and the gateway **accepted its token** (a drifted token"
  summary "is a 401/403, #942), but every model in the gateway's chain is rate-limited by its"
  summary "provider${models_note}; ${reset_note}. Dev project chat is degraded until then."
  summary "Re-running the deploy cannot help - it spends the same exhausted quota. If this recurs,"
  summary "give the po-gateway a fallback model on a different account or provider (#989)."
  if [ -n "$body" ]; then
    summary ""
    summary "Gateway response snippet: \`$body\`"
  fi
  echo "==> smoke (llm, dev): passed with a warning (provider rate-limited${attempts_note})"
  exit 0
fi

echo "::error::smoke (llm, dev): OpenClaw /v1/chat/completions returned HTTP $status (expected 200)" >&2
[ -z "$body" ] || echo "  gateway body snippet: $body" >&2
summary "### ❌ OpenClaw chat completions returned HTTP \`$status\` (expected 200)"
summary ""
if [ "$status" = "401" ] || [ "$status" = "403" ]; then
  summary "A **$status** is a po-gateway auth failure - the pod's \`BLOOM_OPENCLAW_API_KEY\` no"
  summary "longer matches the gateway's \`BLOOM_PO_GATEWAY_TOKEN\` (token drift, #942). Reconcile"
  summary "them from the single source with \`scripts/deploy/sync-po-gateway-token.sh\` (runbook:"
  summary "docs/deployment.md, \"OpenClaw po-gateway token\"), then re-run the deploy."
else
  summary "The pod could not get a 200 from the OpenClaw gateway. Check the gateway is up and that"
  summary "\`BLOOM_OPENCLAW_BASE_URL\` points at it; see docs/deployment.md, \"OpenClaw po-gateway token\"."
fi
if [ -n "$body" ]; then
  summary ""
  summary "Gateway response snippet: \`$body\`"
fi
exit 1