#!/usr/bin/env bash
# view - compact view of the TSC worker (Aria Energy 1.7) (see view.awk for columns and colors).
# Standalone: depends only on ./tsc, .env, docker and nvidia-smi.
#   ./view         machine state, then the live log (Ctrl-C to quit).
#                  CHANGES NOTHING.
#   ./view 10m     state, then the last 10 minutes in chronological order
#   ./view check   state only: docker and its nvidia runtime, containers,
#                  VRAM per GPU and what holds it, disk space, the model
#                  (present, or downloading with progress and estimated
#                  end), shares seen by the pool.
#   ./view fix     state, REPAIR of what can be repaired, then the log
#                  (./view fix 10m: same, time window instead of live)
# Repair (see agir, only with `fix`): docker daemon stopped → started;
# engine dead inside running containers (the supervisor reports it and never
# restarts it) → ./tsc restart; containers missing → VRAM held by an orphan is
# freed by ./tsc stop, then ./tsc start. Nothing else is touched. Without
# `fix`, a worker stopped on purpose stays stopped.
# Color is used only when the output is a terminal, never when NO_COLOR is set.
#
# Shares: since worker 1.6 the proxy does not log one line per share sent
# ("Sent MINE_SHARE" is at debug level). Since 1.7 it logs every 60 s
# "Shares sent: N in last 60s (total M since start)": this is what the
# throughput line shows ("shares +N/min"). In addition, the pool is queried every
# 60 s (/api/miner?addr=<wallet>) and a POOL line shows what THE POOL counted for
# this worker: if the two differ, shares are lost between the worker and the pool.
# Why window mode sorts: without -f, `docker compose logs` prints the
# containers in blocks, not in chronological order.
cd "$(dirname "$0")" || exit 1
# Decimal point everywhere (awk printf follows the locale and would print a decimal comma otherwise).
export LC_NUMERIC=C
color=0; [ -t 1 ] && [ -z "${NO_COLOR:-}" ] && color=1
if [ "$color" = 1 ]; then G=$'\033[32m'; Y=$'\033[33m'; R=$'\033[31m'; D=$'\033[2m'; N=$'\033[0m'; else G=""; Y=""; R=""; D=""; N=""; fi

# The model: name and revision read from docker-compose.yml (this is what the backend
# loads), folder derived from the name following the Hugging Face cache layout
# ("Qwen/Qwen3-8B" → models/hub/models--Qwen--Qwen3-8B).
MODEL_NAME=$(grep -m1 -oE 'MODEL_NAME: *"[^"]+"' docker-compose.yml 2>/dev/null | sed -E 's/.*"([^"]+)"/\1/'); MODEL_NAME=${MODEL_NAME:-Qwen/Qwen3-8B}
MODEL_COMMIT=$(grep -m1 -oE 'MODEL_COMMIT: *"[^"]+"' docker-compose.yml 2>/dev/null | sed -E 's/.*"([^"]+)"/\1/')
MODEL_FILE=$(grep -m1 -oE 'MODEL_FILE: *"[^"]+"' docker-compose.yml 2>/dev/null | sed -E 's/.*"([^"]+)"/\1/'); MODEL_FILE=${MODEL_FILE:-/models/Qwen3-8B-Q8_0-full-v2.gguf}
MODEL_PATH="models/$(basename "$MODEL_FILE")"
# Reference for the progress bar: the weight size announced by the model
# itself (metadata.total_size in model.safetensors.index.json, downloaded
# first). Otherwise, the known size of the model file.
MODEL_BYTES=$(grep -m1 -oE 'MODEL_BYTES: *"[0-9]+"' docker-compose.yml 2>/dev/null | tr -dc '0-9'); MODEL_BYTES=${MODEL_BYTES:-8709518304}
# VRAM per GPU: Q8 weights (8.1 GiB) + q8 KV cache (≈ 78 KB per token × 512 tokens × slots) + 0.6 GiB of buffers.
SLOTS=$(grep -E "^LLAMA_PARALLEL=" .env 2>/dev/null | cut -d= -f2); SLOTS=${SLOTS:-192}
NEED_MIB=$(( 8306 + SLOTS * 512 * 78 / 1024 + 600 ))
WALLET=$(grep -E "^WALLET_ADDRESS=" .env 2>/dev/null | cut -d= -f2)
API=$(grep -E "^MODEL_API_URL=" .env 2>/dev/null | cut -d= -f2); API=${API%/}

# HTTP GET: curl if available, otherwise python3 (always present: view.awk and the installer need it).
http_get() { if command -v curl >/dev/null 2>&1; then curl -sS -m 6 "$1" 2>/dev/null; else python3 -c 'import sys,urllib.request as u; sys.stdout.write(u.urlopen(u.Request(sys.argv[1], headers={"User-Agent": "aria-energy-view"}), timeout=6).read().decode())' "$1" 2>/dev/null; fi; }
ok()   { printf "%s%-10s%s %s✔%s %s\n" "$D" "$1" "$N" "$G" "$N" "$2"; }
warn() { printf "%s%-10s%s %s▲%s %s\n" "$D" "$1" "$N" "$Y" "$N" "$2"; }
ko()   { printf "%s%-10s%s %s✘%s %s\n" "$D" "$1" "$N" "$R" "$N" "$2"; }
gio()  { awk -v b="$1" 'BEGIN{printf "%.1f GiB", b/1073741824}'; }
duree(){ local s=$1; if [ "$s" -lt 60 ]; then echo "${s} s"; elif [ "$s" -lt 3600 ]; then echo "$((s/60)) min $((s%60)) s"; else echo "$((s/3600)) h $(( (s%3600)/60 )) min"; fi; }

# Identifier of THIS worker at the pool: WORKER_NAME from .env (stable and
# readable: the machine name by default). The last "Connected to broker as
# worker …" line of the proxy log is authoritative.
proxies() { docker compose --env-file .env ps --format '{{.Name}}' 2>/dev/null | grep -E '^aria-energy-proxy[0-9]+-1$' | sort; }
worker_id() {   # all workers of this machine (one per GPU), separated by spaces
  local c ids=""
  for c in $(proxies); do
    w=$(docker logs "$c" 2>&1 | grep -ao "Connected to broker as worker [^ ]*" | tail -1 | awk '{print $NF}' | tr -d '\r')
    [ -n "$w" ] && ids="$ids $w"
  done
  echo $ids
}
# One line "POOLAPI hh:mm:ss shares/5min shares_per_s age_s worker": the time
# comes from here (mawk has no strftime). Nothing if the pool does not answer.
# age_s is "x" when the pool has no session left for this worker (it still counts
# its shares of the last 5 minutes but the worker is no longer connected).
pool_line() {
  local w=$1
  [ -n "$w" ] && [ -n "$WALLET" ] && [ -n "$API" ] || return 1
  http_get "$API/api/miner?addr=$WALLET" | python3 -c '
import json, sys, time
w = sys.argv[1]
h = time.strftime("%H:%M:%S", time.gmtime())
try:
    d = json.load(sys.stdin)
except Exception:
    sys.exit(1)
for x in d.get("workers") or []:
    if x.get("worker") == w:
        age = x.get("last_share_s")
        print("POOLAPI", h, x.get("shares", 0), round(float(x.get("hashrate") or 0), 2), "x" if age is None else int(abs(age)), w[:12])
        sys.exit(0)
print("POOLAPI", h, "- - -", w[:12], "unknown")
' "$w"
}

etat() {
  local O="" WB=""; if [ "$color" = 1 ]; then O=$'\033[1;38;2;255;77;0m'; WB=$'\033[1;38;2;237;232;219m'; fi
  echo "${D}== ${N}${O}⚡ ${WB}ARIA ENERGY 1.7${N}${D} · $(hostname) · TSC worker · $(date -u +%H:%M:%S) UTC · $(pwd)${N}"

  # --- docker and its nvidia runtime -----------------------------------------
  if ! v=$(docker info --format '{{.ServerVersion}}' 2>/dev/null); then
    ko docker "daemon unreachable (docker info): nothing can start"
  else
    if docker info --format '{{json .Runtimes}}' 2>/dev/null | grep -q '"nvidia"'; then
      ok docker "$v · nvidia runtime declared$(nvidia-container-cli --version 2>/dev/null | awk '/cli-version/{print " (container-cli " $2 ")"}')"
    else
      ko docker "$v · nvidia runtime MISSING from docker info: the backend will not see the GPUs"
    fi
    c=$(docker compose --env-file .env ps --format '{{.Service}} {{.Status}}' 2>/dev/null | tr '\n' ',' | sed 's/,$//; s/,/ · /g')
    if [ -n "$c" ]; then ok containers "$c"; else warn containers "none (worker stopped)"; fi
  fi

  # --- GPUs: VRAM and what holds it -------------------------------------------
  if ! command -v nvidia-smi >/dev/null 2>&1; then
    ko gpu "nvidia-smi not found"
  else
    apps=$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader,nounits 2>/dev/null)
    nvidia-smi --query-gpu=index,uuid,name,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits 2>/dev/null \
    | while IFS=, read -r i uuid name used total util; do
      i=${i// /}; uuid=${uuid// /}; used=${used// /}; total=${total// /}; util=${util// /}
      need=$NEED_MIB
      free=$((total - used))
      qui=$(printf '%s\n' "$apps" | awk -F', *' -v g="$uuid" '$1==g && $3 ~ /llama-server/ {printf "%s %s (pid %s, %s MiB) ", (n++?"·":""), $3, $2, $4}')
      # Is the engine holding the VRAM running inside a container? If not, it is an
      # orphan of an engine killed during a CUDA load: ./tsc stop kills it.
      hors=""
      for p in $(printf '%s\n' "$apps" | awk -F', *' -v g="$uuid" '$1==g && $3 ~ /llama-server/ {print $2}'); do
        grep -q docker "/proc/$p/cgroup" 2>/dev/null || hors="$hors $p"
      done
      ligne="$i $name  $used / $total MiB  $util%${qui:+  ← $qui}"
      if [ -n "$hors" ]; then
        ko gpu "$ligne  OUTSIDE container (pid$hors): VRAM held by an orphan → ./tsc stop frees it"
      elif [ -n "$qui" ]; then
        ok gpu "$ligne"
      elif [ "$free" -ge "$need" ]; then
        ok gpu "$ligne  free: $free MiB ≥ $need required ($SLOTS slots), ready"
      else
        ko gpu "$ligne  free: $free MiB < $need required: the engine will not load"
      fi
    done
  fi

  # --- disk and model --------------------------------------------------------
  avail_m=$(df -BM --output=avail models 2>/dev/null | tail -1 | tr -dc '0-9')
  # No models/ folder yet: the disk of the current folder will receive the model.
  [ -n "$avail_m" ] || avail_m=$(df -BM --output=avail . 2>/dev/null | tail -1 | tr -dc '0-9')
  if [ -s "$MODEL_PATH" ] && [ "$(stat -L -c %s "$MODEL_PATH" 2>/dev/null)" = "$MODEL_BYTES" ]; then
    ok model "$MODEL_NAME${MODEL_COMMIT:+ @${MODEL_COMMIT:0:8}} · $(basename "$MODEL_PATH") present ($(gio "$MODEL_BYTES")) · disk: $((avail_m/1024)) GiB free"
  elif [ -s "$MODEL_PATH" ]; then
    ko model "$(basename "$MODEL_PATH") has an unexpected size ($(gio "$(stat -L -c %s "$MODEL_PATH")") instead of $(gio "$MODEL_BYTES")): delete it, the next start downloads it again"
  elif [ -s "$MODEL_PATH.part" ]; then
    # Download in progress: two samples 4 s apart give the rate and the end time.
    s1=$(stat -c %s "$MODEL_PATH.part" 2>/dev/null); sleep 4; s2=$(stat -c %s "$MODEL_PATH.part" 2>/dev/null)
    rate=$(( (s2 - s1) / 4 ))
    debit=$(awk -v r="$rate" 'BEGIN{printf "%.0f MiB/s", r/1048576}')
    pct=$(awk -v s="$s2" -v t="$MODEL_BYTES" 'BEGIN{p=100*s/t; if(p>100)p=100; printf "%d", p}')
    bar=$(awk -v p="$pct" 'BEGIN{n=int(p/4); for(i=0;i<25;i++) printf (i<n?"#":"."); }')
    if [ "$rate" -gt 0 ]; then
      eta=$(( (MODEL_BYTES - s2) / rate )); [ "$eta" -lt 0 ] && eta=0
      warn model "$MODEL_NAME · downloading [$bar] $pct% · $(gio "$s2") / $(gio "$MODEL_BYTES") · $debit · ends in ~$(duree "$eta")"
    else
      warn model "$MODEL_NAME · incomplete [$bar] $pct% · $(gio "$s2") / $(gio "$MODEL_BYTES") · no bytes received in 4 s (download stalled? ./tsc start resumes it)"
    fi
    need_m=$(( (MODEL_BYTES - s2) / 1048576 + 512 ))
    if [ "${avail_m:-0}" -lt "$need_m" ]; then ko disk "$((avail_m/1024)) GiB free, $((need_m/1024)) more needed: the download will fail"; else ok disk "$((avail_m/1024)) GiB free ($((need_m/1024)) more needed)"; fi
  else
    need_m=$(( MODEL_BYTES / 1048576 + 512 ))
    if [ "${avail_m:-0}" -lt "$need_m" ]; then
      ko model "$(basename "$MODEL_PATH") missing, and $((avail_m/1024)) GiB free < $((need_m/1024)) GiB required: free up space before starting"
    else
      warn model "$(basename "$MODEL_PATH") missing: the first start downloads $(gio "$MODEL_BYTES") ($((avail_m/1024)) GiB free, enough)"
    fi
  fi

  # --- the pool --------------------------------------------------------------
  ws=$(worker_id)
  if [ -z "$ws" ]; then
    warn pool "worker identifier not found in the proxy logs (never connected?)"
  else
    for w in $ws; do
      if l=$(pool_line "$w"); then
        set -- $l
        if [ "$3" = "-" ]; then ko pool "worker $w unknown to the pool for this wallet"
        elif [ "$5" = "x" ]; then ko pool "worker $w · $3 shares / 5 min but NO SESSION LEFT at the pool (disconnected; the log says why)"
        elif [ "$5" -gt 300 ]; then ko pool "worker $w · $3 shares / 5 min · last one $(duree "$5") ago: nothing is arriving anymore"
        else ok pool "worker $w · $3 shares / 5 min · $4 shares/s · last one $5 s ago  ($API/api/miner)"; fi
      else
        ko pool "$API/api/miner does not answer"
      fi
    done
  fi
  echo
}

conteneurs_up() { docker compose --env-file .env ps -q --status=running 2>/dev/null | grep -q .; }
# The backend supervisor never restarts the engine: its last phase line says
# "llama-server process EXITED unexpectedly … worker is NOT mining" and the
# container stays up. Only this line reveals a dead engine.
moteur_mort()   { local c; for c in $(docker compose --env-file .env ps --format '{{.Name}}' 2>/dev/null | grep -E '^aria-energy-llama[0-9]+-1$'); do docker logs --tail 3000 "$c" 2>&1 | grep -a "\[PHASE\]" | tail -1 | grep -qE "EXITED|NOT mining" && return 0; done; return 1; }

agir() {
  if ! docker info >/dev/null 2>&1; then
    warn action "docker daemon stopped → sudo -n systemctl start docker"
    sudo -n systemctl start docker 2>/dev/null
    i=0; until docker info >/dev/null 2>&1 || [ $i -ge 20 ]; do sleep 1; i=$((i+1)); done
    if docker info >/dev/null 2>&1; then ok action "docker started"; else ko action "docker does not start: check manually (passwordless sudo?)"; return 1; fi
  fi
  if conteneurs_up; then
    if moteur_mort; then
      warn action "llama-server engine dead (last PHASE: EXITED) inside running containers → ./tsc restart"
      ./tsc restart >/dev/null 2>&1 && ok action "worker restarted (the model reloads: 1 to 3 min before the first shares)" || ko action "./tsc restart failed"
    else
      ok action "nothing to do"
    fi
  else
    if nvidia-smi --query-compute-apps=pid,process_name --format=csv,noheader 2>/dev/null | grep -q llama-server; then
      warn action "VRAM held without a container (orphan of a killed engine) → ./tsc stop frees it"
      ./tsc stop >/dev/null 2>&1
    fi
    warn action "worker stopped → ./tsc start"
    ./tsc start >/dev/null 2>&1 && ok action "worker started (the model loads: 1 to 3 min, 20 min to 1 h if it must be downloaded)" || ko action "./tsc start failed"
  fi
  echo
}

mode=${1:-}; fen=""
case "$mode" in
  etat|état|check) etat; exit 0 ;;
  repare|répare|fix) etat; agir; fen=${2:-} ;;
  "") etat ;;
  *) etat; fen=$mode ;;
esac
case "$fen" in
  "")
    # The live log, and every 60 s a POOLAPI line slipped into the stream so
    # that view.awk shows what the pool counted. Both write to the same
    # pipe; awk recognizes the line by its prefix.
    ws=$(worker_id)
    {
      docker compose --env-file .env logs -f --tail 0 2>&1 &
      pid=$!
      ( while sleep 60; do for w in $ws; do pool_line "$w" 2>/dev/null; done; done ) &
      pid2=$!
      # Both die together: otherwise the loop would outlive Ctrl-C until its
      # next write to a closed pipe.
      trap 'kill $pid $pid2 2>/dev/null' EXIT INT TERM
      wait $pid
    } | awk -v color=$color -f view.awk
    ;;
  *)
    docker compose --env-file .env logs -t --since "$fen" 2>&1 | sort -t"|" -k2 | awk -v color=$color -f view.awk
    ;;
esac
