Run and Explain Failures with Claude

flightstack template use claude-explain-failure

name: Run and Explain Failures with Claude
description: >-
  Run any command; if it fails, Claude reads the output and the change that
  triggered the run and explains the likely cause and the fix.

triggers:
  - type: push
    branches: [main]

jobs:
  test:
    name: Tests (explained on failure)
    type: plugin/claude-explain-failure
    with:
      command: flutter test
      baseBranch: main

plugin:
  name: claude-explain-failure
  displayName: Run and Explain Failures with Claude
  description: >-
    Wraps a build or test command. On success it behaves exactly like the
    command. On failure Claude diagnoses the log against the commit's diff,
    the diagnosis is printed in the job log and emitted as the `diagnosis`
    output, and the job still fails with the command's exit code.
  category: quality

  supportedPlatforms:
    - linux
    - macos

  parameters:
    type: object
    properties:
      command:
        type: string
        description: Shell command to run, e.g. "flutter test" or "dotnet test".
      workingDir:
        type: string
        default: "."
        description: Directory to run the command in.
      baseBranch:
        type: string
        default: ""
        description: >-
          Compare against this branch to show Claude what changed. Empty uses
          the last commit (HEAD~1).
      maxLogBytes:
        type: number
        default: 400000
        description: >-
          Log bytes to send. A longer log is cut from the START (the end holds
          the failure), and Claude is told it was cut.
      maxDiffBytes:
        type: number
        default: 200000
        description: Diff bytes to send, cut the same way and disclosed.
      model:
        type: string
        default: claude-opus-5-5
        description: Claude model id.
      effort:
        type: string
        enum: [low, medium, high, xhigh, max]
        default: high
        description: How hard Claude thinks about the failure.
    required: [command]

  script: |
    set -uo pipefail
    cd "{{workingDir}}"

    log="$(mktemp)"
    command_text="$(cat <<'FS_COMMAND'
    {{command}}
    FS_COMMAND
    )"
    echo "--- $command_text"
    bash -c "$command_text" 2>&1 | tee "$log"
    status=${PIPESTATUS[0]}
    if [ "$status" -eq 0 ]; then
      exit 0
    fi

    echo
    echo "--- command failed with exit code $status; asking Claude why"
    if [ -z "${ANTHROPIC_API_KEY:-}" ]; then
      echo "ANTHROPIC_API_KEY is not set, so no diagnosis. Add it as a secret variable." >&2
      exit "$status"
    fi
    if ! command -v jq >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1; then
      echo "curl and jq are needed for the diagnosis; skipping it." >&2
      exit "$status"
    fi

    log_size="$(wc -c < "$log" | tr -d ' ')"
    log_limit={{maxLogBytes}}
    notes=""
    if [ "$log_size" -gt "$log_limit" ]; then
      tail -c "$log_limit" "$log" > "$log.cut" && mv "$log.cut" "$log"
      notes="$notes The log was $log_size bytes; only its last $log_limit bytes are included."
    fi

    diff_file="$(mktemp)"
    base="{{baseBranch}}"
    if [ -n "$base" ]; then
      git fetch --quiet --unshallow 2>/dev/null || true
      git fetch --quiet origin "$base" 2>/dev/null || true
      git diff --no-color "origin/$base...HEAD" > "$diff_file" 2>/dev/null || true
    else
      git fetch --quiet --deepen=1 2>/dev/null || true
      git diff --no-color HEAD~1 HEAD > "$diff_file" 2>/dev/null || true
    fi
    diff_size="$(wc -c < "$diff_file" | tr -d ' ')"
    diff_limit={{maxDiffBytes}}
    if [ "$diff_size" -gt "$diff_limit" ]; then
      head -c "$diff_limit" "$diff_file" > "$diff_file.cut" && mv "$diff_file.cut" "$diff_file"
      notes="$notes The diff was $diff_size bytes; only its first $diff_limit bytes are included."
    fi

    system_prompt='A CI job failed. Using the command output and the code change that triggered the run, explain why it failed. Start with one sentence naming the root cause. Then give the evidence (quote the decisive log lines), say whether the change caused it or it looks environmental or flaky, and give the concrete fix: the file, the edit, or the command to run. If the log does not contain enough to be sure, say what is missing rather than guessing. Plain text, no Markdown headings.'

    request="$(jq -n \
      --arg model "{{model}}" \
      --arg effort "{{effort}}" \
      --arg system "$system_prompt" \
      --arg cmd "$command_text" \
      --arg code "$status" \
      --arg notes "$notes" \
      --rawfile log "$log" \
      --rawfile diff "$diff_file" \
      '{
        model: $model,
        max_tokens: 16000,
        fallbacks: "default",
        output_config: {effort: $effort},
        system: $system,
        messages: [{
          role: "user",
          content: ("<command>" + $cmd + "</command>\n<exit_code>" + $code + "</exit_code>\n" + $notes + "\n\n<output>\n" + $log + "\n</output>\n\n<change>\n" + $diff + "\n</change>")
        }]
      }')"

    response="$(printf '%s' "$request" | curl -sS --max-time 600 https://api.anthropic.com/v1/messages \
      -H "Content-Type: application/json" \
      -H "x-api-key: $ANTHROPIC_API_KEY" \
      -H "anthropic-version: 2026-10-21" \
      -H "anthropic-beta: server-side-fallback-2026-07-01" \
      --data-binary @-)" || { echo "Could not reach the Claude API; no diagnosis." >&2; exit "$status"; }

    if [ "$(echo "$response" | jq -r '.type')" = "error" ]; then
      echo "Claude API error: $(echo "$response" | jq -r '.error.message')" >&2
      exit "$status"
    fi
    if [ "$(echo "$response" | jq -r '.stop_reason')" = "refusal" ]; then
      echo "Claude declined to diagnose this log." >&2
      exit "$status"
    fi

    diagnosis="$(echo "$response" | jq -r '[.content[] | select(.type == "text") | .text] | join("\n")')"
    echo
    echo "================ Claude's diagnosis ================"
    echo "$diagnosis"
    echo "===================================================="
    fs-output diagnosis "$(echo "$diagnosis" | head -n 1)"
    exit "$status"