webclaw/setup.sh

#!/usr/bin/env bash
# setup.sh — Local setup for noxa
#
# Checks prerequisites, builds binaries, configures .env,
# optionally installs Ollama, and wires up the MCP server.
#
# Usage:
#   ./setup.sh              # Interactive full setup
#   ./setup.sh --minimal    # Build only, skip configuration
#   ./setup.sh --check      # Check prerequisites without installing

set -euo pipefail

# ---------------------------------------------------------------------------
# Colors
# ---------------------------------------------------------------------------
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
DIM='\033[2m'
RESET='\033[0m'

info()    { printf "${BLUE}[*]${RESET} %s\n" "$*"; }
success() { printf "${GREEN}[+]${RESET} %s\n" "$*"; }
warn()    { printf "${YELLOW}[!]${RESET} %s\n" "$*"; }
error()   { printf "${RED}[x]${RESET} %s\n" "$*" >&2; }

prompt() {
    local var_name="$1" prompt_text="$2" default="${3:-}"
    if [[ -n "$default" ]]; then
        printf "${CYAN}    %s${DIM} [%s]${RESET}: " "$prompt_text" "$default"
    else
        printf "${CYAN}    %s${RESET}: " "$prompt_text"
    fi
    read -r input
    eval "$var_name=\"${input:-$default}\""
}

prompt_secret() {
    local var_name="$1" prompt_text="$2" default="${3:-}"
    if [[ -n "$default" ]]; then
        printf "${CYAN}    %s${DIM} [%s]${RESET}: " "$prompt_text" "$default"
    else
        printf "${CYAN}    %s${RESET}: " "$prompt_text"
    fi
    read -rs input
    echo
    eval "$var_name=\"${input:-$default}\""
}

prompt_yn() {
    local prompt_text="$1" default="${2:-y}"
    local hint="Y/n"
    [[ "$default" == "n" ]] && hint="y/N"
    printf "${CYAN}    %s${DIM} [%s]${RESET}: " "$prompt_text" "$hint"
    read -r input
    input="${input:-$default}"
    [[ "$input" =~ ^[Yy]$ ]]
}

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# ---------------------------------------------------------------------------
# Step 1: Check prerequisites
# ---------------------------------------------------------------------------
check_prerequisites() {
    echo
    printf "${BOLD}${GREEN}  Step 1: Prerequisites${RESET}\n"
    echo

    local all_good=true

    # Rust
    if command -v rustc &>/dev/null; then
        local rust_version
        rust_version=$(rustc --version | awk '{print $2}')
        success "Rust $rust_version"

        # Check minimum version (1.85 for edition 2024)
        local major minor
        major=$(echo "$rust_version" | cut -d. -f1)
        minor=$(echo "$rust_version" | cut -d. -f2)
        if [[ "$major" -lt 1 ]] || [[ "$major" -eq 1 && "$minor" -lt 85 ]]; then
            warn "Rust 1.85+ required (edition 2024). Run: rustup update"
            all_good=false
        fi
    else
        error "Rust not found. Install: https://rustup.rs"
        all_good=false
    fi

    # cargo
    if command -v cargo &>/dev/null; then
        success "Cargo $(cargo --version | awk '{print $2}')"
    else
        error "Cargo not found (should come with Rust)"
        all_good=false
    fi

    # Ollama (optional)
    if command -v ollama &>/dev/null; then
        success "Ollama installed"
        if curl -sf http://localhost:11434/api/tags &>/dev/null; then
            success "Ollama is running"
            local models
            models=$(curl -sf http://localhost:11434/api/tags | python3 -c "import sys,json; [print(m['name']) for m in json.load(sys.stdin).get('models',[])]" 2>/dev/null || echo "")
            if [[ -n "$models" ]]; then
                success "Models: $(echo "$models" | tr '\n' ', ' | sed 's/,$//')"
            else
                warn "No models pulled yet"
            fi
        else
            warn "Ollama installed but not running. Start with: ollama serve"
        fi
    else
        warn "Ollama not found (optional — needed for local LLM features)"
    fi

    # Git
    if command -v git &>/dev/null; then
        success "Git $(git --version | awk '{print $3}')"
    else
        error "Git not found"
        all_good=false
    fi

    echo
    if $all_good; then
        success "All prerequisites met."
    else
        error "Some prerequisites are missing. Fix them before continuing."
        [[ "${1:-}" == "--check" ]] && exit 1
    fi
}

# ---------------------------------------------------------------------------
# Step 2: Build
# ---------------------------------------------------------------------------
build_binaries() {
    echo
    printf "${BOLD}${GREEN}  Step 2: Build${RESET}\n"
    echo

    info "Building release binaries (this may take a few minutes on first build)..."
    cd "$SCRIPT_DIR"

    if cargo build --release 2>&1 | tail -5; then
        echo
        success "Built 3 binaries:"
        ls -lh target/release/noxa target/release/noxa-server target/release/noxa-mcp 2>/dev/null | \
            awk '{printf "    %-20s %s\n", $NF, $5}'
    else
        error "Build failed. Check the output above."
        exit 1
    fi
}

# ---------------------------------------------------------------------------
# Step 3: Configure .env
# ---------------------------------------------------------------------------
configure_env() {
    echo
    printf "${BOLD}${GREEN}  Step 3: Configuration${RESET}\n"
    echo

    if [[ -f "$SCRIPT_DIR/.env" ]]; then
        warn ".env already exists."
        if ! prompt_yn "Overwrite?"; then
            info "Keeping existing .env"
            return
        fi
    fi

    local ollama_model="qwen3:8b"
    local openai_key=""
    local anthropic_key=""
    local proxy_file=""
    local server_port="3000"
    local auth_key=""

    info "LLM configuration"
    prompt ollama_model "Ollama model (local)" "qwen3:8b"
    prompt_secret openai_key "OpenAI API key (optional, press enter to skip)" ""
    prompt_secret anthropic_key "Anthropic API key (optional, press enter to skip)" ""

    echo
    info "Proxy configuration"
    if [[ -f "$SCRIPT_DIR/proxies.txt" ]]; then
        local proxy_count
        proxy_count=$(grep -cv '^\s*#\|^\s*$' "$SCRIPT_DIR/proxies.txt" 2>/dev/null || echo "0")
        success "proxies.txt found with $proxy_count proxies (auto-loaded)"
    else
        info "To use proxies, create proxies.txt with one proxy per line:"
        printf "    ${DIM}Format: host:port:user:pass${RESET}\n"
        printf "    ${DIM}cp proxies.example.txt proxies.txt${RESET}\n"
    fi
    local proxy_file=""

    echo
    info "Server configuration"
    prompt server_port "REST API port" "3000"
    prompt auth_key "API auth key (press enter to auto-generate)" ""

    if [[ -z "$auth_key" ]]; then
        if command -v openssl &>/dev/null; then
            auth_key=$(openssl rand -hex 16)
        else
            auth_key=$(LC_ALL=C tr -dc 'a-f0-9' < /dev/urandom | head -c 32)
        fi
        info "Generated auth key: $auth_key"
    fi

    # Write .env
    cat > "$SCRIPT_DIR/.env" <<EOF
# noxa configuration — generated by setup.sh

# --- LLM Providers ---
OLLAMA_HOST=http://localhost:11434
OLLAMA_MODEL=$ollama_model
EOF

    if [[ -n "$openai_key" ]]; then
        echo "OPENAI_API_KEY=$openai_key" >> "$SCRIPT_DIR/.env"
    fi
    if [[ -n "$anthropic_key" ]]; then
        echo "ANTHROPIC_API_KEY=$anthropic_key" >> "$SCRIPT_DIR/.env"
    fi

    cat >> "$SCRIPT_DIR/.env" <<EOF

# --- Proxy ---
EOF
    if [[ -n "$proxy_file" ]]; then
        echo "NOXA_PROXY_FILE=$proxy_file" >> "$SCRIPT_DIR/.env"
    else
        echo "# NOXA_PROXY_FILE=/path/to/proxies.txt" >> "$SCRIPT_DIR/.env"
    fi

    cat >> "$SCRIPT_DIR/.env" <<EOF

# --- Server ---
NOXA_PORT=$server_port
NOXA_HOST=0.0.0.0
NOXA_AUTH_KEY=$auth_key

# --- Logging ---
NOXA_LOG=info
EOF

    echo
    success ".env created."
}

# ---------------------------------------------------------------------------
# Step 4: Install Ollama (optional)
# ---------------------------------------------------------------------------
setup_ollama() {
    echo
    printf "${BOLD}${GREEN}  Step 4: Ollama (Local LLM)${RESET}\n"
    echo

    if ! command -v ollama &>/dev/null; then
        info "Ollama is not installed."
        info "It's optional but needed for local LLM features (extract, summarize)."
        info "Without it, you can still use OpenAI/Anthropic APIs."
        echo
        if prompt_yn "Install Ollama?" "y"; then
            info "Installing Ollama..."
            if [[ "$(uname)" == "Darwin" ]]; then
                if command -v brew &>/dev/null; then
                    brew install ollama
                else
                    warn "Install Ollama manually: https://ollama.ai/download"
                    return
                fi
            else
                curl -fsSL https://ollama.ai/install.sh | sh
            fi
            success "Ollama installed."
        else
            info "Skipping Ollama. You can install later: https://ollama.ai"
            return
        fi
    fi

    # Check if running
    if ! curl -sf http://localhost:11434/api/tags &>/dev/null; then
        warn "Ollama is not running."
        if [[ "$(uname)" == "Darwin" ]]; then
            info "On macOS, open the Ollama app or run: ollama serve"
        else
            info "Start with: ollama serve"
        fi
        echo
        if prompt_yn "Start Ollama now?" "y"; then
            nohup ollama serve &>/dev/null &
            sleep 2
            if curl -sf http://localhost:11434/api/tags &>/dev/null; then
                success "Ollama is running."
            else
                warn "Ollama didn't start. Start it manually and re-run setup."
                return
            fi
        else
            return
        fi
    fi

    # Pull model
    local model
    model=$(grep '^OLLAMA_MODEL=' "$SCRIPT_DIR/.env" 2>/dev/null | cut -d= -f2 || echo "qwen3:8b")

    local has_model
    has_model=$(curl -sf http://localhost:11434/api/tags | python3 -c "import sys,json; models=[m['name'] for m in json.load(sys.stdin).get('models',[])]; print('yes' if any('$model' in m for m in models) else 'no')" 2>/dev/null || echo "no")

    if [[ "$has_model" == "yes" ]]; then
        success "Model $model already available."
    else
        info "Model $model not found locally."
        if prompt_yn "Pull $model now? (this downloads ~5GB)" "y"; then
            ollama pull "$model"
            success "Model $model ready."
        fi
    fi
}

# ---------------------------------------------------------------------------
# Step 5: Configure MCP server for Claude Desktop
# ---------------------------------------------------------------------------
setup_mcp() {
    echo
    printf "${BOLD}${GREEN}  Step 5: MCP Server (Claude Desktop integration)${RESET}\n"
    echo

    local mcp_binary="$SCRIPT_DIR/target/release/noxa-mcp"
    if [[ ! -f "$mcp_binary" ]]; then
        warn "noxa-mcp binary not found. Build first."
        return
    fi

    info "The MCP server lets Claude Desktop use noxa's tools directly."
    info "Tools: scrape, crawl, map, batch, extract, summarize, diff, brand"
    echo

    if ! prompt_yn "Configure MCP server for Claude Desktop?" "y"; then
        info "Skipping MCP setup."
        info "You can configure it later by adding to your Claude Desktop config:"
        printf '    {"mcpServers": {"noxa": {"command": "%s"}}}\n' "$mcp_binary"
        return
    fi

    # Find Claude Desktop config
    local config_path=""
    if [[ "$(uname)" == "Darwin" ]]; then
        config_path="$HOME/Library/Application Support/Claude/claude_desktop_config.json"
    else
        config_path="$HOME/.config/claude/claude_desktop_config.json"
    fi

    if [[ ! -f "$config_path" ]]; then
        # Create config directory and file
        mkdir -p "$(dirname "$config_path")"
        echo '{}' > "$config_path"
        info "Created Claude Desktop config at: $config_path"
    fi

    # Read existing config and merge
    local existing
    existing=$(cat "$config_path")

    # Check if noxa is already configured
    if echo "$existing" | python3 -c "import sys,json; c=json.load(sys.stdin); exit(0 if 'noxa' in c.get('mcpServers',{}) else 1)" 2>/dev/null; then
        warn "noxa MCP server already configured in Claude Desktop."
        if ! prompt_yn "Update the path?" "y"; then
            return
        fi
    fi

    # Merge noxa into mcpServers
    local updated
    updated=$(echo "$existing" | python3 -c "
import sys, json
config = json.load(sys.stdin)
if 'mcpServers' not in config:
    config['mcpServers'] = {}
config['mcpServers']['noxa'] = {
    'command': '$mcp_binary'
}
print(json.dumps(config, indent=2))
")

    echo "$updated" > "$config_path"
    success "MCP server configured in Claude Desktop."
    info "Restart Claude Desktop to activate."
}

# ---------------------------------------------------------------------------
# Step 6: Smoke test
# ---------------------------------------------------------------------------
smoke_test() {
    echo
    printf "${BOLD}${GREEN}  Step 6: Smoke Test${RESET}\n"
    echo

    local noxa="$SCRIPT_DIR/target/release/noxa"

    info "Testing extraction..."
    local output
    output=$("$noxa" https://example.com --format llm 2>/dev/null || echo "FAILED")

    if [[ "$output" == "FAILED" ]]; then
        warn "Extraction test failed. Check your network connection."
    else
        local word_count
        word_count=$(echo "$output" | wc -w | tr -d ' ')
        success "Extracted example.com: $word_count words"
    fi

    # Test Ollama if available
    if curl -sf http://localhost:11434/api/tags &>/dev/null; then
        info "Testing LLM summarization..."
        local summary
        summary=$("$noxa" https://example.com --summarize 2>/dev/null || echo "FAILED")
        if [[ "$summary" == "FAILED" ]]; then
            warn "LLM test failed. Check Ollama and model availability."
        else
            success "LLM summarization works."
        fi
    fi
}

# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------
print_summary() {
    local noxa="$SCRIPT_DIR/target/release/noxa"
    local server="$SCRIPT_DIR/target/release/noxa-server"
    local mcp="$SCRIPT_DIR/target/release/noxa-mcp"
    local port
    port=$(grep '^NOXA_PORT=' "$SCRIPT_DIR/.env" 2>/dev/null | cut -d= -f2 || echo "3000")

    echo
    printf "${BOLD}${GREEN}  Setup Complete${RESET}\n"
    echo
    printf "  ${BOLD}CLI:${RESET}\n"
    printf "    %s https://example.com --format llm\n" "$noxa"
    echo
    printf "  ${BOLD}REST API:${RESET}\n"
    printf "    %s\n" "$server"
    printf "    curl http://localhost:%s/health\n" "$port"
    echo
    printf "  ${BOLD}MCP Server:${RESET}\n"
    printf "    Configured in Claude Desktop (restart to activate)\n"
    echo
    printf "  ${BOLD}Config:${RESET}  %s/.env\n" "$SCRIPT_DIR"
    printf "  ${BOLD}Docs:${RESET}    %s/README.md\n" "$SCRIPT_DIR"
    echo
    printf "  ${DIM}Tip: Add to PATH for convenience:${RESET}\n"
    printf "    export PATH=\"%s/target/release:\$PATH\"\n" "$SCRIPT_DIR"
    echo
}

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
main() {
    echo
    printf "${BOLD}${GREEN}  noxa — Local Setup${RESET}\n"
    printf "${DIM}  Web extraction toolkit for AI agents${RESET}\n"
    echo

    local mode="${1:-}"

    if [[ "$mode" == "--check" ]]; then
        check_prerequisites "--check"
        exit 0
    fi

    check_prerequisites

    build_binaries

    if [[ "$mode" == "--minimal" ]]; then
        success "Minimal build complete. Run ./setup.sh for full configuration."
        exit 0
    fi

    configure_env
    setup_ollama
    setup_mcp
    smoke_test
    print_summary
}

main "$@"