mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-06-07 07:55:16 +02:00
feat: add csv upload functionality for OSS (#29)
feat: add csv upload functionality chore: remove redundant arq-worker from docker-compose
This commit is contained in:
Abhishek
GitHub
parent
2633ff0a2a
commit
3babb5ced6
26 changed files with 941 additions and 234 deletions
|
|
@ -4,18 +4,22 @@
|
|||
set -e # Exit on error
|
||||
|
||||
### CONFIGURATION #############################################################
|
||||
ENV_FILE="api/.env"
|
||||
RUN_DIR="run"
|
||||
BASE_LOG_DIR="/home/ubuntu/dograh/logs" # Base logs directory (same as start_services.sh)
|
||||
|
||||
# Determine BASE_DIR as parent of the scripts directory
|
||||
BASE_DIR="$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd)"
|
||||
|
||||
ENV_FILE="$BASE_DIR/api/.env"
|
||||
RUN_DIR="$BASE_DIR/run"
|
||||
BASE_LOG_DIR="$BASE_DIR/logs" # Base logs directory (same as start_services.sh)
|
||||
LATEST_LINK="$BASE_LOG_DIR/latest" # Symlink to latest logs (same as start_services.sh)
|
||||
VENV_PATH="/home/ubuntu/dograh/venv"
|
||||
VENV_PATH="$BASE_DIR/venv"
|
||||
HEALTH_CHECK_ENDPOINT="/api/v1/health" # Adjust as needed
|
||||
MAX_WAIT_SECONDS=310 # Max wait for graceful shutdown (5 minutes + 10 seconds grace)
|
||||
|
||||
# Load environment
|
||||
set -a && . "$ENV_FILE" && set +a
|
||||
|
||||
cd /home/ubuntu/dograh/app
|
||||
cd "$BASE_DIR"
|
||||
|
||||
### FUNCTIONS ##################################################################
|
||||
|
||||
|
|
@ -166,10 +170,13 @@ start_new_uvicorn_workers() {
|
|||
|
||||
log_info "Starting uvicorn with $FASTAPI_WORKERS workers on port $new_port"
|
||||
log_info "Logs: $LOG_FILE_PATH"
|
||||
|
||||
# Start in new process group with setsid (same as start_services.sh)
|
||||
# Each service gets its own LOG_FILE_PATH environment variable
|
||||
setsid nohup bash -c "LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS" >/dev/null 2>&1 &
|
||||
|
||||
# Start in background (same pattern as start_services.sh)
|
||||
(
|
||||
cd "$BASE_DIR"
|
||||
export LOG_FILE_PATH="$log_dir/uvicorn-rollover-${timestamp}-${script_pid}.log"
|
||||
exec uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS >>"$LOG_FILE_PATH" 2>&1
|
||||
) &
|
||||
|
||||
local new_pid=$!
|
||||
echo "$new_pid" > "$RUN_DIR/uvicorn_new.pid"
|
||||
|
|
|
|||
|
|
@ -1,70 +1,101 @@
|
|||
#!/usr/bin/env bash
|
||||
# start_services.sh
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
### CONFIGURATION #############################################################
|
||||
ENV_FILE="api/.env"
|
||||
RUN_DIR="run" # where we keep *.pid
|
||||
BASE_LOG_DIR="/home/ubuntu/dograh/logs" # base logs directory
|
||||
###############################################################################
|
||||
### CONFIGURATION
|
||||
###############################################################################
|
||||
|
||||
# Determine BASE_DIR as parent of the scripts directory
|
||||
BASE_DIR="$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd)"
|
||||
|
||||
ENV_FILE="$BASE_DIR/api/.env"
|
||||
RUN_DIR="$BASE_DIR/run" # Where we keep *.pid
|
||||
BASE_LOG_DIR="$BASE_DIR/logs" # Base logs directory
|
||||
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
LOG_DIR="$BASE_LOG_DIR/$TIMESTAMP" # timestamped log directory
|
||||
LATEST_LINK="$BASE_LOG_DIR/latest" # symlink to latest logs
|
||||
VENV_PATH="/home/ubuntu/dograh/venv"
|
||||
LOG_DIR="$BASE_LOG_DIR/$TIMESTAMP" # Timestamped log directory
|
||||
LATEST_LINK="$BASE_LOG_DIR/latest" # Symlink to latest logs
|
||||
VENV_PATH="$BASE_DIR/venv"
|
||||
|
||||
ARQ_WORKERS=${ARQ_WORKERS:-1}
|
||||
|
||||
# Log startup
|
||||
echo "Starting Dograh Services at $(date)"
|
||||
cd "$BASE_DIR"
|
||||
echo "Starting Dograh Services at $(date) in BASE_DIR: ${BASE_DIR}"
|
||||
|
||||
### 1) Load environment vars so that configurations like FASTAPI_WORKERS are loaded
|
||||
set -a && . "$ENV_FILE" && set +a
|
||||
###############################################################################
|
||||
### 1) Load environment variables
|
||||
###############################################################################
|
||||
|
||||
cd /home/ubuntu/dograh/app
|
||||
|
||||
if [[ -z "${FASTAPI_PORT:-}" ]]; then
|
||||
echo "Error: FASTAPI_PORT environment variable is not set."
|
||||
exit 1
|
||||
# Load environment from a file if it exists
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a && . "$ENV_FILE" && set +a
|
||||
fi
|
||||
|
||||
if [[ -z "${FASTAPI_WORKERS:-}" ]]; then
|
||||
echo "Error: FASTAPI_WORKERS environment variable is not set."
|
||||
exit 1
|
||||
fi
|
||||
FASTAPI_PORT=${FASTAPI_PORT:-8000}
|
||||
FASTAPI_WORKERS=${FASTAPI_WORKERS:-1}
|
||||
|
||||
# map "service name" → "command to run"
|
||||
declare -A SERVICES=(
|
||||
[ari_manager]="python -m api.services.telephony.ari_manager"
|
||||
[campaign_orchestrator]="python -m api.services.campaign.campaign_orchestrator"
|
||||
[uvicorn]="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --workers $FASTAPI_WORKERS"
|
||||
###############################################################################
|
||||
### 2) Define services
|
||||
###############################################################################
|
||||
|
||||
# Map "service name" → "command to run"
|
||||
# Using arrays for bash 3.2 compatibility
|
||||
SERVICE_NAMES=(
|
||||
"ari_manager"
|
||||
"campaign_orchestrator"
|
||||
"uvicorn"
|
||||
)
|
||||
|
||||
# Add ARQ workers dynamically based on ARQ_WORKERS environment variable
|
||||
SERVICE_COMMANDS=(
|
||||
"python -m api.services.telephony.ari_manager"
|
||||
"python -m api.services.campaign.campaign_orchestrator"
|
||||
"uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --workers $FASTAPI_WORKERS"
|
||||
)
|
||||
|
||||
# Add ARQ workers dynamically
|
||||
for ((i=1; i<=ARQ_WORKERS; i++)); do
|
||||
SERVICES[arq$i]="python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG"
|
||||
SERVICE_NAMES+=("arq$i")
|
||||
SERVICE_COMMANDS+=("python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG")
|
||||
done
|
||||
|
||||
### 2) Activate virtual environment #########################################
|
||||
source ${VENV_PATH}/bin/activate
|
||||
###############################################################################
|
||||
### 3) Activate virtual environment
|
||||
###############################################################################
|
||||
|
||||
if [[ -d "$VENV_PATH" && -f "$VENV_PATH/bin/activate" ]]; then
|
||||
source "$VENV_PATH/bin/activate"
|
||||
echo "Virtual environment activated: $VENV_PATH"
|
||||
else
|
||||
echo "Warning: Virtual environment not found at $VENV_PATH"
|
||||
echo "Continuing without virtual environment activation..."
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
### 4) Stop old services
|
||||
###############################################################################
|
||||
|
||||
### 3) Stop old services (only via PID files) #################################
|
||||
mkdir -p "$RUN_DIR"
|
||||
for name in "${!SERVICES[@]}"; do
|
||||
for name in "${SERVICE_NAMES[@]}"; do
|
||||
pidfile="$RUN_DIR/$name.pid"
|
||||
|
||||
if [[ -f $pidfile ]]; then
|
||||
oldpid=$(<"$pidfile")
|
||||
if kill -0 "$oldpid"; then
|
||||
|
||||
if kill -0 "$oldpid" 2>/dev/null; then
|
||||
echo "Stopping $name (PID $oldpid and its process group)…"
|
||||
|
||||
# Kill the entire process group (negative PID)
|
||||
# First try SIGTERM
|
||||
kill -TERM -"$oldpid" || kill -TERM "$oldpid" || true
|
||||
kill -TERM -"$oldpid" 2>/dev/null || kill -TERM "$oldpid" 2>/dev/null || true
|
||||
sleep 4
|
||||
# If still running, use SIGKILL
|
||||
if kill -0 "$oldpid"; then
|
||||
|
||||
if kill -0 "$oldpid" 2>/dev/null; then
|
||||
echo "⚠️ $name did not exit cleanly, forcing stop..."
|
||||
kill -KILL -"$oldpid" || kill -KILL "$oldpid" || true
|
||||
kill -KILL -"$oldpid" 2>/dev/null || kill -KILL "$oldpid" 2>/dev/null || true
|
||||
sleep 1
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -f "$pidfile"
|
||||
else
|
||||
echo "No PID file for $name, skipping stop."
|
||||
|
|
@ -74,14 +105,19 @@ done
|
|||
# Clean up any port tracking files for uvicorn
|
||||
rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid"
|
||||
|
||||
### 4) Run migrations #########################################################
|
||||
alembic -c api/alembic.ini upgrade head
|
||||
###############################################################################
|
||||
### 5) Run migrations
|
||||
###############################################################################
|
||||
|
||||
### 5) Prepare logs ###########################################################
|
||||
mkdir -p "$BASE_LOG_DIR"
|
||||
mkdir -p "$LOG_DIR"
|
||||
alembic -c "$BASE_DIR/api/alembic.ini" upgrade head
|
||||
|
||||
# Remove old symlink if it exists and create new one
|
||||
###############################################################################
|
||||
### 6) Prepare logs
|
||||
###############################################################################
|
||||
|
||||
mkdir -p "$BASE_LOG_DIR" "$LOG_DIR"
|
||||
|
||||
# Remove old symlink and create a new one
|
||||
if [[ -L "$LATEST_LINK" ]]; then
|
||||
rm "$LATEST_LINK"
|
||||
fi
|
||||
|
|
@ -90,33 +126,37 @@ ln -s "$TIMESTAMP" "$LATEST_LINK"
|
|||
echo "Log directory: $LOG_DIR"
|
||||
echo "Latest symlink: $LATEST_LINK -> $TIMESTAMP"
|
||||
|
||||
### 7) Start services #########################################################
|
||||
for name in "${!SERVICES[@]}"; do
|
||||
cmd=${SERVICES[$name]}
|
||||
###############################################################################
|
||||
### 7) Start services
|
||||
###############################################################################
|
||||
|
||||
for i in "${!SERVICE_NAMES[@]}"; do
|
||||
name="${SERVICE_NAMES[$i]}"
|
||||
cmd="${SERVICE_COMMANDS[$i]}"
|
||||
echo "→ Starting $name"
|
||||
|
||||
# Export LOG_FILE_PATH for this specific service
|
||||
export LOG_FILE_PATH="$LOG_DIR/$name.log"
|
||||
|
||||
# Start in new process group with setsid
|
||||
# Each service gets its own LOG_FILE_PATH environment variable
|
||||
setsid nohup bash -c "LOG_FILE_PATH='$LOG_DIR/$name.log' $cmd" >/dev/null 2>&1 &
|
||||
|
||||
# Get the PID of the setsid process
|
||||
|
||||
(
|
||||
cd "$BASE_DIR"
|
||||
export LOG_FILE_PATH="$LOG_DIR/$name.log"
|
||||
exec $cmd >>"$LOG_DIR/$name.log" 2>&1
|
||||
) &
|
||||
|
||||
pid=$!
|
||||
echo $pid >"$RUN_DIR/$name.pid"
|
||||
|
||||
# For uvicorn, also save the port for rolling updates
|
||||
echo " Started with PID $pid"
|
||||
|
||||
if [[ "$name" == "uvicorn" ]]; then
|
||||
echo "$FASTAPI_PORT" >"$RUN_DIR/uvicorn.port"
|
||||
fi
|
||||
done
|
||||
disown -a
|
||||
|
||||
### 8) Summary #################################################################
|
||||
###############################################################################
|
||||
### 8) Summary
|
||||
###############################################################################
|
||||
|
||||
echo
|
||||
echo "──────────────────────────────────────────────────"
|
||||
for name in "${!SERVICES[@]}"; do
|
||||
for name in "${SERVICE_NAMES[@]}"; do
|
||||
pid=$(<"$RUN_DIR/$name.pid")
|
||||
echo "✓ $name (PID $pid) → $LOG_DIR/$name.log"
|
||||
done
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue