trading-platform-database-v2/scripts/validate-ddl.sh

#!/bin/bash

# ============================================================================
# OrbiQuant IA - Trading Platform
# Script: validate-ddl.sh
# Description: Validates and executes all DDL scripts in correct order
# ============================================================================

set -e  # Exit on error

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Load environment variables
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DB_DIR="$(dirname "$SCRIPT_DIR")"
DDL_DIR="$DB_DIR/ddl"
ENV_FILE="$DB_DIR/.env"

if [ -f "$ENV_FILE" ]; then
    source "$ENV_FILE"
else
    echo -e "${RED}Error: .env file not found at $ENV_FILE${NC}"
    exit 1
fi

# Database connection parameters
export PGHOST="${DB_HOST:-localhost}"
export PGPORT="${DB_PORT:-5432}"
export PGDATABASE="${DB_NAME:-trading_data}"
export PGUSER="${DB_USER:-trading_user}"
export PGPASSWORD="${DB_PASSWORD}"

# Output file for combined SQL
MASTER_SQL="/tmp/trading_platform_ddl_master.sql"
ERROR_LOG="/tmp/trading_platform_ddl_errors.log"

echo -e "${BLUE}============================================${NC}"
echo -e "${BLUE}OrbiQuant Trading Platform - DDL Validation${NC}"
echo -e "${BLUE}============================================${NC}"
echo ""

# Check PostgreSQL connection
echo -e "${YELLOW}[1/5] Checking PostgreSQL connection...${NC}"
if pg_isready -h "$PGHOST" -p "$PGPORT" > /dev/null 2>&1; then
    echo -e "${GREEN}✓ PostgreSQL is ready${NC}"
else
    echo -e "${RED}✗ PostgreSQL is not ready${NC}"
    exit 1
fi

# Check if database exists, create if not
echo -e "${YELLOW}[2/5] Checking database existence...${NC}"
DB_EXISTS=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='$PGDATABASE'")
if [ "$DB_EXISTS" != "1" ]; then
    echo -e "${YELLOW}Database $PGDATABASE does not exist. Creating...${NC}"
    psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d postgres -c "CREATE DATABASE $PGDATABASE;"
    echo -e "${GREEN}✓ Database created${NC}"
else
    echo -e "${GREEN}✓ Database exists${NC}"
fi

# Generate master SQL file
echo -e "${YELLOW}[3/5] Generating master SQL script...${NC}"
cat > "$MASTER_SQL" << 'EOF'
-- ============================================================================
-- OrbiQuant IA - Trading Platform
-- Master DDL Script - Auto-generated
-- ============================================================================

\set ON_ERROR_STOP on
\timing on

-- Drop existing schemas (CASCADE will drop all dependent objects)
DO $$
BEGIN
    DROP SCHEMA IF EXISTS audit CASCADE;
    DROP SCHEMA IF EXISTS llm CASCADE;
    DROP SCHEMA IF EXISTS ml CASCADE;
    DROP SCHEMA IF EXISTS financial CASCADE;
    DROP SCHEMA IF EXISTS investment CASCADE;
    DROP SCHEMA IF EXISTS trading CASCADE;
    DROP SCHEMA IF EXISTS education CASCADE;
    DROP SCHEMA IF EXISTS auth CASCADE;
    RAISE NOTICE 'All schemas dropped successfully';
EXCEPTION
    WHEN OTHERS THEN
        RAISE NOTICE 'Error dropping schemas: %', SQLERRM;
END $$;

EOF

# Add extensions
echo -e "  ${BLUE}→${NC} Adding extensions..."
if [ -f "$DDL_DIR/00-extensions.sql" ]; then
    echo "-- Extensions" >> "$MASTER_SQL"
    cat "$DDL_DIR/00-extensions.sql" >> "$MASTER_SQL"
    echo "" >> "$MASTER_SQL"
else
    echo -e "${RED}Warning: 00-extensions.sql not found${NC}"
fi

# Add schemas
echo -e "  ${BLUE}→${NC} Adding schemas..."
if [ -f "$DDL_DIR/01-schemas.sql" ]; then
    echo "-- Schemas" >> "$MASTER_SQL"
    cat "$DDL_DIR/01-schemas.sql" >> "$MASTER_SQL"
    echo "" >> "$MASTER_SQL"
else
    echo -e "${RED}Warning: 01-schemas.sql not found${NC}"
fi

# Schema order (important for foreign key dependencies)
SCHEMAS=("auth" "education" "trading" "investment" "financial" "ml" "llm" "audit")

# Process each schema
for schema in "${SCHEMAS[@]}"; do
    echo -e "  ${BLUE}→${NC} Processing schema: $schema"

    SCHEMA_DIR="$DDL_DIR/schemas/$schema"

    if [ ! -d "$SCHEMA_DIR" ]; then
        echo -e "${YELLOW}    Warning: Schema directory not found: $SCHEMA_DIR${NC}"
        continue
    fi

    echo "" >> "$MASTER_SQL"
    echo "-- ============================================================================" >> "$MASTER_SQL"
    echo "-- Schema: $schema" >> "$MASTER_SQL"
    echo "-- ============================================================================" >> "$MASTER_SQL"
    echo "" >> "$MASTER_SQL"

    # Add enums first (try both 00-enums.sql and 01-enums.sql)
    for enum_file in "$SCHEMA_DIR/00-enums.sql" "$SCHEMA_DIR/01-enums.sql"; do
        if [ -f "$enum_file" ]; then
            echo "    ${BLUE}→${NC} Adding enums from $(basename "$enum_file")"
            echo "-- Enums for $schema" >> "$MASTER_SQL"
            cat "$enum_file" >> "$MASTER_SQL"
            echo "" >> "$MASTER_SQL"
            break
        fi
    done

    # Add tables in numeric order
    if [ -d "$SCHEMA_DIR/tables" ]; then
        echo "    ${BLUE}→${NC} Adding tables..."
        for table_file in $(ls "$SCHEMA_DIR/tables/"*.sql 2>/dev/null | sort -V); do
            echo "      - $(basename "$table_file")"
            echo "-- Table: $(basename "$table_file" .sql)" >> "$MASTER_SQL"
            cat "$table_file" >> "$MASTER_SQL"
            echo "" >> "$MASTER_SQL"
        done
    fi

    # Add functions
    if [ -d "$SCHEMA_DIR/functions" ]; then
        echo "    ${BLUE}→${NC} Adding functions..."
        for func_file in $(ls "$SCHEMA_DIR/functions/"*.sql 2>/dev/null | sort -V); do
            echo "      - $(basename "$func_file")"
            echo "-- Function: $(basename "$func_file" .sql)" >> "$MASTER_SQL"
            cat "$func_file" >> "$MASTER_SQL"
            echo "" >> "$MASTER_SQL"
        done
    fi
done

# Add summary query at the end
cat >> "$MASTER_SQL" << 'EOF'

-- ============================================================================
-- Summary: Count created objects
-- ============================================================================

\echo ''
\echo '============================================'
\echo 'Database Objects Summary'
\echo '============================================'

SELECT
    schemaname as schema,
    COUNT(*) as table_count
FROM pg_tables
WHERE schemaname IN ('auth', 'education', 'trading', 'investment', 'financial', 'ml', 'llm', 'audit')
GROUP BY schemaname
ORDER BY schemaname;

\echo ''
\echo 'Functions by Schema:'

SELECT
    n.nspname as schema,
    COUNT(*) as function_count
FROM pg_proc p
JOIN pg_namespace n ON p.pronamespace = n.oid
WHERE n.nspname IN ('auth', 'education', 'trading', 'investment', 'financial', 'ml', 'llm', 'audit')
GROUP BY n.nspname
ORDER BY n.nspname;

\echo ''
\echo 'Enums by Schema:'

SELECT
    n.nspname as schema,
    COUNT(*) as enum_count
FROM pg_type t
JOIN pg_namespace n ON t.typnamespace = n.oid
WHERE t.typtype = 'e'
  AND n.nspname IN ('auth', 'education', 'trading', 'investment', 'financial', 'ml', 'llm', 'audit')
GROUP BY n.nspname
ORDER BY n.nspname;

\echo ''
\echo '============================================'
\echo 'DDL Validation Complete'
\echo '============================================'

EOF

echo -e "${GREEN}✓ Master SQL script generated: $MASTER_SQL${NC}"

# Execute the master SQL file
echo -e "${YELLOW}[4/5] Executing DDL scripts...${NC}"
echo -e "${BLUE}Database: $PGDATABASE${NC}"
echo -e "${BLUE}User: $PGUSER${NC}"
echo -e "${BLUE}Host: $PGHOST:$PGPORT${NC}"
echo ""

if psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -f "$MASTER_SQL" 2> "$ERROR_LOG"; then
    echo ""
    echo -e "${GREEN}✓ DDL execution completed successfully${NC}"

    # Show any warnings (if error log has content but exit code was 0)
    if [ -s "$ERROR_LOG" ]; then
        echo -e "${YELLOW}Warnings/Notices:${NC}"
        cat "$ERROR_LOG"
    fi
else
    echo ""
    echo -e "${RED}✗ DDL execution failed${NC}"
    echo -e "${RED}Error details:${NC}"
    cat "$ERROR_LOG"

    echo ""
    echo -e "${YELLOW}Analyzing errors...${NC}"

    # Try to identify which file caused the error
    if grep -q "syntax error" "$ERROR_LOG"; then
        echo -e "${RED}Syntax errors found:${NC}"
        grep -A 2 "syntax error" "$ERROR_LOG"
    fi

    if grep -q "does not exist" "$ERROR_LOG"; then
        echo -e "${RED}Missing dependencies (tables/functions):${NC}"
        grep "does not exist" "$ERROR_LOG"
    fi

    if grep -q "violates foreign key" "$ERROR_LOG"; then
        echo -e "${RED}Foreign key violations:${NC}"
        grep "violates foreign key" "$ERROR_LOG"
    fi

    exit 1
fi

# Final validation
echo -e "${YELLOW}[5/5] Final validation...${NC}"

# Count created objects
TOTAL_TABLES=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -tAc "
    SELECT COUNT(*)
    FROM pg_tables
    WHERE schemaname IN ('auth', 'education', 'trading', 'investment', 'financial', 'ml', 'llm', 'audit')
")

TOTAL_FUNCTIONS=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -tAc "
    SELECT COUNT(*)
    FROM pg_proc p
    JOIN pg_namespace n ON p.pronamespace = n.oid
    WHERE n.nspname IN ('auth', 'education', 'trading', 'investment', 'financial', 'ml', 'llm', 'audit')
")

TOTAL_ENUMS=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -tAc "
    SELECT COUNT(*)
    FROM pg_type t
    JOIN pg_namespace n ON t.typnamespace = n.oid
    WHERE t.typtype = 'e'
      AND n.nspname IN ('auth', 'education', 'trading', 'investment', 'financial', 'ml', 'llm', 'audit')
")

echo ""
echo -e "${GREEN}============================================${NC}"
echo -e "${GREEN}Validation Summary${NC}"
echo -e "${GREEN}============================================${NC}"
echo -e "Total Tables:    ${GREEN}$TOTAL_TABLES${NC}"
echo -e "Total Functions: ${GREEN}$TOTAL_FUNCTIONS${NC}"
echo -e "Total Enums:     ${GREEN}$TOTAL_ENUMS${NC}"
echo ""
echo -e "${GREEN}✓ All DDL scripts validated successfully${NC}"
echo ""
echo -e "Master SQL file: ${BLUE}$MASTER_SQL${NC}"
echo -e "Error log:       ${BLUE}$ERROR_LOG${NC}"