rckrdmrd
45e77e9a9c
DDL schemas for Trading Platform: - User management - Authentication - Payments - Education - ML predictions - Trading data Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
97 lines
2.9 KiB
Bash
Executable File
97 lines
2.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Migrate all tickers from MySQL to PostgreSQL
|
|
# Run in background: nohup ./migrate_all_tickers.sh > migration.log 2>&1 &
|
|
|
|
set -e
|
|
|
|
MYSQL_HOST="72.60.226.4"
|
|
MYSQL_USER="root"
|
|
MYSQL_PASS="AfcItz2391,."
|
|
MYSQL_DB="db_trading_meta"
|
|
|
|
PG_USER="trading_user"
|
|
PG_PASS="trading_dev_2025"
|
|
PG_DB="trading_data"
|
|
|
|
# Ticker mapping
|
|
declare -A TICKERS=(
|
|
["C:EURUSD"]="2"
|
|
["C:GBPUSD"]="3"
|
|
["C:USDJPY"]="4"
|
|
["C:USDCAD"]="5"
|
|
["C:AUDUSD"]="6"
|
|
["C:NZDUSD"]="7"
|
|
["C:EURGBP"]="8"
|
|
["C:EURAUD"]="9"
|
|
["C:EURCHF"]="10"
|
|
["C:GBPJPY"]="11"
|
|
["C:GBPAUD"]="12"
|
|
["C:GBPCAD"]="13"
|
|
["C:GBPNZD"]="14"
|
|
["C:AUDCAD"]="15"
|
|
["C:AUDCHF"]="16"
|
|
["C:AUDNZD"]="17"
|
|
["C:XAUUSD"]="18"
|
|
)
|
|
|
|
migrate_ticker() {
|
|
local mysql_sym=$1
|
|
local pg_id=$2
|
|
local tmpfile="/tmp/ticker_${pg_id}.tsv"
|
|
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $mysql_sym (id=$pg_id)..."
|
|
|
|
# Export from MySQL
|
|
mysql -h $MYSQL_HOST -u $MYSQL_USER -p"$MYSQL_PASS" $MYSQL_DB \
|
|
--batch --skip-column-names \
|
|
-e "SELECT $pg_id, date_agg, open, high, low, close, volume, vwap, ts FROM tickers_agg_data WHERE ticker='$mysql_sym' ORDER BY date_agg" 2>/dev/null \
|
|
> "$tmpfile"
|
|
|
|
local count=$(wc -l < "$tmpfile")
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Exported $count rows for $mysql_sym"
|
|
|
|
# Truncate staging and import
|
|
PGPASSWORD=$PG_PASS psql -h localhost -U $PG_USER -d $PG_DB -c "TRUNCATE market_data.ohlcv_5m_staging;" >/dev/null
|
|
|
|
PGPASSWORD=$PG_PASS psql -h localhost -U $PG_USER -d $PG_DB \
|
|
-c "\COPY market_data.ohlcv_5m_staging FROM '$tmpfile' WITH (FORMAT text, DELIMITER E'\t')" >/dev/null
|
|
|
|
# Upsert to main table
|
|
local inserted=$(PGPASSWORD=$PG_PASS psql -h localhost -U $PG_USER -d $PG_DB -t -c "
|
|
INSERT INTO market_data.ohlcv_5m (ticker_id, timestamp, open, high, low, close, volume, vwap, ts_epoch)
|
|
SELECT DISTINCT ON (ticker_id, timestamp)
|
|
ticker_id, timestamp, open, high, low, close, volume, vwap, ts_epoch
|
|
FROM market_data.ohlcv_5m_staging
|
|
ON CONFLICT (ticker_id, timestamp) DO NOTHING;
|
|
SELECT COUNT(*) FROM market_data.ohlcv_5m WHERE ticker_id = $pg_id;
|
|
")
|
|
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Completed $mysql_sym: $inserted rows in PostgreSQL"
|
|
rm -f "$tmpfile"
|
|
}
|
|
|
|
echo "============================================"
|
|
echo "Starting full migration at $(date)"
|
|
echo "============================================"
|
|
|
|
for mysql_sym in "${!TICKERS[@]}"; do
|
|
pg_id=${TICKERS[$mysql_sym]}
|
|
migrate_ticker "$mysql_sym" "$pg_id"
|
|
done
|
|
|
|
echo ""
|
|
echo "============================================"
|
|
echo "Migration complete at $(date)"
|
|
echo "============================================"
|
|
|
|
# Final verification
|
|
echo ""
|
|
echo "Final row counts:"
|
|
PGPASSWORD=$PG_PASS psql -h localhost -U $PG_USER -d $PG_DB -c "
|
|
SELECT t.symbol, COUNT(*) as rows
|
|
FROM market_data.ohlcv_5m o
|
|
JOIN market_data.tickers t ON t.id = o.ticker_id
|
|
GROUP BY t.symbol
|
|
ORDER BY t.symbol;
|
|
"
|