rckrdmrd
62a9f3e1d9
Data aggregation and distribution service: - Market data collection - OHLCV aggregation - Real-time data feeds - Data API endpoints Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
332 lines
10 KiB
Python
332 lines
10 KiB
Python
"""
|
|
Data Synchronization API Routes
|
|
OrbiQuant IA Trading Platform - Data Service
|
|
|
|
Endpoints for managing data synchronization with Massive.com/Polygon.io
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from typing import Optional, List, Dict, Any
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, status, BackgroundTasks
|
|
from pydantic import BaseModel, Field
|
|
|
|
import asyncpg
|
|
|
|
from providers.polygon_client import AssetType, Timeframe
|
|
from services.sync_service import DataSyncService, SyncStatus
|
|
from .dependencies import get_db_pool, get_polygon_client
|
|
|
|
|
|
router = APIRouter(prefix="/api/sync", tags=["Data Sync"])
|
|
|
|
|
|
# =============================================================================
|
|
# Request/Response Models
|
|
# =============================================================================
|
|
|
|
class SyncSymbolRequest(BaseModel):
|
|
"""Request to sync a specific symbol."""
|
|
asset_type: AssetType = Field(AssetType.FOREX, description="Asset type")
|
|
timeframe: Timeframe = Field(Timeframe.MINUTE_5, description="Timeframe to sync")
|
|
backfill_days: int = Field(30, ge=1, le=365, description="Days to backfill")
|
|
|
|
|
|
class SyncSymbolResponse(BaseModel):
|
|
"""Response from sync operation."""
|
|
status: str
|
|
symbol: str
|
|
timeframe: str
|
|
rows_inserted: int
|
|
start_date: Optional[str] = None
|
|
end_date: Optional[str] = None
|
|
error: Optional[str] = None
|
|
|
|
|
|
class SyncStatusResponse(BaseModel):
|
|
"""Sync status for a ticker."""
|
|
symbol: str
|
|
asset_type: str
|
|
timeframe: Optional[str] = None
|
|
last_sync: Optional[str] = None
|
|
rows_synced: Optional[int] = None
|
|
status: Optional[str] = None
|
|
error: Optional[str] = None
|
|
updated_at: Optional[str] = None
|
|
|
|
|
|
class SyncAllResponse(BaseModel):
|
|
"""Response from syncing all tickers."""
|
|
total_tickers: int
|
|
successful: int
|
|
failed: int
|
|
total_rows_inserted: int
|
|
message: str
|
|
|
|
|
|
class SymbolInfo(BaseModel):
|
|
"""Information about a supported symbol."""
|
|
symbol: str
|
|
polygon_symbol: str
|
|
mt4_symbol: Optional[str] = None
|
|
asset_type: str
|
|
pip_value: Optional[float] = None
|
|
supported: bool = True
|
|
|
|
|
|
class SymbolsListResponse(BaseModel):
|
|
"""List of supported symbols."""
|
|
symbols: List[SymbolInfo]
|
|
total: int
|
|
asset_types: List[str]
|
|
|
|
|
|
# =============================================================================
|
|
# Dependency Functions
|
|
# =============================================================================
|
|
|
|
async def get_sync_service(
|
|
db_pool: asyncpg.Pool = Depends(get_db_pool),
|
|
polygon_client = Depends(get_polygon_client)
|
|
) -> DataSyncService:
|
|
"""Get DataSyncService instance."""
|
|
return DataSyncService(
|
|
polygon_client=polygon_client,
|
|
db_pool=db_pool,
|
|
batch_size=10000
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# Symbols Endpoints
|
|
# =============================================================================
|
|
|
|
@router.get("/symbols", response_model=SymbolsListResponse)
|
|
async def list_supported_symbols(
|
|
asset_type: Optional[AssetType] = Query(None, description="Filter by asset type"),
|
|
sync_service: DataSyncService = Depends(get_sync_service)
|
|
):
|
|
"""
|
|
Get list of symbols supported by Massive.com/Polygon.io.
|
|
|
|
Returns all symbols configured in the system with their mappings.
|
|
Can be filtered by asset type (forex, crypto, index, stock).
|
|
"""
|
|
symbols = await sync_service.get_supported_symbols(asset_type=asset_type)
|
|
|
|
# Get unique asset types
|
|
asset_types = list(set(s["asset_type"] for s in symbols))
|
|
|
|
return SymbolsListResponse(
|
|
symbols=[SymbolInfo(**s) for s in symbols],
|
|
total=len(symbols),
|
|
asset_types=sorted(asset_types)
|
|
)
|
|
|
|
|
|
@router.get("/symbols/{symbol}")
|
|
async def get_symbol_info(
|
|
symbol: str,
|
|
sync_service: DataSyncService = Depends(get_sync_service)
|
|
):
|
|
"""
|
|
Get detailed information about a specific symbol.
|
|
|
|
Includes sync status and configuration.
|
|
"""
|
|
# Get symbol from supported list
|
|
symbols = await sync_service.get_supported_symbols()
|
|
symbol_info = next((s for s in symbols if s["symbol"].upper() == symbol.upper()), None)
|
|
|
|
if not symbol_info:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail=f"Symbol {symbol} not supported"
|
|
)
|
|
|
|
# Get sync status
|
|
sync_status = await sync_service.get_sync_status(symbol=symbol)
|
|
|
|
return {
|
|
**symbol_info,
|
|
"sync_status": sync_status
|
|
}
|
|
|
|
|
|
# =============================================================================
|
|
# Sync Control Endpoints
|
|
# =============================================================================
|
|
|
|
@router.post("/sync/{symbol}", response_model=SyncSymbolResponse)
|
|
async def sync_symbol(
|
|
symbol: str,
|
|
request: SyncSymbolRequest,
|
|
background_tasks: BackgroundTasks,
|
|
sync_service: DataSyncService = Depends(get_sync_service)
|
|
):
|
|
"""
|
|
Trigger data synchronization for a specific symbol.
|
|
|
|
This will fetch historical data from Massive.com/Polygon.io and store it
|
|
in the database. The operation runs in the background and returns immediately.
|
|
|
|
Parameters:
|
|
- **symbol**: Ticker symbol (e.g., 'EURUSD', 'BTCUSD')
|
|
- **asset_type**: Type of asset (forex, crypto, index, stock)
|
|
- **timeframe**: Data timeframe (1m, 5m, 15m, 1h, 4h, 1d)
|
|
- **backfill_days**: Number of days to backfill (1-365)
|
|
"""
|
|
# Validate symbol is supported
|
|
symbols = await sync_service.get_supported_symbols()
|
|
if not any(s["symbol"].upper() == symbol.upper() for s in symbols):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail=f"Symbol {symbol} not supported. Use /api/sync/symbols to see available symbols."
|
|
)
|
|
|
|
# Start sync in background
|
|
result = await sync_service.sync_ticker_data(
|
|
symbol=symbol.upper(),
|
|
asset_type=request.asset_type,
|
|
timeframe=request.timeframe,
|
|
backfill_days=request.backfill_days
|
|
)
|
|
|
|
return SyncSymbolResponse(**result)
|
|
|
|
|
|
@router.post("/sync-all", response_model=SyncAllResponse)
|
|
async def sync_all_symbols(
|
|
background_tasks: BackgroundTasks,
|
|
timeframe: Timeframe = Query(Timeframe.MINUTE_5, description="Timeframe to sync"),
|
|
backfill_days: int = Query(1, ge=1, le=30, description="Days to backfill"),
|
|
sync_service: DataSyncService = Depends(get_sync_service)
|
|
):
|
|
"""
|
|
Trigger synchronization for all active tickers.
|
|
|
|
This is a heavy operation and may take a while depending on the number
|
|
of active tickers and the API rate limits.
|
|
|
|
Only use this for initial setup or manual full sync.
|
|
"""
|
|
# Run sync in background
|
|
def run_sync():
|
|
import asyncio
|
|
loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(loop)
|
|
result = loop.run_until_complete(
|
|
sync_service.sync_all_active_tickers(
|
|
timeframe=timeframe,
|
|
backfill_days=backfill_days
|
|
)
|
|
)
|
|
loop.close()
|
|
return result
|
|
|
|
background_tasks.add_task(run_sync)
|
|
|
|
return SyncAllResponse(
|
|
total_tickers=0,
|
|
successful=0,
|
|
failed=0,
|
|
total_rows_inserted=0,
|
|
message="Sync started in background. Check /api/sync/status for progress."
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# Status Endpoints
|
|
# =============================================================================
|
|
|
|
@router.get("/status", response_model=List[SyncStatusResponse])
|
|
async def get_sync_status(
|
|
symbol: Optional[str] = Query(None, description="Filter by symbol"),
|
|
sync_service: DataSyncService = Depends(get_sync_service)
|
|
):
|
|
"""
|
|
Get synchronization status for all tickers or a specific symbol.
|
|
|
|
Shows:
|
|
- Last sync timestamp
|
|
- Number of rows synced
|
|
- Sync status (success, failed, in_progress)
|
|
- Error messages if any
|
|
"""
|
|
status_list = await sync_service.get_sync_status(symbol=symbol)
|
|
|
|
return [SyncStatusResponse(**s) for s in status_list]
|
|
|
|
|
|
@router.get("/status/{symbol}", response_model=List[SyncStatusResponse])
|
|
async def get_symbol_sync_status(
|
|
symbol: str,
|
|
sync_service: DataSyncService = Depends(get_sync_service)
|
|
):
|
|
"""
|
|
Get detailed sync status for a specific symbol across all timeframes.
|
|
"""
|
|
status_list = await sync_service.get_sync_status(symbol=symbol)
|
|
|
|
if not status_list:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail=f"No sync status found for symbol {symbol}"
|
|
)
|
|
|
|
return [SyncStatusResponse(**s) for s in status_list]
|
|
|
|
|
|
# =============================================================================
|
|
# Health Check
|
|
# =============================================================================
|
|
|
|
@router.get("/health")
|
|
async def sync_health_check(
|
|
polygon_client = Depends(get_polygon_client),
|
|
db_pool: asyncpg.Pool = Depends(get_db_pool)
|
|
):
|
|
"""
|
|
Check health of sync service and data providers.
|
|
|
|
Verifies:
|
|
- Database connectivity
|
|
- Polygon/Massive API accessibility
|
|
- Rate limit status
|
|
"""
|
|
health = {
|
|
"status": "healthy",
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
"providers": {}
|
|
}
|
|
|
|
# Check database
|
|
try:
|
|
async with db_pool.acquire() as conn:
|
|
await conn.fetchval("SELECT 1")
|
|
health["providers"]["database"] = {
|
|
"status": "connected",
|
|
"type": "PostgreSQL"
|
|
}
|
|
except Exception as e:
|
|
health["status"] = "unhealthy"
|
|
health["providers"]["database"] = {
|
|
"status": "error",
|
|
"error": str(e)
|
|
}
|
|
|
|
# Check Polygon API (basic connectivity)
|
|
try:
|
|
health["providers"]["polygon"] = {
|
|
"status": "configured",
|
|
"base_url": polygon_client.base_url,
|
|
"rate_limit": f"{polygon_client.rate_limit} req/min"
|
|
}
|
|
except Exception as e:
|
|
health["status"] = "degraded"
|
|
health["providers"]["polygon"] = {
|
|
"status": "error",
|
|
"error": str(e)
|
|
}
|
|
|
|
return health
|