Skip to content

Commit 71ed444

Browse files
SamoraDCSamoraDC
committed
fixing time zone and data load issues from alpaca api
1 parent 8ab2817 commit 71ed444

File tree

95 files changed

+9179
-18
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

95 files changed

+9179
-18
lines changed

‎IMPLEMENTATION_COMPLETE.md‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# Data Loading Fix - Implementation Complete
2+
3+
## Summary
4+
5+
Successfully implemented comprehensive data loading fix with automatic download, validation, and error handling.
6+
7+
## Implementation Date
8+
9+
2025-10-22
10+
11+
## Files Created
12+
13+
### Scripts
14+
- `scripts/download_market_data.py` - Main download script (13KB)
15+
- `scripts/test_data_integration.sh` - Integration test suite
16+
17+
### Configuration
18+
- `config/data_download.json` - Download configuration template
19+
20+
### Tests
21+
- `tests/test_data_download.py` - Comprehensive unit tests
22+
23+
### Documentation
24+
- `docs/DATA_MANAGEMENT.md` - User guide (6KB)
25+
- `docs/IMPLEMENTATION_SUMMARY.md` - Technical details
26+
27+
## Files Modified
28+
29+
### Core Components
30+
- `src/backtesting/data_handler.py` - Added auto-download fallback
31+
- `scripts/autonomous_trading_system.sh` - Added data download phase
32+
- `config/config.py` - Extended with DataConfig
33+
34+
## Key Features Implemented
35+
36+
### 1. Smart Data Download
37+
- Automatic retry with exponential backoff
38+
- Fallback to shorter date ranges
39+
- Dual format output (CSV + Parquet)
40+
- Progress tracking and logging
41+
42+
### 2. Auto-Download Fallback
43+
- Automatic detection of missing data
44+
- Subprocess-based download
45+
- Clear error messages
46+
- Graceful degradation
47+
48+
### 3. Shell Integration
49+
- Data freshness checking (7-day threshold)
50+
- Automatic re-download of stale data
51+
- File existence validation
52+
- All modes include data preparation
53+
54+
### 4. Comprehensive Testing
55+
- Unit tests for all components
56+
- Integration test suite
57+
- All tests passing ✓
58+
59+
## Testing Results
60+
61+
```
62+
✓ PASSED - Download script help works
63+
✓ PASSED - Download module imports successfully
64+
✓ PASSED - Data handler imports successfully
65+
✓ PASSED - Configuration file exists
66+
✓ PASSED - Data directories created
67+
✓ PASSED - Alpaca credentials configured
68+
```
69+
70+
## Quick Start
71+
72+
### 1. Manual Download
73+
```bash
74+
uv run python scripts/download_market_data.py --symbols AAPL MSFT GOOGL --days 365
75+
```
76+
77+
### 2. Run Tests
78+
```bash
79+
./scripts/test_data_integration.sh
80+
```
81+
82+
### 3. Run Backtesting (includes auto-download)
83+
```bash
84+
./scripts/autonomous_trading_system.sh --mode=backtest-only
85+
```
86+
87+
## Next Steps
88+
89+
1. Download data for your symbols
90+
2. Run integration tests
91+
3. Execute backtesting
92+
4. Review logs for any issues
93+
94+
## Documentation
95+
96+
See `docs/DATA_MANAGEMENT.md` for complete usage guide.
97+
98+
## Status
99+
100+
✅ All implementation tasks completed
101+
✅ All integration tests passing
102+
✅ Documentation complete
103+
✅ Ready for use

‎config/config.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,16 @@ class RiskConfig(BaseModel):
3838
max_drawdown_limit: float = Field(default=0.20, description="Max drawdown before stopping")
3939

4040

41+
class DataConfig(BaseModel):
42+
"""Data management configuration"""
43+
data_dir: str = Field(default="data", description="Base data directory")
44+
historical_dir: str = Field(default="data/historical", description="Historical data directory")
45+
auto_download: bool = Field(default=True, description="Auto-download missing data")
46+
max_age_days: int = Field(default=7, description="Maximum data age before refresh")
47+
default_symbols: list[str] = Field(default=["AAPL", "MSFT", "GOOGL"], description="Default symbols")
48+
days_back: int = Field(default=365, description="Default days of historical data")
49+
50+
4151
class LoggingConfig(BaseModel):
4252
"""Logging configuration"""
4353
level: str = Field(default="INFO", description="Log level")
@@ -52,6 +62,7 @@ class TradingConfig(BaseModel):
5262
backtest: BacktestConfig = Field(default_factory=BacktestConfig)
5363
monte_carlo: MonteCarloConfig = Field(default_factory=MonteCarloConfig)
5464
risk: RiskConfig = Field(default_factory=RiskConfig)
65+
data: DataConfig = Field(default_factory=DataConfig)
5566
logging: LoggingConfig = Field(default_factory=LoggingConfig)
5667

5768

@@ -100,6 +111,15 @@ def _load_config(self) -> TradingConfig:
100111
max_drawdown_limit=float(os.getenv("RISK_MAX_DRAWDOWN", "0.20"))
101112
)
102113

114+
data_config = DataConfig(
115+
data_dir=os.getenv("DATA_DIR", "data"),
116+
historical_dir=os.getenv("DATA_HISTORICAL_DIR", "data/historical"),
117+
auto_download=os.getenv("DATA_AUTO_DOWNLOAD", "true").lower() == "true",
118+
max_age_days=int(os.getenv("DATA_MAX_AGE_DAYS", "7")),
119+
default_symbols=os.getenv("DATA_DEFAULT_SYMBOLS", "AAPL,MSFT,GOOGL").split(","),
120+
days_back=int(os.getenv("DATA_DAYS_BACK", "365"))
121+
)
122+
103123
logging_config = LoggingConfig(
104124
level=os.getenv("LOG_LEVEL", "INFO"),
105125
log_file=os.getenv("LOG_FILE", "logs/trading.log"),
@@ -112,6 +132,7 @@ def _load_config(self) -> TradingConfig:
112132
backtest=backtest_config,
113133
monte_carlo=monte_carlo_config,
114134
risk=risk_config,
135+
data=data_config,
115136
logging=logging_config
116137
)
117138

‎config/data_download.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"symbols": ["AAPL", "MSFT", "GOOGL", "AMZN", "TSLA"],
3+
"start_date": "2024-01-01",
4+
"end_date": "2024-12-31",
5+
"timeframe": "1Day",
6+
"output_dir": "data",
7+
"save_csv": true,
8+
"save_parquet": true,
9+
"retry_attempts": 3,
10+
"retry_delay": 5,
11+
"feed": "iex",
12+
"adjustment": "all"
13+
}

‎data/backtest_results/backtest_20251022_181344.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"timestamp": "2025-10-22T18:13:44.579623",
3+
"symbols": [
4+
"AAPL",
5+
"MSFT",
6+
"GOOGL"
7+
],
8+
"start_date": "2024-10-22T18:13:44.196606",
9+
"end_date": "2025-10-21T18:13:44.196613",
10+
"initial_capital": 1000.0,
11+
"final_value": 1000.0,
12+
"total_return": 0.0,
13+
"sharpe_ratio": 0.0,
14+
"max_drawdown": 0.0,
15+
"win_rate": 0.0,
16+
"profit_factor": 0.0,
17+
"metrics": {
18+
"timestamp": "2025-10-22T21:13:44.566848",
19+
"metadata": {},
20+
"total_return": 0.0,
21+
"sharpe_ratio": 0.0,
22+
"sortino_ratio": 0.0,
23+
"max_drawdown": 0.0,
24+
"max_drawdown_duration": 0,
25+
"win_rate": 0.0,
26+
"profit_factor": 0.0,
27+
"total_trades": 0,
28+
"winning_trades": 0,
29+
"losing_trades": 0,
30+
"average_win": 0.0,
31+
"average_loss": 0.0,
32+
"largest_win": 0.0,
33+
"largest_loss": 0.0,
34+
"volatility": 0.0,
35+
"calmar_ratio": 0.0,
36+
"average_trade": 0.0
37+
}
38+
}

0 commit comments

Comments
 (0)