Files
docker-configs/backtest/strategy/emotional-damage/backtest_emotional_damage_enhanced.py
2025-07-18 00:00:01 -05:00

600 lines
25 KiB
Python

import sqlite3
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.backends.backend_pdf import PdfPages
import warnings
warnings.filterwarnings('ignore')
class EnhancedEmotionalDamageStrategy:
def __init__(self, initial_capital=100000):
self.initial_capital = initial_capital
self.cash = initial_capital
self.positions = {} # ticker: shares
self.portfolio_value = []
self.trades = []
self.fear_threshold = 25
self.greed_threshold = 75
self.top_stocks_count = 10
self.stop_loss_threshold = 0.15 # 15% stop loss
# New state management for gradual transitions
self.state = 'QQQ_HOLD' # QQQ_HOLD, FEAR_TRANSITION, GREED_TRANSITION, VOLATILE_STOCKS
self.transition_steps = 4
self.current_transition_step = 0
self.transition_target = None
self.transition_stocks = []
self.last_fear_date = None
self.last_greed_date = None
def get_data(self):
"""Load Fear & Greed Index and stock data"""
conn = sqlite3.connect('data/stock_data.db')
# Get Fear & Greed Index
fg_data = pd.read_sql_query('''
SELECT date, fear_greed_index
FROM fear_greed_index
ORDER BY date
''', conn)
fg_data['date'] = pd.to_datetime(fg_data['date'])
fg_data.set_index('date', inplace=True)
# Get QQQ price data
spy_data = pd.read_sql_query('''
SELECT date, spy_close
FROM fear_greed_data
ORDER BY date
''', conn)
spy_data['date'] = pd.to_datetime(spy_data['date'])
spy_data.set_index('date', inplace=True)
# Get available tickers
cursor = conn.cursor()
cursor.execute('SELECT ticker FROM ticker_list WHERE records > 1000')
self.available_tickers = [row[0] for row in cursor.fetchall()]
conn.close()
# Merge data
self.data = pd.merge(fg_data, spy_data, left_index=True, right_index=True, how='inner')
self.data.sort_index(inplace=True)
print(f"Loaded data from {self.data.index.min().strftime('%Y-%m-%d')} to {self.data.index.max().strftime('%Y-%m-%d')}")
print(f"Available tickers for selection: {len(self.available_tickers)}")
def get_stock_price(self, ticker, date):
"""Get stock price for a specific ticker and date"""
conn = sqlite3.connect('data/stock_data.db')
query = f'''
SELECT close FROM {ticker.lower()}
WHERE date <= ?
ORDER BY date DESC
LIMIT 1
'''
cursor = conn.cursor()
cursor.execute(query, (date.strftime('%Y-%m-%d'),))
result = cursor.fetchone()
conn.close()
return result[0] if result else None
def get_stock_data(self, ticker, start_date, end_date):
"""Get historical stock data for technical analysis"""
conn = sqlite3.connect('data/stock_data.db')
query = f'''
SELECT date, open, high, low, close, volume
FROM {ticker.lower()}
WHERE date >= ? AND date <= ?
ORDER BY date
'''
df = pd.read_sql_query(query, conn, params=(
start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d')
))
conn.close()
if not df.empty:
df['date'] = pd.to_datetime(df['date'])
df.set_index('date', inplace=True)
return df
return None
def calculate_technical_indicators(self, df):
"""Calculate MACD, RSI, and EMA indicators"""
if len(df) < 50: # Need sufficient data
return None
# RSI
delta = df['close'].diff()
gain = (delta.where(delta > 0, 0)).rolling(window=14).mean()
loss = (-delta.where(delta < 0, 0)).rolling(window=14).mean()
rs = gain / loss
rsi = 100 - (100 / (1 + rs))
# MACD
exp1 = df['close'].ewm(span=12).mean()
exp2 = df['close'].ewm(span=26).mean()
macd = exp1 - exp2
signal = macd.ewm(span=9).mean()
# EMA
ema5 = df['close'].ewm(span=5).mean()
ema20 = df['close'].ewm(span=20).mean()
return {
'rsi': rsi,
'macd': macd,
'signal': signal,
'ema5': ema5,
'ema20': ema20
}
def check_signal_direction(self, indicators, df):
"""Check if indicators are turning upward"""
if indicators is None:
return False
signals = []
# RSI upward turn (first derivative positive)
if len(indicators['rsi']) >= 3:
rsi_current = indicators['rsi'].iloc[-1]
rsi_prev = indicators['rsi'].iloc[-2]
rsi_slope = rsi_current - rsi_prev
signals.append(rsi_slope > 0)
# MACD golden cross (MACD crosses above signal)
if len(indicators['macd']) >= 3:
macd_current = indicators['macd'].iloc[-1]
signal_current = indicators['signal'].iloc[-1]
macd_prev = indicators['macd'].iloc[-2]
signal_prev = indicators['signal'].iloc[-2]
# Golden cross: macd crosses above signal
golden_cross = (macd_prev <= signal_prev) and (macd_current > signal_current)
signals.append(golden_cross)
# EMA crossover (EMA5 crosses above EMA20)
if len(indicators['ema5']) >= 3:
ema5_current = indicators['ema5'].iloc[-1]
ema20_current = indicators['ema20'].iloc[-1]
ema5_prev = indicators['ema5'].iloc[-2]
ema20_prev = indicators['ema20'].iloc[-2]
ema_crossover = (ema5_prev <= ema20_prev) and (ema5_current > ema20_current)
signals.append(ema_crossover)
# Need at least 2 out of 3 signals positive
return sum(signals) >= 2
def calculate_volatility(self, ticker, start_date, end_date):
"""Calculate historical volatility for a single ticker"""
conn = sqlite3.connect('data/stock_data.db')
try:
query = f'''
SELECT date, close FROM {ticker.lower()}
WHERE date >= ? AND date <= ?
ORDER BY date
'''
df = pd.read_sql_query(query, conn, params=(
start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d')
))
if len(df) > 10:
df['returns'] = df['close'].pct_change()
volatility = df['returns'].std() * np.sqrt(252)
conn.close()
return volatility
except Exception as e:
pass
conn.close()
return 0
def select_stocks_with_technical_filter(self, fear_start_date, fear_end_date):
"""Select stocks using technical indicators + volatility ranking"""
candidates = []
# Extend the period for more data
extended_start = fear_start_date - timedelta(days=30)
extended_end = fear_end_date + timedelta(days=5)
for ticker in self.available_tickers:
stock_data = self.get_stock_data(ticker, extended_start, extended_end)
if stock_data is not None and len(stock_data) >= 30:
volatility = self.calculate_volatility(ticker, fear_start_date, fear_end_date)
if volatility > 0.1: # Minimum volatility threshold
# Check technical indicators on recent data
recent_data = stock_data.tail(30)
indicators = self.calculate_technical_indicators(recent_data)
# Be more lenient - accept if at least some indicators are positive
technical_score = 0
if indicators is not None:
# Simplified scoring - just check if recent trend is up
recent_trend = recent_data['close'].pct_change().tail(5).sum()
if recent_trend > -0.02: # Not strongly declining
technical_score += 1
# Check if RSI is not oversold
if len(indicators['rsi']) > 0 and indicators['rsi'].iloc[-1] > 30:
technical_score += 1
# Accept if basic criteria met or if volatility is high
if technical_score >= 1 or volatility > 0.5:
candidates.append((ticker, volatility))
# Sort by volatility and select top stocks
candidates.sort(key=lambda x: x[1], reverse=True)
selected = [ticker for ticker, vol in candidates[:self.top_stocks_count]]
print(f"Selected {len(selected)} stocks from {len(candidates)} candidates")
if selected:
print(f"Top stocks: {selected}")
return selected
def execute_gradual_transition(self, date, target_state):
"""Execute gradual 4-step position transitions"""
if target_state == 'CASH':
# Gradually sell to cash
if self.current_transition_step < self.transition_steps:
step_size = 1.0 / self.transition_steps
step_pct = step_size * (self.current_transition_step + 1)
# Sell portion of holdings
for ticker in list(self.positions.keys()):
if ticker != 'QQQ':
shares_to_sell = int(self.positions[ticker] * step_pct)
if shares_to_sell > 0:
price = self.get_stock_price(ticker, date)
if price:
value = shares_to_sell * price
self.cash += value
self.positions[ticker] -= shares_to_sell
if self.positions[ticker] <= 0:
del self.positions[ticker]
self.trades.append({
'date': date,
'action': 'SELL_GRADUAL',
'ticker': ticker,
'shares': shares_to_sell,
'price': price,
'value': value
})
self.current_transition_step += 1
if self.current_transition_step >= self.transition_steps:
return True # Transition complete
elif target_state == 'VOLATILE':
# Gradually buy volatile stocks
if self.current_transition_step < self.transition_steps:
step_size = 1.0 / self.transition_steps
step_pct = step_size * (self.current_transition_step + 1)
if self.transition_stocks:
amount_per_stock = (self.cash * step_pct) / len(self.transition_stocks)
for ticker in self.transition_stocks:
price = self.get_stock_price(ticker, date)
if price:
shares = amount_per_stock / price
if ticker in self.positions:
self.positions[ticker] += shares
else:
self.positions[ticker] = shares
self.cash -= amount_per_stock
self.trades.append({
'date': date,
'action': 'BUY_GRADUAL',
'ticker': ticker,
'shares': shares,
'price': price,
'value': amount_per_stock
})
self.current_transition_step += 1
if self.current_transition_step >= self.transition_steps:
return True # Transition complete
elif target_state == 'QQQ':
# Gradually buy QQQ
if self.current_transition_step < self.transition_steps:
step_size = 1.0 / self.transition_steps
step_pct = step_size * (self.current_transition_step + 1)
qqq_price = self.data.loc[date, 'spy_close']
total_value = self.calculate_portfolio_value(date)
target_qqq_value = total_value * step_pct
if 'QQQ' not in self.positions:
self.positions['QQQ'] = 0
shares_to_buy = (target_qqq_value - (self.positions.get('QQQ', 0) * qqq_price)) / qqq_price
if shares_to_buy > 0:
self.positions['QQQ'] += shares_to_buy
# Sell other positions proportionally
other_positions = {k: v for k, v in self.positions.items() if k != 'QQQ'}
for ticker, shares in other_positions.items():
shares_to_sell = shares * (step_size / (1 - (self.current_transition_step * step_size)))
price = self.get_stock_price(ticker, date)
if price:
value = shares_to_sell * price
self.cash += value
self.positions[ticker] -= shares_to_sell
if self.positions[ticker] <= 0:
del self.positions[ticker]
self.current_transition_step += 1
if self.current_transition_step >= self.transition_steps:
return True # Transition complete
return False # Transition ongoing
def check_stop_loss(self, date):
"""Check for 15% stop loss and replace with QQQ"""
stop_loss_trades = []
for ticker, shares in list(self.positions.items()):
if ticker == 'QQQ':
continue
current_price = self.get_stock_price(ticker, date)
if current_price:
# Find buy price from recent trades
buy_trades = [t for t in self.trades if t['ticker'] == ticker and t['action'] in ['BUY_VOLATILE', 'BUY_GRADUAL']]
if buy_trades:
avg_buy_price = sum(t['price'] * t['shares'] for t in buy_trades) / sum(t['shares'] for t in buy_trades)
loss_pct = (current_price - avg_buy_price) / avg_buy_price
if loss_pct <= -self.stop_loss_threshold:
# Sell the losing position
value = shares * current_price
self.cash += value
del self.positions[ticker]
stop_loss_trades.append({
'date': date,
'action': 'STOP_LOSS',
'ticker': ticker,
'shares': shares,
'price': current_price,
'value': value,
'loss_pct': loss_pct * 100
})
# Immediately buy QQQ with the proceeds
qqq_price = self.data.loc[date, 'spy_close']
qqq_shares = value / qqq_price
self.positions['QQQ'] = self.positions.get('QQQ', 0) + qqq_shares
stop_loss_trades.append({
'date': date,
'action': 'BUY_QQQ_STOPLOSS',
'ticker': 'QQQ',
'shares': qqq_shares,
'price': qqq_price,
'value': value
})
self.trades.extend(stop_loss_trades)
return len(stop_loss_trades) > 0
def calculate_portfolio_value(self, date):
"""Calculate total portfolio value at given date"""
total_value = self.cash
for ticker, shares in self.positions.items():
if ticker == 'QQQ':
price = self.data.loc[date, 'spy_close']
else:
price = self.get_stock_price(ticker, date)
if price:
total_value += shares * price
return total_value
def run_backtest(self):
"""Run the enhanced emotional damage strategy backtest"""
print("Running Enhanced Emotional Damage Strategy Backtest...")
self.get_data()
# Start with QQQ
first_date = self.data.index[0]
qqq_price = self.data.loc[first_date, 'spy_close']
qqq_shares = self.cash / qqq_price
self.positions['QQQ'] = qqq_shares
self.cash = 0
fear_start_date = None
greed_start_date = None
for i, (date, row) in enumerate(self.data.iterrows()):
fg_index = row['fear_greed_index']
# Check stop loss first
self.check_stop_loss(date)
if self.state == 'QQQ_HOLD':
# Check if Fear & Greed drops below 25
if fg_index < self.fear_threshold:
self.state = 'FEAR_TRANSITION'
self.transition_target = 'CASH'
self.current_transition_step = 0
self.last_fear_date = date
print(f"{date.strftime('%Y-%m-%d')}: Fear & Greed {fg_index:.1f} < 25, starting gradual transition to cash")
elif self.state == 'FEAR_TRANSITION':
# Continue gradual transition to cash
completed = self.execute_gradual_transition(date, 'CASH')
if completed:
# Check if we should transition to volatile stocks
if fg_index >= self.fear_threshold and self.last_fear_date:
# Select stocks using technical filters
top_stocks = self.select_stocks_with_technical_filter(
self.last_fear_date, date
)
if top_stocks:
self.transition_stocks = top_stocks
self.state = 'GREED_TRANSITION' # Transition to volatile stocks
self.transition_target = 'VOLATILE'
self.current_transition_step = 0
print(f"{date.strftime('%Y-%m-%d')}: Fear & Greed recovered, starting transition to volatile stocks: {top_stocks}")
# If fear continues, stay in cash
elif fg_index < self.fear_threshold:
self.state = 'CASH_WAIT'
print(f"{date.strftime('%Y-%m-%d')}: Transition to cash complete, holding cash")
elif self.state == 'CASH_WAIT':
# Waiting in cash, check for recovery
if fg_index >= self.fear_threshold and self.last_fear_date:
# Select stocks using technical filters
top_stocks = self.select_stocks_with_technical_filter(
self.last_fear_date, date
)
if top_stocks:
self.transition_stocks = top_stocks
self.state = 'GREED_TRANSITION' # Transition to volatile stocks
self.transition_target = 'VOLATILE'
self.current_transition_step = 0
print(f"{date.strftime('%Y-%m-%d')}: Fear & Greed recovered, starting transition to volatile stocks: {top_stocks}")
elif self.state == 'GREED_TRANSITION':
# Continue gradual transition to volatile stocks
completed = self.execute_gradual_transition(date, 'VOLATILE')
if completed:
self.state = 'VOLATILE_STOCKS'
self.last_greed_date = date
print(f"{date.strftime('%Y-%m-%d')}: Transition to volatile stocks complete")
elif self.state == 'VOLATILE_STOCKS':
# Check if Fear & Greed exceeds 75 (extreme greed)
if fg_index > self.greed_threshold:
self.state = 'FEAR_TRANSITION' # Transition to QQQ
self.transition_target = 'QQQ'
self.current_transition_step = 0
self.last_greed_date = date
print(f"{date.strftime('%Y-%m-%d')}: Fear & Greed {fg_index:.1f} > 75, starting transition to QQQ")
# Record portfolio value
portfolio_value = self.calculate_portfolio_value(date)
self.portfolio_value.append({
'date': date,
'value': portfolio_value,
'state': self.state,
'fg_index': fg_index,
'cash': self.cash
})
print(f"Backtest completed! Total trades: {len(self.trades)}")
def calculate_performance_metrics(self, returns):
"""Calculate performance metrics"""
total_return = (returns.iloc[-1] / returns.iloc[0] - 1) * 100
annual_return = ((returns.iloc[-1] / returns.iloc[0]) ** (252 / len(returns)) - 1) * 100
# Calculate max drawdown
peak = returns.expanding().max()
drawdown = (returns - peak) / peak
max_drawdown = drawdown.min() * 100
# Find max drawdown period
max_dd_date = drawdown.idxmin()
max_dd_year = max_dd_date.year
# Calculate Sharpe ratio
daily_returns = returns.pct_change().dropna()
sharpe_ratio = np.sqrt(252) * daily_returns.mean() / daily_returns.std()
# Annual returns by year
annual_rets = {}
for year in returns.index.year.unique():
year_data = returns[returns.index.year == year]
if len(year_data) > 1:
year_return = (year_data.iloc[-1] / year_data.iloc[0] - 1) * 100
annual_rets[year] = year_return
return {
'total_return': total_return,
'annual_return': annual_return,
'max_drawdown': max_drawdown,
'max_drawdown_date': max_dd_date,
'max_drawdown_year': max_dd_year,
'sharpe_ratio': sharpe_ratio,
'annual_returns': annual_rets
}
def run_enhanced_backtest():
"""Run the enhanced emotional damage strategy"""
# Run strategy
strategy = EnhancedEmotionalDamageStrategy(initial_capital=100000)
strategy.run_backtest()
# Convert results to DataFrame
portfolio_df = pd.DataFrame(strategy.portfolio_value)
portfolio_df.set_index('date', inplace=True)
# Get benchmark data
conn = sqlite3.connect('data/stock_data.db')
benchmark_data = pd.read_sql_query('''
SELECT date, spy_close
FROM fear_greed_data
ORDER BY date
''', conn)
benchmark_data['date'] = pd.to_datetime(benchmark_data['date'])
benchmark_data.set_index('date', inplace=True)
conn.close()
# Align dates
common_dates = portfolio_df.index.intersection(benchmark_data.index)
portfolio_df = portfolio_df.loc[common_dates]
benchmark_data = benchmark_data.loc[common_dates]
# Normalize to starting value for comparison
start_value = 100000
# Create QQQ and SPY buy-and-hold benchmarks
benchmark_data['qqq_value'] = start_value * (benchmark_data['spy_close'] / benchmark_data['spy_close'].iloc[0])
benchmark_data['spy_value'] = start_value * (benchmark_data['spy_close'] / benchmark_data['spy_close'].iloc[0])
# Calculate performance metrics
strategy_metrics = strategy.calculate_performance_metrics(portfolio_df['value'])
qqq_metrics = strategy.calculate_performance_metrics(benchmark_data['qqq_value'])
spy_metrics = strategy.calculate_performance_metrics(benchmark_data['spy_value'])
return {
'strategy': strategy,
'portfolio_df': portfolio_df,
'benchmark_data': benchmark_data,
'strategy_metrics': strategy_metrics,
'qqq_metrics': qqq_metrics,
'spy_metrics': spy_metrics
}
if __name__ == "__main__":
results = run_enhanced_backtest()
print("Enhanced backtest completed! Results ready for PDF generation.")