482 lines
20 KiB
Python
482 lines
20 KiB
Python
import sqlite3
|
|
import pandas as pd
|
|
import numpy as np
|
|
from datetime import datetime, timedelta
|
|
import warnings
|
|
warnings.filterwarnings('ignore')
|
|
|
|
class EnhancedEmotionalDamageStrategy:
|
|
def __init__(self, initial_capital=100000):
|
|
self.initial_capital = initial_capital
|
|
self.cash = initial_capital
|
|
self.positions = {} # ticker: shares
|
|
self.portfolio_value = []
|
|
self.trades = []
|
|
self.fear_threshold = 25
|
|
self.greed_threshold = 75
|
|
self.top_stocks_count = 10
|
|
self.stop_loss_threshold = 0.15 # 15% stop loss
|
|
|
|
# New state management
|
|
self.state = 'QQQ_HOLD'
|
|
self.transition_steps = 4
|
|
self.current_step = 0
|
|
self.target_allocation = {}
|
|
self.last_fear_date = None
|
|
|
|
# For gradual transitions - store transition plan
|
|
self.transition_plan = {}
|
|
self.transition_cash_pool = 0
|
|
|
|
def get_data(self):
|
|
"""Load Fear & Greed Index and stock data"""
|
|
import os
|
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
|
backtest_dir = os.path.dirname(os.path.dirname(script_dir))
|
|
db_path = os.path.join(backtest_dir, 'data', 'stock_data.db')
|
|
print(f"Strategy connecting to database at: {db_path}")
|
|
conn = sqlite3.connect(db_path)
|
|
|
|
# Get Fear & Greed Index
|
|
fg_data = pd.read_sql_query('''
|
|
SELECT date, fear_greed_index
|
|
FROM fear_greed_index
|
|
ORDER BY date
|
|
''', conn)
|
|
fg_data['date'] = pd.to_datetime(fg_data['date'])
|
|
fg_data.set_index('date', inplace=True)
|
|
|
|
# Get SPY price data as QQQ proxy
|
|
spy_data = pd.read_sql_query('''
|
|
SELECT date, spy_close
|
|
FROM fear_greed_data
|
|
ORDER BY date
|
|
''', conn)
|
|
spy_data['date'] = pd.to_datetime(spy_data['date'])
|
|
spy_data.set_index('date', inplace=True)
|
|
|
|
# Get available tickers
|
|
cursor = conn.cursor()
|
|
cursor.execute('SELECT ticker FROM ticker_list WHERE records > 1000')
|
|
self.available_tickers = [row[0] for row in cursor.fetchall()]
|
|
|
|
conn.close()
|
|
|
|
# Merge data
|
|
self.data = pd.merge(fg_data, spy_data, left_index=True, right_index=True, how='inner')
|
|
self.data.sort_index(inplace=True)
|
|
|
|
print(f"Loaded data from {self.data.index.min().strftime('%Y-%m-%d')} to {self.data.index.max().strftime('%Y-%m-%d')}")
|
|
print(f"Available tickers: {len(self.available_tickers)}")
|
|
|
|
def get_stock_price(self, ticker, date):
|
|
"""Get stock price for a specific ticker and date"""
|
|
import os
|
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
|
backtest_dir = os.path.dirname(os.path.dirname(script_dir))
|
|
db_path = os.path.join(backtest_dir, 'data', 'stock_data.db')
|
|
conn = sqlite3.connect(db_path)
|
|
|
|
query = f'''
|
|
SELECT close FROM {ticker.lower()}
|
|
WHERE date <= ?
|
|
ORDER BY date DESC
|
|
LIMIT 1
|
|
'''
|
|
|
|
cursor = conn.cursor()
|
|
cursor.execute(query, (date.strftime('%Y-%m-%d'),))
|
|
result = cursor.fetchone()
|
|
conn.close()
|
|
|
|
return result[0] if result else None
|
|
|
|
def calculate_volatility(self, ticker, start_date, end_date):
|
|
"""Calculate historical volatility"""
|
|
import os
|
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
|
backtest_dir = os.path.dirname(os.path.dirname(script_dir))
|
|
db_path = os.path.join(backtest_dir, 'data', 'stock_data.db')
|
|
conn = sqlite3.connect(db_path)
|
|
|
|
try:
|
|
query = f'''
|
|
SELECT date, close FROM {ticker.lower()}
|
|
WHERE date >= ? AND date <= ?
|
|
ORDER BY date
|
|
'''
|
|
|
|
df = pd.read_sql_query(query, conn, params=(
|
|
start_date.strftime('%Y-%m-%d'),
|
|
end_date.strftime('%Y-%m-%d')
|
|
))
|
|
|
|
if len(df) > 10:
|
|
df['returns'] = df['close'].pct_change()
|
|
volatility = df['returns'].std() * np.sqrt(252)
|
|
conn.close()
|
|
return volatility
|
|
|
|
except Exception as e:
|
|
pass
|
|
|
|
conn.close()
|
|
return 0
|
|
|
|
def select_volatile_stocks(self, fear_start_date, fear_end_date):
|
|
"""Select top volatile stocks"""
|
|
volatilities = {}
|
|
|
|
for ticker in self.available_tickers:
|
|
vol = self.calculate_volatility(ticker, fear_start_date, fear_end_date)
|
|
if vol > 0.2: # Minimum volatility threshold
|
|
volatilities[ticker] = vol
|
|
|
|
# Sort by volatility and select top N
|
|
sorted_vol = sorted(volatilities.items(), key=lambda x: x[1], reverse=True)
|
|
top_stocks = [ticker for ticker, vol in sorted_vol[:self.top_stocks_count]]
|
|
|
|
return top_stocks
|
|
|
|
def execute_trade(self, date, action, ticker=None, shares=None, price=None, value=None):
|
|
"""Record a trade"""
|
|
self.trades.append({
|
|
'date': date,
|
|
'action': action,
|
|
'ticker': ticker,
|
|
'shares': shares,
|
|
'price': price,
|
|
'value': value
|
|
})
|
|
|
|
def calculate_portfolio_value(self, date):
|
|
"""Calculate total portfolio value"""
|
|
total_value = self.cash
|
|
|
|
for ticker, shares in self.positions.items():
|
|
if ticker == 'QQQ':
|
|
price = self.data.loc[date, 'spy_close']
|
|
else:
|
|
price = self.get_stock_price(ticker, date)
|
|
|
|
if price:
|
|
total_value += shares * price
|
|
|
|
return total_value
|
|
|
|
def check_stop_loss(self, date):
|
|
"""Check 15% stop loss"""
|
|
for ticker in list(self.positions.keys()):
|
|
if ticker == 'QQQ':
|
|
continue
|
|
|
|
current_price = self.get_stock_price(ticker, date)
|
|
if not current_price:
|
|
continue
|
|
|
|
# Find average buy price
|
|
buy_trades = [t for t in self.trades
|
|
if t['ticker'] == ticker and t['action'] in ['BUY_VOLATILE']]
|
|
if buy_trades:
|
|
total_cost = sum(t['price'] * t['shares'] for t in buy_trades)
|
|
total_shares = sum(t['shares'] for t in buy_trades)
|
|
avg_price = total_cost / total_shares
|
|
|
|
loss_pct = (current_price - avg_price) / avg_price
|
|
if loss_pct <= -self.stop_loss_threshold:
|
|
# Sell and buy QQQ
|
|
shares = self.positions[ticker]
|
|
value = shares * current_price
|
|
self.cash += value
|
|
del self.positions[ticker]
|
|
|
|
self.execute_trade(date, 'STOP_LOSS', ticker, shares, current_price, value)
|
|
|
|
# Buy QQQ
|
|
qqq_price = self.data.loc[date, 'spy_close']
|
|
qqq_shares = value / qqq_price
|
|
self.positions['QQQ'] = self.positions.get('QQQ', 0) + qqq_shares
|
|
|
|
self.execute_trade(date, 'BUY_QQQ_STOPLOSS', 'QQQ', qqq_shares, qqq_price, value)
|
|
|
|
print(f"{date.strftime('%Y-%m-%d')}: Stop loss triggered for {ticker}, loss: {loss_pct*100:.1f}%")
|
|
|
|
def start_transition(self, date, target_type, stocks=None):
|
|
"""Initialize transition plan to avoid compounding errors"""
|
|
self.transition_plan = {'type': target_type, 'stocks': stocks}
|
|
|
|
if target_type == 'CASH':
|
|
# Plan to sell all non-QQQ positions over 4 steps
|
|
self.transition_plan['positions_to_sell'] = {}
|
|
for ticker in self.positions:
|
|
if ticker != 'QQQ':
|
|
self.transition_plan['positions_to_sell'][ticker] = self.positions[ticker]
|
|
|
|
elif target_type == 'QQQ':
|
|
# Plan to sell all non-QQQ positions and convert to cash pool
|
|
self.transition_cash_pool = 0
|
|
cash_from_positions = 0
|
|
|
|
for ticker in self.positions:
|
|
if ticker != 'QQQ':
|
|
price = self.get_stock_price(ticker, date)
|
|
if price:
|
|
cash_from_positions += self.positions[ticker] * price
|
|
|
|
self.transition_cash_pool = self.cash + cash_from_positions
|
|
self.transition_plan['total_cash_to_invest'] = self.transition_cash_pool
|
|
self.transition_plan['positions_to_sell'] = {}
|
|
for ticker in self.positions:
|
|
if ticker != 'QQQ':
|
|
self.transition_plan['positions_to_sell'][ticker] = self.positions[ticker]
|
|
|
|
elif target_type == 'VOLATILE' and stocks:
|
|
# Plan to invest available cash in volatile stocks
|
|
self.transition_plan['total_cash_to_invest'] = self.cash
|
|
|
|
def gradual_transition(self, date, target_type, stocks=None):
|
|
"""Handle 4-step gradual transitions with fixed allocation"""
|
|
step_size = 1.0 / self.transition_steps
|
|
|
|
if target_type == 'CASH':
|
|
# Sell positions gradually based on initial plan
|
|
for ticker in list(self.transition_plan.get('positions_to_sell', {})):
|
|
if ticker in self.positions:
|
|
total_shares_to_sell = self.transition_plan['positions_to_sell'][ticker]
|
|
shares_to_sell = int(total_shares_to_sell * step_size)
|
|
|
|
if shares_to_sell > 0 and shares_to_sell <= self.positions[ticker]:
|
|
price = self.get_stock_price(ticker, date)
|
|
if price:
|
|
value = shares_to_sell * price
|
|
self.cash += value
|
|
self.positions[ticker] -= shares_to_sell
|
|
if self.positions[ticker] <= 0:
|
|
del self.positions[ticker]
|
|
self.execute_trade(date, 'SELL_GRADUAL', ticker, shares_to_sell, price, value)
|
|
|
|
elif target_type == 'VOLATILE' and stocks:
|
|
# Buy volatile stocks gradually using fixed cash allocation
|
|
total_cash = self.transition_plan.get('total_cash_to_invest', 0)
|
|
cash_this_step = total_cash * step_size
|
|
|
|
if cash_this_step > 0 and self.cash >= cash_this_step:
|
|
amount_per_stock = cash_this_step / len(stocks)
|
|
for ticker in stocks:
|
|
price = self.get_stock_price(ticker, date)
|
|
if price and amount_per_stock > 0:
|
|
shares = amount_per_stock / price
|
|
self.positions[ticker] = self.positions.get(ticker, 0) + shares
|
|
self.cash -= amount_per_stock
|
|
self.execute_trade(date, 'BUY_GRADUAL', ticker, shares, price, amount_per_stock)
|
|
|
|
elif target_type == 'QQQ':
|
|
# Sell positions gradually and buy QQQ with fixed allocation
|
|
# First sell positions
|
|
for ticker in list(self.transition_plan.get('positions_to_sell', {})):
|
|
if ticker in self.positions:
|
|
total_shares_to_sell = self.transition_plan['positions_to_sell'][ticker]
|
|
shares_to_sell = int(total_shares_to_sell * step_size)
|
|
|
|
if shares_to_sell > 0 and shares_to_sell <= self.positions[ticker]:
|
|
price = self.get_stock_price(ticker, date)
|
|
if price:
|
|
value = shares_to_sell * price
|
|
self.cash += value
|
|
self.positions[ticker] -= shares_to_sell
|
|
if self.positions[ticker] <= 0:
|
|
del self.positions[ticker]
|
|
self.execute_trade(date, 'SELL_GRADUAL', ticker, shares_to_sell, price, value)
|
|
|
|
# Then buy QQQ with step portion of planned cash
|
|
total_cash = self.transition_plan.get('total_cash_to_invest', 0)
|
|
cash_this_step = total_cash * step_size
|
|
|
|
if cash_this_step > 0 and self.cash >= cash_this_step:
|
|
qqq_price = self.data.loc[date, 'spy_close']
|
|
qqq_shares = cash_this_step / qqq_price
|
|
|
|
self.positions['QQQ'] = self.positions.get('QQQ', 0) + qqq_shares
|
|
self.cash -= cash_this_step
|
|
self.execute_trade(date, 'BUY_GRADUAL', 'QQQ', qqq_shares, qqq_price, cash_this_step)
|
|
|
|
def run_backtest(self):
|
|
"""Run the enhanced strategy backtest"""
|
|
print("Running Enhanced Emotional Damage Strategy...")
|
|
|
|
self.get_data()
|
|
|
|
# Start with 100% QQQ
|
|
first_date = self.data.index[0]
|
|
qqq_price = self.data.loc[first_date, 'spy_close']
|
|
qqq_shares = self.initial_capital / qqq_price
|
|
self.positions['QQQ'] = qqq_shares
|
|
|
|
fear_start_date = None
|
|
|
|
for date, row in self.data.iterrows():
|
|
fg_index = row['fear_greed_index']
|
|
|
|
# Check stop loss
|
|
self.check_stop_loss(date)
|
|
|
|
if self.state == 'QQQ_HOLD':
|
|
# Check for fear threshold
|
|
if fg_index < self.fear_threshold:
|
|
fear_start_date = date
|
|
self.state = 'FEAR_TRANSITION'
|
|
self.current_step = 0
|
|
self.start_transition(date, 'CASH')
|
|
print(f"{date.strftime('%Y-%m-%d')}: Fear threshold hit ({fg_index:.1f}), starting transition to cash")
|
|
|
|
elif self.state == 'FEAR_TRANSITION':
|
|
# Gradual transition to cash
|
|
self.gradual_transition(date, 'CASH')
|
|
self.current_step += 1
|
|
|
|
if self.current_step >= self.transition_steps:
|
|
self.state = 'CASH_WAIT'
|
|
print(f"{date.strftime('%Y-%m-%d')}: Transition to cash complete")
|
|
|
|
elif self.state == 'CASH_WAIT':
|
|
# Wait for recovery, then select volatile stocks
|
|
if fg_index >= self.fear_threshold and fear_start_date:
|
|
# Select top volatile stocks
|
|
top_stocks = self.select_volatile_stocks(fear_start_date, date)
|
|
|
|
if top_stocks:
|
|
self.state = 'GREED_TRANSITION'
|
|
self.current_step = 0
|
|
self.transition_stocks = top_stocks
|
|
self.start_transition(date, 'VOLATILE', top_stocks)
|
|
print(f"{date.strftime('%Y-%m-%d')}: Fear recovered, starting transition to volatile stocks: {top_stocks}")
|
|
else:
|
|
# No suitable stocks, go back to QQQ
|
|
self.state = 'QQQ_TRANSITION'
|
|
self.current_step = 0
|
|
self.start_transition(date, 'QQQ')
|
|
print(f"{date.strftime('%Y-%m-%d')}: Fear recovered, no suitable stocks, returning to QQQ")
|
|
|
|
elif self.state == 'GREED_TRANSITION':
|
|
# Gradual transition to volatile stocks
|
|
self.gradual_transition(date, 'VOLATILE', self.transition_stocks)
|
|
self.current_step += 1
|
|
|
|
if self.current_step >= self.transition_steps:
|
|
self.state = 'VOLATILE_STOCKS'
|
|
print(f"{date.strftime('%Y-%m-%d')}: Transition to volatile stocks complete")
|
|
|
|
elif self.state == 'VOLATILE_STOCKS':
|
|
# Check for greed threshold
|
|
if fg_index > self.greed_threshold:
|
|
self.state = 'QQQ_TRANSITION'
|
|
self.current_step = 0
|
|
self.start_transition(date, 'QQQ')
|
|
print(f"{date.strftime('%Y-%m-%d')}: Greed threshold hit ({fg_index:.1f}), starting transition to QQQ")
|
|
|
|
elif self.state == 'QQQ_TRANSITION':
|
|
# Gradual transition back to QQQ
|
|
self.gradual_transition(date, 'QQQ')
|
|
self.current_step += 1
|
|
|
|
if self.current_step >= self.transition_steps:
|
|
self.state = 'QQQ_HOLD'
|
|
print(f"{date.strftime('%Y-%m-%d')}: Transition to QQQ complete")
|
|
|
|
# Record portfolio value
|
|
portfolio_value = self.calculate_portfolio_value(date)
|
|
self.portfolio_value.append({
|
|
'date': date,
|
|
'value': portfolio_value,
|
|
'state': self.state,
|
|
'fg_index': fg_index
|
|
})
|
|
|
|
print(f"Backtest completed! Total trades: {len(self.trades)}")
|
|
|
|
def calculate_performance_metrics(self, returns):
|
|
"""Calculate performance metrics"""
|
|
total_return = (returns.iloc[-1] / returns.iloc[0] - 1) * 100
|
|
annual_return = ((returns.iloc[-1] / returns.iloc[0]) ** (252 / len(returns)) - 1) * 100
|
|
|
|
# Calculate max drawdown
|
|
peak = returns.expanding().max()
|
|
drawdown = (returns - peak) / peak
|
|
max_drawdown = drawdown.min() * 100
|
|
|
|
# Find max drawdown period
|
|
max_dd_date = drawdown.idxmin()
|
|
|
|
# Calculate Sharpe ratio
|
|
daily_returns = returns.pct_change().dropna()
|
|
sharpe_ratio = np.sqrt(252) * daily_returns.mean() / daily_returns.std()
|
|
|
|
# Annual returns by year
|
|
annual_rets = {}
|
|
for year in returns.index.year.unique():
|
|
year_data = returns[returns.index.year == year]
|
|
if len(year_data) > 1:
|
|
year_return = (year_data.iloc[-1] / year_data.iloc[0] - 1) * 100
|
|
annual_rets[year] = year_return
|
|
|
|
return {
|
|
'total_return': total_return,
|
|
'annual_return': annual_return,
|
|
'max_drawdown': max_drawdown,
|
|
'max_drawdown_date': max_dd_date,
|
|
'sharpe_ratio': sharpe_ratio,
|
|
'annual_returns': annual_rets
|
|
}
|
|
|
|
def run_enhanced_backtest():
|
|
"""Run the enhanced strategy"""
|
|
|
|
strategy = EnhancedEmotionalDamageStrategy(initial_capital=100000)
|
|
strategy.run_backtest()
|
|
|
|
# Convert results
|
|
portfolio_df = pd.DataFrame(strategy.portfolio_value)
|
|
portfolio_df.set_index('date', inplace=True)
|
|
|
|
# Get benchmark data
|
|
import os
|
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
|
backtest_dir = os.path.dirname(os.path.dirname(script_dir))
|
|
db_path = os.path.join(backtest_dir, 'data', 'stock_data.db')
|
|
conn = sqlite3.connect(db_path)
|
|
benchmark_data = pd.read_sql_query('''
|
|
SELECT date, spy_close
|
|
FROM fear_greed_data
|
|
ORDER BY date
|
|
''', conn)
|
|
benchmark_data['date'] = pd.to_datetime(benchmark_data['date'])
|
|
benchmark_data.set_index('date', inplace=True)
|
|
conn.close()
|
|
|
|
# Align dates
|
|
common_dates = portfolio_df.index.intersection(benchmark_data.index)
|
|
portfolio_df = portfolio_df.loc[common_dates]
|
|
benchmark_data = benchmark_data.loc[common_dates]
|
|
|
|
# Normalize benchmarks
|
|
start_value = 100000
|
|
benchmark_data['qqq_value'] = start_value * (benchmark_data['spy_close'] / benchmark_data['spy_close'].iloc[0])
|
|
benchmark_data['spy_value'] = start_value * (benchmark_data['spy_close'] / benchmark_data['spy_close'].iloc[0])
|
|
|
|
# Calculate metrics
|
|
strategy_metrics = strategy.calculate_performance_metrics(portfolio_df['value'])
|
|
qqq_metrics = strategy.calculate_performance_metrics(benchmark_data['qqq_value'])
|
|
spy_metrics = strategy.calculate_performance_metrics(benchmark_data['spy_value'])
|
|
|
|
return {
|
|
'strategy': strategy,
|
|
'portfolio_df': portfolio_df,
|
|
'benchmark_data': benchmark_data,
|
|
'strategy_metrics': strategy_metrics,
|
|
'qqq_metrics': qqq_metrics,
|
|
'spy_metrics': spy_metrics
|
|
}
|
|
|
|
if __name__ == "__main__":
|
|
results = run_enhanced_backtest()
|
|
print("Enhanced backtest completed!") |