bt/price_yahoo.py at master · cujeu/bt · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
import pandas as pd
from yahoofinancials import YahooFinancials
import datetime

from config import *
from sp500_symbols import *

def get_price_yahoo(all_tickers, years):
    #,"CSCO","AMZN","INTC"]
    # extracting stock data (historical close price) for the stocks identified
    close_prices = pd.DataFrame()
    #end_date = (datetime.date.today()-datetime.timedelta(1)).strftime('%Y-%m-%d')
    #end_date = (datetime.date.today()).strftime('%Y-%m-%d')
    #beg_date = datetime.datetime(2010, 1, 1).strftime('%Y-%m-%d')
    #beg_date = (datetime.date.today()-datetime.timedelta(365*years)).strftime('%Y-%m-%d')
    today = datetime.datetime.today().date()
    shift = datetime.timedelta(max(1,(today.weekday() + 6) % 7 - 3))
    end_date = (today- shift + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
    beg_date = (datetime.date.today()-datetime.timedelta(365*years)).strftime('%Y-%m-%d')

    cp_tickers = all_tickers
    attempt = 0
    bad_list = ['ADYEN', 'KSPI','FB','ZY','DYNS','KVSB']
    drop = bad_list
    # names in cols are features of get_historical_price_data
    cols = ["formatted_date","open", "high", "low", "close","adjclose","volume"]
    #cols = ["Date","Open", "High", "Low", "Close","Adj Close","Volume"]
    # other order: Date,Open,High,Low,Close,Adj Close,Volume
    # anther order Date,Open,High,Low,Close,Volume,Adj Close
    while len(cp_tickers) != 0 and attempt < 2:
        print("-----------------")
        print("attempt number ",attempt)
        print("-----------------")
        cp_tickers = [j for j in cp_tickers if j not in drop]
        for i in range(len(cp_tickers)):
            try:
                yahoo_financials = YahooFinancials(cp_tickers[i])
                json_obj = yahoo_financials.get_historical_price_data(beg_date,end_date,"daily")
                ohlv = json_obj[cp_tickers[i]]['prices']
                # now pick formatted_date, open, high, low, close, adjclose
                temp_df = pd.DataFrame(ohlv)[cols]
                temp_df.set_index("formatted_date",inplace=True)
                temp_df.sort_index(inplace = True, ascending=False)
                ##temp_df = temp_df.sort_index(axis=1 ,ascending=False)
                file_name = conf_rawdata_path + cp_tickers[i] + '.csv'
                print("writing " + file_name)
                temp_df.to_csv(file_name)
                ##print(temp_df.head())
                temp_df.rename(columns={'open' : 'Open', 'high' : 'High', 'low' : 'Low',
                                   'close' : 'Close', 'adjclose' : 'Adj Close',
                                   'volume' : 'Volume'}, inplace=True)
                temp_df.index.names = ['Date']
                reversed_df = temp_df.iloc[::-1]
                file_name = conf_backtest_data_path + cp_tickers[i] + '.csv'
                reversed_df.to_csv(file_name)

                """
                #here is howto add all adjclose + ticker into one table
                #json_obj = yahoo_financials.get_historical_stock_data(beg_date,end_date,"daily")
                ohlv = json_obj[cp_tickers[i]]['prices']
                temp = pd.DataFrame(ohlv)[["formatted_date","adjclose"]]
                temp.set_index("formatted_date",inplace=True)
                temp2 = temp[~temp.index.duplicated(keep='first')]
                close_prices[cp_tickers[i]] = temp2["adjclose"]
                """
                drop.append(cp_tickers[i])
            except:
                print(cp_tickers[i]," :failed to fetch data...retrying")
                continue
        attempt+=1
    #end of while

def download_etf_prices(years):
    tlist = get_etf_symbols()
    print("download_etf")
    get_price_yahoo(tlist, years)

# close watch the list from https://www.getrevue.co/profile/lonecapital
def download_mywatch_prices(years):
    tlist = get_mywatch_symbols()
    print("download_mywatch")
    get_price_yahoo(tlist, years)


def download_all_ark_prices(years):
    DATETIME_FORMAT = '%y%m%d'
    date_str = datetime.datetime.strftime(datetime.datetime.now(), DATETIME_FORMAT)
    tlist = get_all_ark_symbol(date_str)
    #print(tlist)
    print("download_all ark")
    get_price_yahoo(tlist, years)

def download_ark_prices(years):
    tlist = get_ark_symbols()
    print("download ark")
    get_price_yahoo(tlist, years)

def download_sector_prices(sector_name, years):
    tlist = get_sector_symbols(sector_name)
    print("download sector")
    get_price_yahoo(tlist, years)

def download_russell_prices(years):
    tlist = get_russell_symbols()
    print("download russell")
    get_price_yahoo(tlist, years)

def download_growth_russell_prices(years):
    tlist = get_growth_russell_symbols()
    print("download growth")
    get_price_yahoo(tlist, years)

def download_all_prices(years):
    tlist = get_all_symbols()
    print("download all")
    get_price_yahoo(tlist, years)

if __name__ == "__main__":
    ##download_sector_prices("Information Technology",10)
    ##download_sector_prices("Consumer Staples",10)
    #download_ark_prices(10)

    #download_all_prices(10)    #sp500 data
    #download_growth_russell_prices(10)

    download_russell_prices(10)
    download_etf_prices(10)
    download_mywatch_prices(10)
    #download_all_ark_prices(10)