CBBI/fetch_bitcoin_data.py at main · jthole/CBBI · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import cli_ui
import requests
from filecache import filecache

from globals import *
from utils import *


@filecache(3600 * 2)  # 2 hours cache
def fetch_bitcoin_data() -> pd.DataFrame:
    """
    Fetches historical Bitcoin data into a DataFrame.
    Very early data is discarded due to high volatility.

    Returns:
        DataFrame containing Bitcoin data.
    """
    cli_ui.info_2('Requesting historical Bitcoin data')

    response = requests.get('https://api.blockchair.com/bitcoin/blocks', {
        'a': 'date,count(),min(id),max(id),sum(generation),sum(generation_usd)',
        's': 'date(desc)',
    }, timeout=HTTP_TIMEOUT)
    response.raise_for_status()
    response_json = response.json()

    df = pd.DataFrame(response_json['data'][::-1])
    df.rename(columns={
        'date': 'Date',
        'count()': 'TotalBlocks',
        'min(id)': 'MinBlockID',
        'max(id)': 'MaxBlockID',
        'sum(generation)': 'TotalGeneration',
        'sum(generation_usd)': 'TotalGenerationUSD'
    }, inplace=True)

    df['Date'] = pd.to_datetime(df['Date'])
    df['TotalGeneration'] /= 1e8
    df['BlockGeneration'] = df['TotalGeneration'] / df['TotalBlocks']
    df['BlockGenerationUSD'] = df['TotalGenerationUSD'] / df['TotalBlocks']

    df = df.merge(fetch_price_data(), on='Date', how='left')
    df.loc[df['Price'].isna(), 'Price'] = df['BlockGenerationUSD'] / df['BlockGeneration']
    df['PriceLog'] = np.log(df['Price'])
    df['PriceLogInterp'] = np.interp(df['PriceLog'],
                                     (df['PriceLog'].min(), df['PriceLog'].max()),
                                     (0, 1))

    df = df.loc[df['Date'] >= '2011-06-27']
    df.reset_index(drop=True, inplace=True)

    df = fix_current_day_data(df)
    df = add_block_halving_data(df)
    df = mark_highs_lows(df, 'Price', False, round(365 * 2), 365)
    df = mark_days_since(df, ['PriceHigh', 'PriceLow', 'Halving'])

    current_price = df['Price'].tail(1).values[0]
    cli_ui.info_1(f'Current Bitcoin price: ${round(current_price):,}')

    return df


def fetch_price_data() -> pd.DataFrame:
    response = requests.get('https://api.coinmarketcap.com/data-api/v3/cryptocurrency/detail/chart', {
        'id': 1,
        'range': 'ALL',
    }, timeout=HTTP_TIMEOUT)

    response.raise_for_status()
    response_json = response.json()
    response_x = response_json['data']['points'].keys()
    response_y = [value['v'][0] for value in response_json['data']['points'].values()]

    df = pd.DataFrame({
        'Date': response_x,
        'Price': response_y,
    })
    df['Date'] = pd.to_datetime(df['Date'], unit='s').dt.tz_localize(None).dt.floor('d')
    df.sort_values(by='Date', inplace=True)
    df.drop_duplicates('Date', keep='last', inplace=True)

    return df


def fix_current_day_data(df: pd.DataFrame) -> pd.DataFrame:
    row = df.iloc[-1].copy()

    target_total_blocks = 24 * 6
    target_scale = target_total_blocks / row['TotalBlocks']

    for col_name in ['TotalBlocks', 'TotalGeneration', 'TotalGenerationUSD']:
        row[col_name] *= target_scale

    df.iloc[-1] = row
    return df


def add_block_halving_data(df: pd.DataFrame) -> pd.DataFrame:
    reward_halving_every = 210000
    current_block_halving_id = reward_halving_every
    current_block_production = 50
    df['Halving'] = 0
    df['NextHalvingBlock'] = current_block_halving_id

    while True:
        df.loc[(current_block_halving_id - reward_halving_every) <= df[
            'MaxBlockID'], 'BlockGeneration'] = current_block_production

        block_halving_row = df[(df['MinBlockID'] <= current_block_halving_id) &
                               (df['MaxBlockID'] >= current_block_halving_id)].squeeze()

        if block_halving_row.shape[0] == 0:
            break

        current_block_halving_id += reward_halving_every
        current_block_production /= 2
        df.loc[block_halving_row.name, 'Halving'] = 1
        df.loc[df.index > block_halving_row.name, 'NextHalvingBlock'] = current_block_halving_id

    df['DaysToHalving'] = pd.TimedeltaIndex((df['NextHalvingBlock'] - df['MaxBlockID']) / (24 * 6), unit='D')
    df['NextHalvingDate'] = df['Date'] + df['DaysToHalving']
    return df