用值填充非常大的数据框的快速方法

Question

我有一个非常大的数据框，其中有 100 年的日期作为列标题（即 ~36500 列）和 100 年的日期作为索引（即 ~36500 行）。我有一个函数可以计算数据帧的每个元素的值，该函数需要运行 36500^2 次。

好吧，问题不在于该函数是否非常快，而在于向数据帧分配值。即使我以这种方式分配一个常量，每 6 次分配也需要大约 1 秒。显然，正如你所见，我很厚：

for i, row in df_mBase.iterrows():
    for idx, val in enumerate(row):
        df_mBase.ix[i][idx] = 1
    print(i)

通常在 C/Java 中，我会简单地循环遍历 36500x36500 双循环，并通过索引直接访问预先分配的内存，这可以在恒定时间内实现，几乎没有任何开销。但这似乎不是 python 中的一个选项？

将这些数据存储在数据框中的最快方法是什么？无论是否是Pythonian，我只追求速度——我不关心优雅。

Answer 1

有几个原因可能会导致速度变慢

.ix

.ix是一个魔术类型索引器，它可以执行标签索引和位置索引，但将.loc由于基于标签和.iloc基于索引的更严格，将被弃用。我认为.ix在幕后做了很多魔法来确定是否需要标签或基于位置的索引

.iterrows

返回一个（新的？）Series为每一行基于列的迭代可能会更快，因为.iteritems迭代列

[][]

df_mBase.ix[i][idx]返回 a Series，然后从中获取元素idx，并为其分配值 1。

df_mBase.loc[i, idx] = 1

应该改进这个

标杆管理

import pandas as pd

import itertools
import timeit


def generate_dummy_data(years=1):
    period = pd.Timedelta(365 * years, unit='D')

    start = pd.Timestamp('19000101')
    offset = pd.Timedelta(10, unit='h')

    dates1 = pd.DatetimeIndex(start=start, end=start + period, freq='d')
    dates2 = pd.DatetimeIndex(start=start + offset, end=start + offset + period, freq='d')

    return pd.DataFrame(index=dates1, columns=dates2, dtype=float)


def assign_original(df_orig):
    df_new = df_orig.copy(deep=True)
    for i, row in df_new.iterrows():
        for idx, val in enumerate(row):
            df_new.ix[i][idx] = 1
    return df_new


def assign_other(df_orig):
    df_new = df_orig.copy(deep=True)
    for (i, idx_i), (j, idx_j) in itertools.product(enumerate(df_new.index), enumerate(df_new.columns)):
        df_new[idx_j][idx_i] = 1
    return df_new


def assign_loc(df_orig):
    df_new = df_orig.copy(deep=True)
    for i, row in df_new.iterrows():
        for idx, val in enumerate(row):
            df_new.loc[i][idx] = 1
    return df_new


def assign_loc_product(df_orig):
    df_new = df_orig.copy(deep=True)
    for i, j in itertools.product(df_new.index, df_new.columns):
        df_new.loc[i, j] = 1
    return df_new


def assign_iloc_product(df_orig):
    df_new = df_orig.copy(deep=True)
    for (i, idx_i), (j, idx_j) in itertools.product(enumerate(df_new.index), enumerate(df_new.columns)):
        df_new.iloc[i, j] = 1
    return df_new


def assign_iloc_product_range(df_orig):
    df_new = df_orig.copy(deep=True)
    for i, j in itertools.product(range(len(df_new.index)), range(len(df_new.columns))):
        df_new.iloc[i, j] = 1
    return df_new


def assign_index(df_orig):
    df_new = df_orig.copy(deep=True)
    for (i, idx_i), (j, idx_j) in itertools.product(enumerate(df_new.index), enumerate(df_new.columns)):
        df_new[idx_j][idx_i] = 1
    return df_new


def assign_column(df_orig):
    df_new = df_orig.copy(deep=True)
    for c, column in df_new.iteritems():
        for idx, val in enumerate(column):
            df_new[c][idx] = 1
    return df_new


def assign_column2(df_orig):
    df_new = df_orig.copy(deep=True)
    for c, column in df_new.iteritems():
        for idx, val in enumerate(column):
            column[idx] = 1
    return df_new


def assign_itertuples(df_orig):
    df_new = df_orig.copy(deep=True)
    for i, row in enumerate(df_new.itertuples()):
        for idx, val in enumerate(row[1:]):
            df_new.iloc[i, idx] = 1
    return df_new


def assign_applymap(df_orig):
    df_new = df_orig.copy(deep=True)
    df_new = df_new.applymap(lambda x: 1)
    return df_new


def assign_vectorized(df_orig):
    df_new = df_orig.copy(deep=True)
    for i in df_new:
        df_new[i] = 1
    return df_new


methods = [
    ('assign_original', assign_original),
    ('assign_loc', assign_loc),
    ('assign_loc_product', assign_loc_product),
    ('assign_iloc_product', assign_iloc_product),
    ('assign_iloc_product_range', assign_iloc_product_range),
    ('assign_index', assign_index),
    ('assign_column', assign_column),
    ('assign_column2', assign_column2),
    ('assign_itertuples', assign_itertuples),
    ('assign_vectorized', assign_vectorized),
    ('assign_applymap', assign_applymap),
]


def get_timings(period=1, methods=()):
    print('=' * 10)
    print(f'generating timings for a period of {period} years')
    df_orig = generate_dummy_data(period)
    df_orig.info(verbose=False)
    repeats = 1
    for method_name, method in methods:
        result = pd.DataFrame()

        def my_method():
            """
            This looks a bit icky, but is the best way I found to make sure the values are really changed,
            and not just on a copy of a DataFrame
            """
            nonlocal result
            result = method(df_orig)

        t = timeit.Timer(my_method).timeit(number=repeats)

        assert result.iloc[3, 3] == 1

        print(f'{method_name} took {t / repeats} seconds')
        yield (method_name, {'time': t, 'memory': result.memory_usage(deep=True).sum()/1024})


periods = [0.03, 0.1, 0.3, 1, 3]


results = {period: dict(get_timings(period, methods)) for period in periods}

print(results)

timings_dict = {period: {k: v['time'] for k, v in result.items()} for period, result in results.items()}

df = pd.DataFrame.from_dict(timings_dict)
df.transpose().plot(logy=True).figure.savefig('test.png')

                              0.03        0.1         0.3         1.0         3.0
assign_applymap               0.001989    0.009862    0.018018    0.105569    0.549511
assign_vectorized             0.002974    0.008428    0.035994    0.162565    3.810138
assign_index                  0.013717    0.137134    1.288852    14.190128   111.102662
assign_column2                0.026260    0.186588    1.664345    19.204453   143.103077
assign_column                 0.016811    0.212158    1.838733    21.053627   153.827845
assign_itertuples             0.025130    0.249886    2.125968    24.639593   185.975111
assign_iloc_product_range     0.026982    0.247069    2.199019    23.902244   186.548500
assign_iloc_product           0.021225    0.233454    2.437183    25.143673   218.849143
assign_loc_product            0.018743    0.290104    2.515379    32.778794   258.244436
assign_loc                    0.029050    0.349551    2.822797    32.087433   294.052933
assign_original               0.034315    0.337207    2.714154    30.361072   332.327008

结论

如果您可以使用矢量化，请这样做。根据计算的不同，您可以使用其他方法。如果您只需要使用的值，则applymap似乎最快。如果您也需要索引和/或列，请使用列

如果您无法矢量化，则工作速度最快，紧随其后的是对df[column][index] = x列进行迭代df.iteritems()

Answer 2

您应该在本机 python 或 numpy 中创建数据结构，并将数据传递给 DataFrame 构造函数。如果您的函数可以使用 numpy 的函数/操作编写，那么您可以使用 numpy 的向量化性质来避免循环所有索引。

这是一个带有组合函数的示例：

import numpy as np
import pandas as pd
import datetime as dt
import dateutil as du

dates = [dt.date(2017, 1, 1) - du.relativedelta.relativedelta(days=i) for i in range(36500)]
data = np.zeros((36500,36500), dtype=np.uint8)

def my_func(i, j):
    return (sum(divmod(i,j)) - sum(divmod(j,i))) % 255

for i in range(1, 36500):
    for j in range(1, 36500):
        data[i,j] = my_func(i,j)

df = pd.DataFrame(data, columns=dates, index=dates)

df.head(5)
#returns:

            2017-08-21  2017-08-20  2017-08-19  2017-08-18  2017-08-17  \
2017-08-21           0           0           0           0           0
2017-08-20           0           0         254         253         252
2017-08-19           0           1           0           0           0
2017-08-18           0           2           0           0           1
2017-08-17           0           3           0         254           0

               ...      1917-09-19  1917-09-18  1917-09-17  1917-09-16
2017-08-21     ...               0           0           0           0
2017-08-20     ...             225         224         223         222
2017-08-19     ...             114         113         113         112
2017-08-18     ...              77          76          77          76
2017-08-17     ...              60          59          58          57