为什么`np.sum(range(N))`非常慢？

Question

下面是我用来测试的脚本，在我所知道的情况下（主要来自视频），以及我在我的机器上得到的结果（python 3.8.10,numpy 1.19.5):

更新脚本：

import numpy as np
from timeit import timeit

N = 10_000_000
repetition = 10

def sum0(N = N):
    s = 0
    i = 0
    while i < N: # condition is checked in python
        s += i
        i += 1 # both additions are done in python
    return s

def sum1(N = N):
    s = 0
    for i in range(N): # increment in C
        s += i # addition in python
    return s

def sum2(N = N):
    return sum(range(N)) # everything in C

def sum3(N = N):
    return sum(list(range(N)))

def sum4(N = N):
    return np.sum(range(N)) # very slow np.array conversion

def sum5(N = N):
    # much faster np.array conversion
    return np.sum(np.fromiter(range(N),dtype = np.int)) 

def sum6(N = N):
    # possibly slow conversion to Py_long from np.int
    return sum(np.arange(N))

def sum7(N = N):
    # list returns a list of np.int-s
    return sum(list(np.arange(N)))

def sum7v2(N = N):
    # tolist conversion to python int seems faster than the implicit conversion
    # in sum(list()) (tolist returns a list of python int-s)
    return sum(np.arange(N).tolist())

def sum8(N = N):
    return np.sum(np.arange(N)) # everything in numpy (fortran libblas?)

def array_basic(N = N):
    return np.array(range(N))

def array_dtype(N = N):
    return np.array(range(N),dtype = np.int)

def array_iter(N = N):
    # np.sum's source code mentions to use fromiter to convert from generators
    return np.fromiter(range(N),dtype = np.int)

print(f"while loop:         {timeit(sum0, number = repetition)}")
print(f"for loop:           {timeit(sum1, number = repetition)}")
print(f"sum_range:          {timeit(sum2, number = repetition)}")
print(f"sum_rangelist:      {timeit(sum3, number = repetition)}")
print(f"npsum_range:        {timeit(sum4, number = repetition)}")
print(f"npsum_fromiterrange:{timeit(sum5, number = repetition)}")
print(f"sum_arange:         {timeit(sum6, number = repetition)}")
print(f"sum_list_arange:    {timeit(sum7, number = repetition)}")
print(f"sum_arange_tolist:  {timeit(sum7v2, number = repetition)}")
print(f"npsum_arange:       {timeit(sum8, number = repetition)}")
print(f"array_basic:        {timeit(array_basic, number = repetition)}")
print(f"array_dtype:        {timeit(array_dtype, number = repetition)}")
print(f"array_iter:         {timeit(array_iter,  number = repetition)}")

# Example output:
#
# while loop:         9.249794696999743
# for loop:           6.026467555000636
# sum_range:          1.4830789409988938
# sum_rangelist:      3.6745876889999636
# npsum_range:        16.216972655000063
# npsum_fromiterrange:3.47655400199983
# sum_arange:         16.656015603000924
# sum_list_arange:    19.500842117000502
# sum_arange_tolist:  4.004777374000696
# npsum_arange:       0.2332638230000157
# array_basic:        16.1631146109994
# array_dtype:        16.550737804000164
# array_iter:         3.9803170430004684

Answer 1

让我们看看我能不能总结一下结果。

sum可以处理任何可迭代的值，反复请求下一个值并将其添加。range是一个生成器，它很乐意提供下一个值

# sum_range:          1.4830789409988938

从一个范围中列出一个列表需要时间：

# sum_rangelist:      3.6745876889999636

%%timeit x = list(range(N))
    ...: sum(x)

# npsum_range:        16.216972655000063

# npsum_fromiterrange:3.47655400199983

numpy数组上的迭代比列表慢，因为它必须“解框”每个元素。

# sum_arange:         16.656015603000924

同样，从数组中生成列表也很慢；类似于python级别的迭代。

# sum_list_arange:    19.500842117000502

arr.tolist()相对较快，可以在编译后的代码中创建纯python列表。所以速度类似于从范围列出一个列表。

# sum_arange_tolist:  4.004777374000696

# npsum_arange:       0.2332638230000157

# array_basic:        16.1631146109994
# array_dtype:        16.550737804000164
# array_iter:         3.9803170430004684