使用Python多处理的高内存使用率

Question

import multiprocessing as mp
import time

def calculate(num):
    l = [num*num for num in range(num)]
    s = sum(l)
    del l       # delete lists as an  option
    return s

if __name__ == "__main__":
    pool = mp.Pool(processes=2)
    time.sleep(5)
    print "launching calculation"
    num_tasks = 1000
    tasks =  [pool.apply_async(calculate,(i,)) for i in range(num_tasks)]
    for f in tasks:    
        print f.get(5)
    print "calculation finished"
    time.sleep(10)
    print "closing  pool"
    pool.close()
    print "closed pool"
    print "joining pool"
    pool.join()
    print "joined pool"
    time.sleep(5)

+---------------+----------------------+----------------------+
|  num_tasks    |  memory with del     | memory without del   |
|               | proc_1   | proc_2    | proc_1   | proc_2    |
+---------------+----------------------+----------------------+
| 1000          | 4884     | 4694      | 4892     | 4952      |
| 5000          | 5588     | 5596      | 6140     | 6268      |
| 10000         | 6528     | 6580      | 6640     | 6644      |
+---------------+----------------------+----------------------+

null

Answer 1

我做了很多研究，但找不到解决问题的办法。但是有一个不错的工作，可以防止内存井喷，成本很小，尤其是在服务器端长时间运行的代码上。

解决方案基本上是在完成固定数量的任务后重新启动各个辅助进程。python中的pool类将MaxTaskSperChild作为参数。您可以指定maxtasksperchild=1000，从而限制每个子进程运行1000个任务。到达MaxTaskSperChild号后，池将刷新其子进程。对于最大任务使用一个谨慎的数字，可以平衡所消耗的最大内存和与重新启动后端进程相关的启动成本。池构造如下：

pool = mp.Pool(processes=2,maxtasksperchild=1000)

我把我的全部解决方案放在这里，这样它就可以对其他人有用！

import multiprocessing as mp
import time

def calculate(num):
    l = [num*num for num in range(num)]
    s = sum(l)
    del l       # delete lists as an  option
    return s

if __name__ == "__main__":

    # fix is in the following line #
    pool = mp.Pool(processes=2,maxtasksperchild=1000)

    time.sleep(5)
    print "launching calculation"
    num_tasks = 1000
    tasks =  [pool.apply_async(calculate,(i,)) for i in range(num_tasks)]
    for f in tasks:    
        print f.get(5)
    print "calculation finished"
    time.sleep(10)
    print "closing  pool"
    pool.close()
    print "closed pool"
    print "joining pool"
    pool.join()
    print "joined pool"
    time.sleep(5)