python3.6_多程序_multiprocessing.pool_concurrent.futures_ProcessPoolExecutor

轉載註明來源: 本文連結 來自osnosn的部落格，寫於 2020-06-27.

多程序的多種寫法，在大量任務的情況下，效率的對比。

import time
from multiprocessing.pool import Pool
from concurrent.futures import as_completed, ProcessPoolExecutor

NUMBERS = range(1, 60000)
K = 50

def f(x):
    r = 0
    for k in range(1, K+2):
        r += x ** (1 / k**1.5)
    return ['xx',r]

if __name__ == '__main__':
    if 1:
        print('-------------------\n no multiProcessing:')
        start = time.time()
        l = []
        for nn in NUMBERS:
            result=f(nn)
            l.append(result)
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n multiprocessing.pool.Pool:')
        start = time.time()
        l = []
        pool = Pool(4)
        for num, result in zip(NUMBERS, pool.map(f, NUMBERS)):
            l.append(result)
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n multiprocessing.pool.Pool, apply_async:')
        start = time.time()
        l = []
        pool = Pool(4)
        res=[]
        for nn in NUMBERS:
            res.append(pool.apply_async(f,(nn,)))
        pool.close()
        print('middle COST: {}'.format(time.time() - start))
        pool.join()
        for rr in res:
            l.append(rr.get())
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n multiprocessing.pool.Pool, apply_async,maxtasksperchild=1000 :')
        start = time.time()
        l = []
        pool = Pool(4,maxtasksperchild=1000)
        res=[]
        for nn in NUMBERS:
            res.append(pool.apply_async(f,(nn,)))
        pool.close()
        print('middle COST: {}'.format(time.time() - start))
        pool.join()
        for rr in res:
            l.append(rr.get())
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n ProcessPoolExecutor with chunksize,1/4:')
        start = time.time()
        l = []
        with ProcessPoolExecutor(max_workers=4) as executor:
            chunksize, extra = divmod(len(NUMBERS), executor._max_workers * 4)
            print('chunksize',chunksize)
            for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
                l.append(result)
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n ProcessPoolExecutor with chunksize,1/10:')
        start = time.time()
        l = []
        with ProcessPoolExecutor(max_workers=4) as executor:
            chunksize, extra = divmod(len(NUMBERS), executor._max_workers * 10)
            print('chunksize',chunksize)
            for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
                l.append(result)
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n ProcessPoolExecutor with chunksize,1/100:')
        start = time.time()
        l = []
        with ProcessPoolExecutor(max_workers=4) as executor:
            chunksize, extra = divmod(len(NUMBERS), executor._max_workers * 100)
            print('chunksize',chunksize)
            for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
                l.append(result)
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n ProcessPoolExecutor with chunksize,1/300:')
        start = time.time()
        l = []
        with ProcessPoolExecutor(max_workers=4) as executor:
            chunksize, extra = divmod(len(NUMBERS), executor._max_workers * 300)
            print('chunksize',chunksize)
            for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
                l.append(result)
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n ProcessPoolExecutor with chunksize,500:')
        start = time.time()
        l = []
        with ProcessPoolExecutor(max_workers=4) as executor:
            chunksize=500
            print('chunksize',chunksize)
            for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
                l.append(result)
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n ProcessPoolExecutor submit:')
        start = time.time()
        pool_res=[]
        executor=ProcessPoolExecutor(max_workers=4)
        for nn in NUMBERS:
            res=executor.submit(f,nn)
            pool_res.append(res)
        print('middle COST: {}'.format(time.time() - start))
        l = []
        for p_res in as_completed(pool_res):
            result=p_res.result()
            l.append(result)
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))
    if 1:
        print('-------------------\n ProcessPoolExecutor without chunksize:')
        start = time.time()
        l = []
        with ProcessPoolExecutor(max_workers=4) as executor:
            for num, result in zip(NUMBERS, executor.map(f, NUMBERS)):
                l.append(result)
        print(len(l), l[0])
        print('COST: {}'.format(time.time() - start))

    print('')
 

結果：

-------------------
 no multiProcessing:
59999 ['xx', 51.0]
COST: 1.2773692607879639
-------------------
 multiprocessing.pool.Pool:
59999 ['xx', 51.0]
COST: 0.4585001468658447
-------------------
 multiprocessing.pool.Pool, apply_async:
middle COST: 1.492830514907837
59999 ['xx', 51.0]
COST: 4.116384267807007
-------------------
 multiprocessing.pool.Pool, apply_async,maxtasksperchild=1000 :
middle COST: 2.0289459228515625
59999 ['xx', 51.0]
COST: 5.032078266143799
-------------------
 ProcessPoolExecutor with chunksize,1/4:
chunksize 3749
59999 ['xx', 51.0]
COST: 0.4767882823944092
-------------------
 ProcessPoolExecutor with chunksize,1/10:
chunksize 1499
59999 ['xx', 51.0]
COST: 0.5644888877868652
-------------------
 ProcessPoolExecutor with chunksize,1/100:
chunksize 149
59999 ['xx', 51.0]
COST: 0.4668114185333252
-------------------
 ProcessPoolExecutor with chunksize,1/300:
chunksize 49
59999 ['xx', 51.0]
COST: 0.673607587814331
-------------------
 ProcessPoolExecutor with chunksize,500:
chunksize 500
59999 ['xx', 51.0]
COST: 0.45476365089416504
-------------------
 ProcessPoolExecutor submit:
middle COST: 11.38172698020935
59999 ['xx', 16145.294670113708]
COST: 21.179430723190308
-------------------
 ProcessPoolExecutor without chunksize:
59999 ['xx', 51.0]
COST: 20.61406421661377
 

區別還挺大， 測試機器cpu有6核。python3.6

轉載註明來源: 本文連結 來自osnosn的部落格.