python之multiprocessing
阿新 • • 發佈:2018-10-01
__main__ 全局 reading 串行 div print \n 處理模塊 fun
1、multiprocessing簡介
multiprocessing模塊提供本地和遠程並發性,通過使用子進程而不是線程來有效地避開全局解釋器鎖。由於這個原因,多處理模塊允許程序員在給定的機器上充分利用多個處理器。它在Unix和Windows上運行。
2、進程的創建
2.1、創建一個進程
import multiprocessing,time def run(name): time.sleep(2) print("hello", name) if __name__ == "__main__": p = multiprocessing.Process(target=run, args=(‘bob‘,)) p.start() p.join()
2.2、創建多個進程
import multiprocessing,time def run(name): time.sleep(2) print("hello", name) if __name__ == "__main__": for i in range(10): p = multiprocessing.Process(target=run, args=(‘bob %s‘ % i,)) p.start() # p.join()
2.3、進程中創建線程
import multiprocessing,time,threading def thread_run(): print("threading id :%s" %threading.get_ident()) def run(name): time.sleep(2) print("hello", name) t = threading.Thread(target=thread_run,) t.start() if __name__ == "__main__": for i in range(10): p = multiprocessing.Process(target=run, args=(‘bob %s‘ % i,)) p.start()
2.4、獲取進程id
from multiprocessing import Process import os def info(title): print(title) print(‘module name:‘, __name__) print(‘parent process:‘, os.getppid()) print(‘process id:‘, os.getpid()) print("\n\n") def f(name): info(‘\033[31;1mcalled from child process function f\033[0m‘) print(‘hello‘, name) if __name__ == ‘__main__‘: info(‘\033[32;1mmain process line\033[0m‘)
輸出結果:
main process line module name: __main__ parent process: 6024 process id: 30756
2.5、獲取進程id和子進程id
from multiprocessing import Process import os def info(title): print(title) print(‘module name:‘, __name__) print(‘parent process:‘, os.getppid()) print(‘process id:‘, os.getpid()) print("\n\n") def f(name): info(‘\033[31;1mcalled from child process function f\033[0m‘) print(‘hello‘, name) if __name__ == ‘__main__‘: info(‘\033[32;1mmain process line\033[0m‘) p = Process(target=f, args=(‘bob‘,)) p.start() # p.join()
輸出結果:
main process line module name: __main__ parent process: 6024 process id: 30756
3、進程間通信
不同進程間內存是不共享的,要實現兩個進程間的數據交換,可以使用一下方法:Queue,Pipes,Manager
3.1、Queue
from multiprocessing import Process, Queue def f(q): q.put([42, None, ‘hello‘]) if __name__ == ‘__main__‘: q = Queue() p = Process(target=f, args=(q,)) p.start() print(q.get())
3.2、Pipes
from multiprocessing import Process, Pipe def f(conn): conn.send([42, None, ‘hello from child‘]) conn.send([42, None, ‘hello from child3‘]) print("",conn.recv()) conn.close() if __name__ == ‘__main__‘: parent_conn, child_conn = Pipe() p = Process(target=f, args=(child_conn,)) p.start() print("parent",parent_conn.recv()) # prints "[42, None, ‘hello‘]" print("parent",parent_conn.recv()) # prints "[42, None, ‘hello‘]" parent_conn.send(" from hshs") # prints "[42, None, ‘hello‘]" p.join()
3.3、Manager
from multiprocessing import Process, Manager import os def f(d, l): d[os.getpid()] =os.getpid() l.append(os.getpid()) print(l) if __name__ == ‘__main__‘: with Manager() as manager: d = manager.dict() #{} #生成一個字典,可在多個進程間共享和傳遞 l = manager.list(range(5))#生成一個列表,可在多個進程間共享和傳遞 p_list = [] for i in range(10): p = Process(target=f, args=(d, l)) p.start() p_list.append(p) for res in p_list: #等待結果 res.join() print(d) print(l)
3.4、進程同步
from multiprocessing import Process, Lock def f(l, i): l.acquire() print(‘hello world‘, i) l.release() if __name__ == ‘__main__‘: lock = Lock() for num in range(100): Process(target=f, args=(lock, num)).start()
4、進程池
4.1、串行:apply
from multiprocessing import Process, Pool,freeze_support import time import os def Foo(i): time.sleep(2) print("in process",os.getpid()) return i + 100 def Bar(arg): print(‘-->exec done:‘, arg,os.getpid()) if __name__ == ‘__main__‘: #freeze_support() pool = Pool(processes=3) #允許進程池同時放入5個進程 print("主進程",os.getpid()) for i in range(10): pool.apply(func=Foo, args=(i,)) #串行 print(‘end‘) pool.close() pool.join() #進程池中進程執行完畢後再關閉,如果註釋,那麽程序直接關閉。.join()
4.2、並行:apply_async
from multiprocessing import Process, Pool,freeze_support import time import os def Foo(i): time.sleep(2) print("in process",os.getpid()) return i + 100 def Bar(arg): print(‘-->exec done:‘, arg,os.getpid()) if __name__ == ‘__main__‘: #freeze_support() pool = Pool(processes=3) #允許進程池同時放入5個進程 print("主進程",os.getpid()) for i in range(10): pool.apply_async(func=Foo, args=(i,), callback=Bar) #callback=回調 #pool.apply_async(func=Foo, args=(i,)) #串行 print(‘end‘) pool.close() pool.join() #進程池中進程執行完畢後再關閉,如果註釋,那麽程序直接關閉。.join()
python之multiprocessing