python兩段多線程的例子
阿新 • • 發佈：2019-02-13
reads mod 比較 x86 pytho 記錄 package %d 目錄
記錄瞬間
=====================其一=====================
# coding:UTF-8 import os import threading from time import ctime def loop(loops, list): # list存放著每個線程需要處理的文本文件名 print(‘線程 %d 處理的文件列表 %s \n‘ % (loops + 1, list)) list_len = len(list) for i in range(list_len): f = open(list[i], mode="r", encoding="UTF-8") rows = len(f.readlines()) # 此處，我直接將整個文件讀入，所以會比較卡，可以在此設置每次讀入的大小 f.close() print(‘文件 %s __ %d 行\n‘ % (list[i], rows)) def main(): print(‘all start at:‘, ctime()) cwd = os.getcwd() dir_list = os.listdir(cwd) file_list = [] # 該列表用來存放當前目錄下的所有txt文件print(‘當前文件夾 {} 下的所有txt文件:‘.format(dir_list)) for l in dir_list: if l.rfind(‘log‘) >= 0 and os.path.isfile(l): print(‘ ‘, l) file_list.append(l) threads = [] threads_num = 4 # 線程數 在此處修改下線程數就可以比較多線程與單線程處理文件的速度差異 print(‘共有線程數:%d個‘ % threads_num) per_thread= len(file_list) / threads_num # 每個線程處理的文本數量 print(per_thread) for i in range(threads_num): if threads_num - i == 1: # 最後一個線程，分擔余下的所有工作量 t = threading.Thread(target=loop, args=(i, file_list[i * int(per_thread):])) else: t = threading.Thread(target=loop, args=(i, file_list[i * int(per_thread): i * int(per_thread) + int(per_thread)])) threads.append(t) for i in range(threads_num): threads[i].start() for i in range(threads_num): # 等待所有的線程結束 threads[i].join() print(‘all end at:‘, ctime()) if __name__ == ‘__main__‘: main()
上述代碼，主要解決了多線程在進行讀寫時使用的一些技巧，可以將這段代碼引用於寫一套文件上，然後將多線程寫的文件做以合並。這樣可以很好的解決執行速度的問題。
=====================其二=====================
混合使用多進程和多線程的例子。
#!/usr/bin/python import re import commands import time import multiprocessing import threading def download_image(url): print ‘*****the %s rpm begin to download *******‘ % url commands.getoutput(‘wget %s‘ % url) def get_rpm_url_list(url): commands.getoutput(‘wget %s‘ % url) rpm_info_str = open(‘index.html‘).read() regu_mate = ‘(?<=<a href=")(.*?)(?=">)‘ rpm_list = re.findall(regu_mate, rpm_info_str) rpm_url_list = [url + rpm_name for rpm_name in rpm_list] print ‘the count of rpm list is: ‘, len(rpm_url_list) return rpm_url_list
基礎方法定義 =↑=
def multi_thread(rpm_url_list): threads = [] # url = ‘https://mirrors.ustc.edu.cn/centos/7/os/x86_64/Packages/‘ # rpm_url_list = get_rpm_url_list(url) for index in range(len(rpm_url_list)): print ‘rpm_url is:‘, rpm_url_list[index] one_thread = threading.Thread(target=download_image, args=(rpm_url_list[index],)) threads.append(one_thread) thread_num = 5 # set threading pool, you have put 4 threads in it while 1: count = min(thread_num, len(threads)) print ‘**********count*********‘, count ###25,25,...6707%25 res = [] for index in range(count): x = threads.pop() res.append(x) for thread_index in res: thread_index.start() for j in res: j.join() if not threads: break
多線程的定義 =↑=
def multi_process(rpm_url_list): # process num at the same time is 4 process = [] rpm_url_group_0 = [] rpm_url_group_1 = [] rpm_url_group_2 = [] rpm_url_group_3 = [] for index in range(len(rpm_url_list)): if index % 4 == 0: rpm_url_group_0.append(rpm_url_list[index]) elif index % 4 == 1: rpm_url_group_1.append(rpm_url_list[index]) elif index % 4 == 2: rpm_url_group_2.append(rpm_url_list[index]) elif index % 4 == 3: rpm_url_group_3.append(rpm_url_list[index]) rpm_url_groups = [rpm_url_group_0, rpm_url_group_1, rpm_url_group_2, rpm_url_group_3] for each_rpm_group in rpm_url_groups: each_process = multiprocessing.Process(target = multi_thread, args = (each_rpm_group,)) process.append(each_process) for one_process in process: one_process.start() for one_process in process: one_process.join() # for each_url in rpm_url_list: # print ‘*****the %s rpm begin to download *******‘ %each_url # # commands.getoutput(‘wget %s‘ %each_url)
多進程調用多線程的定義 =↑=
def main(): url = ‘https://mirrors.ustc.edu.cn/centos/7/os/x86_64/Packages/‘ url_paas = ‘http://mirrors.ustc.edu.cn/centos/7.3.1611/paas/x86_64/openshift-origin/‘ url_paas2 =‘http://mirrors.ustc.edu.cn/fedora/development/26/Server/x86_64/os/Packages/u/‘ start_time = time.time() rpm_list = get_rpm_url_list(url_paas) print multi_process(rpm_list) # print multi_thread(rpm_list) #print multi_process() # print multi_thread(rpm_list) # for index in range(len(rpm_list)): # print ‘rpm_url is:‘, rpm_list[index] end_time = time.time() print ‘the download time is:‘, end_time - start_time print main()
主方法 =↑=
代碼來源：https://blog.csdn.net/nfzhlk/article/details/76946281
其中獲取cpu核數的方法可以使用
from multiprocessing import cpu_count print(cpu_count())
一般地，想要多線程快速做事情，我們不加鎖，加了鎖後，容易導致執行的效率跟單線程保持一致了。
這樣做不劃算，當然要看具體的需求是否需要使用多線程加鎖的方式。
python兩段多線程的例子