线程:
# encoding: utf-8 # 版权所有 2023 涂聚文有限公司 # 许可信息查看: # 描述: # Author : geovindu,Geovin Du 涂聚文. # IDE : PyCharm 2023.1 python 311 # Datetime : 2023/7/16 18:55 # User : geovindu # Product : PyCharm # Project : pythonTkinterDemo # File : crawlerDouyu.py # explain : 学习 import json import requests import os import threading def saveimageUrl(url, dirName): """ 线程 :param url: 链接 :param dirName: 文件夹名字 :return: """ # if not os.path.exists('website'): # os.mkdir('website') try: if os.path.exists('website'): os.chdir('website') if not os.path.exists(f"{dirName}"): os.mkdir(dirName) content = requests.get(url) # print(content.text) data = json.loads(content.text) # 转字典 # print(data) cn = 1 for d in data['data']['rl'][0:30]: # 切片为前30个数据 print(d['nn'], d['rs16'], d['rn']) #if not d['av']: #av=d['Info'][av] av = 'https://apic.douyucdn.cn/upload' + '/' + d['av'] + '_big' + '.jpg' print(av) name = d['nn'] # 创建多线程 t = threading.Thread(target=loadAvator, args=(av, dirName, name)) t.start() # img1=Image.open(av) # img1=img1.save('img'+cn+'.jpg') # imagedata = requests.get(av) # with open(fr'{dirName}\{d["nn"]}.jpg','wb') as fp: # fp.write(imagedata) except Exception as ex: print(ex) def loadAvator(av, dirname, name): """ :param av:图片 :param dirname:文件夹名字 :param name:博主名称 :return: """ print("jpg:", fr'{dirname}\{name}.jpg') imagedata = requests.get(av) try: with open(fr'{dirname}\{name}.jpg', 'wb') as fp: fp.write(imagedata.content) # fp.close() except Exception as ex: print(ex)
进程:
# encoding: utf-8 # 版权所有 2023 涂聚文有限公司 # 许可信息查看: # 描述: # Author : geovindu,Geovin Du 涂聚文. # IDE : PyCharm 2023.1 python 311 # Datetime : 2023/7/16 19:10 # User : geovindu # Product : PyCharm # Project : pythonTkinterDemo # File : crawierProcessing.py # explain : 学习 import multiprocessing import threading import time import json import requests import os import sys def saveimageUrl(url, dirName): """ 进程 并发 :param url: 链接 :param dirName: 文件夹名字 :return: """ # if not os.path.exists('website'): # os.mkdir('website') try: if os.path.exists('website'): os.chdir('website') if not os.path.exists(f"{dirName}"): os.mkdir(dirName) content = requests.get(url) # print(content.text) data = json.loads(content.text) # 转字典 # print(data) cn = 1 for d in data['data']['rl'][0:30]: # 切片为前30个数据 print(d['nn'], d['rs16'], d['rn']) #if not d['av']: #av=d['Info'][av] av = 'https://apic.douyucdn.cn/upload' + '/' + d['av'] + '_big' + '.jpg' print(av) name = d['nn'] # 创建进程 t = multiprocessing.Process(target=loadAvator, args=(av, dirName, name)) t.start() # img1=Image.open(av) # img1=img1.save('img'+cn+'.jpg') # imagedata = requests.get(av) # with open(fr'{dirName}\{d["nn"]}.jpg','wb') as fp: # fp.write(imagedata) except Exception as ex: print(ex) def loadAvator(av, dirname, name): """ :param av:图片 :param dirname:文件夹名字 :param name:博主名称 :return: """ print("jpg:", fr'{dirname}\{name}.jpg') imagedata = requests.get(av) try: with open(fr'{dirname}\{name}.jpg', 'wb') as fp: fp.write(imagedata.content) # fp.close() except Exception as ex: print(ex) def dance(): """ :return: """ while True: print('dance') time.sleep(0.5) def sing(): """ :return: """ while True: print('sing') time.sleep(0.5) def main(): """ 线程 是并发 多CPU,多核 :return: """ ''' print(multiprocessing.current_process().name) print(multiprocessing.current_process().ident) p1=multiprocessing.Process(target=dance) p2=multiprocessing.Process(target=sing) p1.start() p2.start() print("cpu个数:",multiprocessing.cpu_count) ''' for _ in range(10): t = threadSing.sing() t2 = threadSing.dance() for _ in range(10): p1 = multiprocessing.Process(target=dance) p2 = multiprocessing.Process(target=sing) p1.start() p2.start()
调用:
if not os.path.exists('website'): os.mkdir('website') os.chdir('website') #Common.crawlerDouyu.saveimageUrl("https://www.douyu.com/gapi/rkc/directory/mixList/2_1/2", 'geovindu') #Common.crawierProcessing.saveimageUrl("https://www.douyu.com/gapi/rkc/directory/mixList/2_1/2", 'process') #Common.crawierProcessing.saveimageUrl("https://www.douyu.com/wgapi/ordnc/live/web/room/yzList/1", 'face') #Common.crawierProcessing.saveimageUrl("https://www.douyu.com/gapi/rkc/directory/mixList/2_194/2", 'sugar') Common.crawierProcessing.saveimageUrl("https://www.douyu.com/wgapi/ordnc/live/web/room/mixList/2/1008/0/2", 'stars')