1 import requests 2 from time import sleep 3 from lxml import etree 4 from fake_useragent import UserAgent 5 from random import randint 6 import re 7 from threading import Thread 8 from queue import Queue 9 from os import remove 10 from copy import copy, deepcopy 11 from bs4 import BeautifulSoup, Comment 12 from pyquery import PyQuery 13 14 session = requests.session() 15 headers = { 16 'User-Agent': UserAgent().random 17 } 18 def get_html(url): 19 sleep( randint(1,3) ) 20 response = session.get(url, headers=headers) 21 if response.status_code == 200: 22 return response.text 23 else: 24 return None 25 26 def save_image(filename, url, data): 27 f1 = re.findall(r'/([^/]+\.[^/]+)\?', url) 28 if not f1: 29 f1 = re.findall(r'/([^/]+\.[^/]+)$', url) 30 url = url + '?imageView2/1/w/160/h/220' 31 if f1: 32 f1 = f1[0] 33 else: 34 f1 = '' 35 36 if (f1 == '') or (f1 == None) : 37 f1 = filename 38 39 domain = re.findall(r'^https://([^/]+.com)', url)[0] 40 url = url.replace(domain, r'img1.abcdbio.com') 41 r = session.get(url, stream=True, headers=headers) 42 if r.status_code == 200: 43 data['file_content'] = deepcopy(r.content) 44 # with open(f1, 'wb') as f: 45 # f.write(r.content) 46 47 print(f1) 48 r.close() 49 50 def parse_film_bs4(html): 51 bsoup = BeautifulSoup(html, 'lxml') 52 film_list = bsoup.select('ol', class_='grid_view')[0].select('li') 53 result_list = [] 54 for film in film_list: 55 detail_url = film.select('div > a')[0].attrs['href'] 56 film_name = film.select('div > a > img')[0].attrs['alt'] 57 film_theme = film.select('div > a > img')[0].attrs['src'] 58 film_info = film.select('div', class_='bd')[0].select('p')[1].span.string 59 result_list.append({ 60 'detail_url': detail_url, 61 'film_name': film_name, 62 'film_theme': film_theme, 63 'film_info': film_info, 64 'file_content': None 65 }) 66 return result_list 67 68 def parse_film_pyquery(html): 69 doc = PyQuery(html) 70 film_list = doc('ol')('.grid_view').children() 71 result_list = [] 72 for index in range(0, len(film_list)): 73 film = film_list.eq(index) 74 detail_url = film('div').eq(0)('a').eq(0).attr('href') 75 film_name = film('div').eq(0)('a').eq(0)('img').eq(0).attr('alt') 76 film_theme = film('div').eq(0)('a').eq(0)('img').eq(0).attr('src') 77 film_info = film.find('div.bd p').eq(1).text() 78 result_list.append({ 79 'detail_url': detail_url, 80 'film_name': film_name, 81 'film_theme': film_theme, 82 'film_info': film_info, 83 'file_content': None 84 }) 85 return result_list 86 87 def parse_film_xpath(html): 88 e = etree.HTML(html) 89 film_list = e.xpath(r"//ol[@class='grid_view']/li") 90 result_list = [] 91 for film in film_list: 92 detail_url = film.xpath('div//a')[0].attrib['href'] 93 film_name = film.xpath('div//a/img')[0].attrib['alt'] 94 film_theme = film.xpath('div//a/img')[0].attrib['src'] 95 film_info = film.xpath("div//div[@class='bd']/p[2]/span")[0].xpath('string(.)') 96 result_list.append({ 97 'detail_url': detail_url, 98 'film_name': film_name, 99 'film_theme': film_theme, 100 'film_info': film_info, 101 'file_content': None 102 }) 103 return result_list 104 105 def save_film(data): 106 data1 = { 107 'name': data['film_name'], 108 'detail_url': data['detail_url'], 109 'info': data['film_info'], 110 } 111 files = { 112 'theme': data['file_content'] 113 } 114 115 r = requests.post('http://localhost:8069/abcdb/film', data=data1, files=files) 116 r.close() 117 118 class PostToServer(Thread): 119 def __init__(self, data_queue): 120 Thread.__init__(self) 121 self.data_queue = data_queue 122 123 def run(self): 124 while True: 125 if self.data_queue.empty(): 126 sleep(0.1) 127 continue 128 s = self.data_queue.get() 129 print(s) 130 save_film(s) 131 132 def main(): 133 data_queue = Queue() 134 th1 = PostToServer(data_queue) 135 th1.start() 136 137 url = 'https://movie.abcdb.com/top250?start=0&filter=' 138 html = get_html(url) 139 # data = parse_film_xpath(html) 140 # data = parse_film_bs4(html) 141 data = parse_film_pyquery(html) 142 for film in data[:5]: 143 save_image('theme.jpg', film['film_theme'], film) 144 data_queue.put(film) 145 th1.join() 146 147 if __name__ == '__main__': 148 main()