pyquery+requests爬取猫眼电影热映口碑榜

import requests
import threading
import csv

from queue import Queue
from pyquery import PyQuery as pq


class Producer(threading.Thread):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) '
                      'AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/76.0.3809.132 Safari/537.36'
    }

    def __init__(self, movie_url, movie_data, *args, **kwargs):
        super(Producer, self).__init__(*args, **kwargs)
        self.movie_url = movie_url
        self.movie_data = movie_data

    def run(self):
        while True:
            if self.movie_url.empty():
                break
            url = self.movie_url.get()
            self.parse_page(url)

    def parse_page(self, url):
        response = requests.get(url=url, headers=self.headers)
        text_code = response.encoding
        text = response.content.decode(text_code, 'ignore')
        html = pq(text)
        dds = html('dl.board-wrapper dd')
        for dd in dds.items():
            sort = dd('i').text()
            href = dd('a').attr('href')
            title = dd('a').attr('title')
            self.movie_data.put((sort, href, title))


class Consumer(threading.Thread):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) '
                      'AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/76.0.3809.132 Safari/537.36'
    }

    def __init__(self, movie_url, movie_data, *args, **kwargs):
        super(Consumer, self).__init__(*args, **kwargs)
        self.movie_url = movie_url
        self.movie_data = movie_data

    def run(self):
        headers = ['sort', 'href', 'title']
        data = []
        while True:
            if self.movie_url.empty() and self.movie_data.empty():
                break
            data_z = {}
            data_z['sort'], data_z['href'], data_z['title'] = self.movie_data.get()
            data.append(data_z)
        # 将文件保存到csv文件中
        with open("movie.csv", 'w', encoding='utf-8', newline='') as fp:
            writer = csv.DictWriter(fp, headers)
            writer.writeheader()
            writer.writerows(data)


def main():
    movie_url = Queue(10)
    movie_data = Queue(100)

    for x in range(10):
        data = x * 10
        url = "https://maoyan.com/board/4?offset=%s" % data
        movie_url.put(url)

    for x in range(4):
        t = Producer(movie_url, movie_data)
        t.start()

    for x in range(4):
        t = Consumer(movie_url, movie_data)
        t.start()


if __name__ == '__main__':
    main()