# 本程序亲测有效,用于理解爬虫相关的基础知识，不足之处希望大家批评指正
from gevent import monkey
monkey.patch_all()
from gevent.pool import Pool
from queue import Queue
import requests
from lxml import etree
import time
"""爬取目标：http://www.qiushibaike.com/8hr/page/1
利用协程池实现
"""
class QiuShi:
def __init__(self):
# url和headers
self.base_url = 'http://www.qiushibaike.com/8hr/page/{}'
self.headers = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
# 定义队列，用来传递数据
self.url_queue = Queue()
self.request_queue = Queue()
self.html_queue = Queue()
# 初始化线程池
self.pool = Pool()
# 定义计数器
self.request_num = 0
self.response_num = 0
self.is_running = True
def get_url_list(self):
"""获取所有的url"""
for i in range(1, 14):
target_url = self.base_url.format(i)
print(target_url)
self.request_num += 1
self.url_queue.put(target_url)
def request_url(self):
"""向url发起请求"""
target_url = self.url_queue.get()
response = requests.get(target_url, self.headers)
self.request_queue.put(response)
self.url_queue.task_done()
def get_content(self):
"""获取数据"""
html_text = self.request_queue.get().content.decode()
html = etree.HTML(html_text)
div_list = html.xpath('//div[@id="content-left"]/div')
content_list = []
for div in div_list:
item = {}
item['author'] = div.xpath('.//h2/text()')[0].strip()
item['content'] = div.xpath('.//span/text()')[0].strip()
print(item)
content_list.append(item)
self.html_queue.put(content_list)
self.request_queue.task_done()
def save_data(self):
"""保存入库"""
data_list = self.html_queue.get()
for data in data_list:
with open('qiushi.text', 'a+') as f:
f.write(str(data))
f.write('\r\n')
self.html_queue.task_done()
def execute_request_response_save(self):
"""线程池子,执行请求，回应，保存"""
self.request_url()
self.get_content()
self.save_data()
self.response_num += 1
def _callback(self, temp):
"""线程池的回调函数："""
if self.is_running:
self.pool.apply_async(self.execute_request_response_save, callback=self._callback)
def main(self):
"""主程序逻辑"""
self.get_url_list()
# 执行线程池
for n in range(4): # 用4个线程去执行线程池任务
self.pool.apply_async(self.execute_request_response_save, callback=self._callback)
# time.sleep(10)
while True:
time.sleep(0.0001) # 为什么要写sleep? 注销执行一下,看一下结果
if self.request_num == self.response_num:
self.is_running = False
break
if __name__ == '__main__':
qiushi = QiuShi()
qiushi.main()