Python3：爬取新浪、网易、今日头条、UC四大网站新闻标题及内容

以爬取相应网站的社会新闻内容为例：

一、新浪：

新浪网的新闻比较好爬取，我是用BeautifulSoup直接解析的，它并没有使用JS异步加载，直接爬取就行了。

'''

新浪新闻：http://news.sina.com.cn/society/

Date：20180920

Author：lizm

Description：获取新浪新闻

'''

import requests

from bs4 import BeautifulSoup

from urllib import request

import sys

import re

import os

def getNews(title,url,m):

    Hostreferer = {

        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'

    }

    req = request.Request(url)

    response = request.urlopen(req)

    #过滤非utf-8的网页新闻

    response = response.read().decode('utf-8',"ignore")

    soup = BeautifulSoup(response,'lxml')

    tag = soup.find('div',class_='article')

    if tag == None:

        return 0

    #获取文章发布时间

    fb_date = soup.find('div','date-source').span.string

    #获取发布网站名称

    fb_www= soup.find('div','date-source').a.string

    #获取文章内容

    rep = re.compile("[\s+\.\!\/_,$%^*(+\"\']+|[+<>?、~*（）]+")

    title = rep.sub('',title)

    title = title.replace(':','：')

    filename = sys.path[0]+"/news/"+title+".txt"

    with open(filename,'w',encoding='utf8') as file_object:

        file_object.write(fb_date + " " + fb_www)

        file_object.write("\n")

        file_object.write("网址:"+url)

        file_object.write("\n")

        file_object.write(title)

        file_object.write(tag.get_text())

    i = 0

    for image in tag.find_all('div','img_wrapper'):

        title_img = title +str(i)

        #保存图片

        #判断目录是否存在

        if (os.path.exists(sys.path[0]+"/news/"+title)):

            pass

        else:

            #不存在，则新建目录

            os.mkdir(sys.path[0]+"/news/"+title)

        os.chdir(sys.path[0]+"/news/"+title)

        file_name = "http://news.sina.com.cn/"+image.img.get('src').replace('//','')

        html = requests.get(file_name, headers=Hostreferer)

        # 图片不是文本文件，以二进制格式写入，所以是html.content

        title_img = title_img +".jpg"

        f = open(title_img, 'wb')

        f.write(html.content)

        f.close()

        i+=1

    print('成功爬取第', m,'个新闻',title)

    return 0

#获取社会新闻（最新的162条新闻）

def getTitle(url):

    req = request.Request(url)

    response = request.urlopen(req)

    response = response.read().decode('utf8')

    soup = BeautifulSoup(response,'lxml')

    y = 0

    for tag in soup.find('ul',class_='seo_data_list').find_all('li'):

        if tag.a != None:

            #if y== 27:

            print(y,tag.a.string,tag.a.get('href'))

            temp = tag.a.string

            getNews(temp,tag.a.get('href'),y)

            y += 1

if __name__ == '__main__':

    url = 'http://news.sina.com.cn/society/'

    getTitle(url)

二、网易：

网易新闻的标题及内容是使用js异步加载的，单纯的下载网页源代码是没有标题及内容的，我们可以在Network的js中找到我们需要的内容，这里我使用了正则表达式来获取我们需要的标题及其链接，并使用了BeautifulSoup来获取相应标题的内容。

import re

from urllib import request

from bs4 import BeautifulSoup

def download(title, url):

    req = request.urlopen(url)

    res = req.read()

    soup = BeautifulSoup(res,'lxml')

    #print(soup.prettify())

    tag = soup.find('div',class_='post_text')

    #print(tag.get_text())

    title = title.replace(':','')

    title = title.replace('"','')

    title = title.replace('|','')

    title = title.replace('/','')

    title = title.replace('\\','')

    title = title.replace('*','')

    title = title.replace('<','')

    title = title.replace('>','')

    title = title.replace('?','')

    #print(title)

    file_name = r'D:\code\python\spider_news\NetEase_news\sociaty\\' +title + '.txt'

    file = open(file_name,'w',encoding = 'utf-8')

    file.write(tag.get_text())

if __name__ == '__main__':

    urls = ['http://temp.163.com/special/00804KVA/cm_shehui.js?callback=data_callback',

            'http://temp.163.com/special/00804KVA/cm_shehui_02.js?callback=data_callback',

            'http://temp.163.com/special/00804KVA/cm_shehui_03.js?callback=data_callback']

    for url in urls:

    #url = 'http://temp.163.com/special/00804KVA/cm_shehui_02.js?callback=data_callback'

        req = request.urlopen(url)

        res = req.read().decode('gbk')

        #print(res)

        pat1 = r'"title":"(.*?)",'

        pat2 = r'"tlink":"(.*?)",'

        m1 = re.findall(pat1,res)

        news_title = []

        for i in m1:

            news_title.append(i)

        m2 = re.findall(pat2,res)

        news_url = []

        for j in m2:

            news_url.append(j)

        for i in range(0,len(news_url)):

            #print(news_title[i],news_body[i])

            download(news_title[i],news_url[i])

            print('正在爬取第' + str(i) + '个新闻',news_title[i])

 

三、头条：

头条的新闻跟前两个也都不一样，它的标题和链接是封装到json文件中的，但是他json文件的url参数是通过一个js随机算法变化的，所以我们需要模拟json文件的参数，否则我们找不到json文件的具体url，我是通过http://www.jianshu.com/p/5a93673ce1c0这篇博客才了解到url获取方法的，而且也解决了总是下载重复新闻的问题，该网站自带反爬机制，需要添加cookie。关于新闻的内容，我用了正则表达式提取了中文。

from urllib import request

import requests

import json

import time

import math

import hashlib

import re

from bs4 import BeautifulSoup

def get_url(max_behot_time, AS, CP):

    url = 'https://www.toutiao.com/api/pc/feed/?category=news_society&utm_source=toutiao&widen=1' \

          '&max_behot_time={0}' \

          '&max_behot_time_tmp={0}' \

          '&tadrequire=true' \

          '&as={1}' \

          '&cp={2}'.format(max_behot_time, AS, CP)

    return url

def get_ASCP():

    t = int(math.floor(time.time()))

    e = hex(t).upper()[2:]

    m = hashlib.md5()

    m.update(str(t).encode(encoding='utf-8'))

    i = m.hexdigest().upper()

    if len(e) != 8:

        AS = '479BB4B7254C150'

        CP = '7E0AC8874BB0985'

        return AS,CP

    n = i[0:5]

    a = i[-5:]

    s = ''

    r = ''

    for o in range(5):

        s += n[o] + e[o]

        r += e[o + 3] + a[o]

    AS = 'AL'+ s + e[-3:]

    CP = e[0:3] + r + 'E1'

   # print("AS:"+ AS,"CP:" + CP)

    return AS,CP

def download(title, news_url):

   # print('正在爬')

    req = request.urlopen(news_url)

    if req.getcode() != 200:

        return 0

    res = req.read().decode('utf-8')

    #print(res)

    pat1 = r'content:(.*?),'

    pat2 = re.compile('[\u4e00-\u9fa5]+')

    result1 = re.findall(pat1,res)

    #print(len(result1))

    if len(result1) == 0:

        return 0

    print(result1)

    result2 = re.findall(pat2,str(result1))

    result3 = []

    for i in result2:

        if i not in result3:

            result3.append(i)

    #print(result2)

    title = title.replace(':','')

    title = title.replace('"','')

    title = title.replace('|','')

    title = title.replace('/','')

    title = title.replace('\\','')

    title = title.replace('*','')

    title = title.replace('<','')

    title = title.replace('>','')

    title = title.replace('?','')

    with open(r'D:\code\python\spider_news\Toutiao_news\society\\' + title + '.txt','w') as file_object:

        file_object.write('\t\t\t\t')

        file_object.write(title)

        file_object.write('\n')

        file_object.write('该新闻地址：')

        file_object.write(news_url)

        file_object.write('\n')

        for i in result3:

            #print(i)

            file_object.write(i)

            file_object.write('\n')

       # file_object.write(tag.get_text())

    #print('正在爬取')

def get_item(url):

    #time.sleep(5)

    cookies = {'tt_webid': ''}

    wbdata = requests.get(url,cookies = cookies)

    wbdata2 = json.loads(wbdata.text)

    data = wbdata2['data']

    for news in data:

        title = news['title']

        news_url = news['source_url']

        news_url = 'https://www.toutiao.com' + news_url

        print(title, news_url)

        if 'ad_label' in news:

            print(news['ad_label'])

            continue

        download(title,news_url)

    next_data = wbdata2['next']

    next_max_behot_time = next_data['max_behot_time']

   # print("next_max_behot_time:{0}".format(next_max_behot_time))

    return next_max_behot_time

if __name__ == '__main__':

    refresh = 50

    for x in range(0,refresh+1):

        print('第{0}次：'.format(x))

        if x == 0:

            max_behot_time = 0

        else:

            max_behot_time = next_max_behot_time

            #print(next_max_behot_time)

        AS,CP = get_ASCP()

        url = get_url(max_behot_time,AS,CP)

        next_max_behot_time = get_item(url)

 

四、UC

UC和新浪差不多，没有太复杂的反爬虫，直接解析爬取就好。

from bs4 import BeautifulSoup

from urllib import request

def download(title,url):

    req = request.Request(url)

    response = request.urlopen(req)

    response = response.read().decode('utf-8')

    soup = BeautifulSoup(response,'lxml')

    tag = soup.find('div',class_='sm-article-content')

    if tag == None:

        return 0

    title = title.replace(':','')

    title = title.replace('"','')

    title = title.replace('|','')

    title = title.replace('/','')

    title = title.replace('\\','')

    title = title.replace('*','')

    title = title.replace('<','')

    title = title.replace('>','')

    title = title.replace('?','')

    with open(r'D:\code\python\spider_news\UC_news\society\\' + title + '.txt','w',encoding='utf-8') as file_object:

        file_object.write('\t\t\t\t')

        file_object.write(title)

        file_object.write('\n')

        file_object.write('该新闻地址：')

        file_object.write(url)

        file_object.write('\n')

        file_object.write(tag.get_text())

    #print('正在爬取')

if __name__ == '__main__':

    for i in range(0,7):

        url = 'https://news.uc.cn/c_shehui/'

    #    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.91 Safari/537.36",

    #               "cookie":"sn=3957284397500558579; _uc_pramas=%7B%22fr%22%3A%22pc%22%7D"}

    #    res = request.Request(url,headers = headers)

        res = request.urlopen(url)

        req = res.read().decode('utf-8')

        soup = BeautifulSoup(req,'lxml')

        #print(soup.prettify())

        tag = soup.find_all('div',class_ = 'txt-area-title')

        #print(tag.name)

        for x in tag:

            news_url = 'https://news.uc.cn' + x.a.get('href')

            print(x.a.string,news_url)

            download(x.a.string,news_url)