基于urllib库的网页数据爬取

import urllib.request #导入urllib库

response = urllib.request.urlopen('http://www.baidu.com')

html=response.read()#读取网页

print(html)#输出网页



import urllib.request

import urllib.parse

def baidu_search(keyword):

    base_url = "https://www.baidu.com/s"

    params = {"wd": keyword}  # 构建查询参数

    # 将查询参数编码并拼接到URL中

    url = base_url + '?' + urllib.parse.urlencode(params)

    # 发送HTTP请求并获取响应

    response = urllib.request.urlopen(url)

    html_content = response.read()

    return html_content

# 使用关键字"机器学习"进行搜索

search_keyword = "机器学习"

search_result = baidu_search(search_keyword)

# 打印搜索结果的前 500 个字符

print(search_result[:500].decode('utf-8'))