零基础写Java知乎爬虫之抓取知乎答案(2)

Zhihu.java

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Zhihu {
 public String question;// 问题
 public String questionDescription;// 问题描述
 public String zhihuUrl;// 网页链接
 public ArrayList<String> answers;// 存储所有回答的数组
 // 构造方法初始化数据
 public Zhihu(String url) {
  // 初始化属性
  question = "";
  questionDescription = "";
  zhihuUrl = "";
  answers = new ArrayList<String>();
  // 判断url是否合法
  if (getRealUrl(url)) {
   System.out.println("正在抓取" + zhihuUrl);
   // 根据url获取该问答的细节
   String content = Spider.SendGet(zhihuUrl);
   Pattern pattern;
   Matcher matcher;
   // 匹配标题
   pattern = Pattern.compile("zh-question-title.+?<h2.+?>(.+?)</h2>");
   matcher = pattern.matcher(content);
   if (matcher.find()) {
    question = matcher.group(1);
   }
   // 匹配描述
   pattern = Pattern
     .compile("zh-question-detail.+?<div.+?>(.*?)</div>");
   matcher = pattern.matcher(content);
   if (matcher.find()) {
    questionDescription = matcher.group(1);
   }
   // 匹配答案
   pattern = Pattern.compile("/answer/content.+?<div.+?>(.*?)</div>");
   matcher = pattern.matcher(content);
   boolean isFind = matcher.find();
   while (isFind) {
    answers.add(matcher.group(1));
    isFind = matcher.find();
   }
  }
 }
 // 根据自己的url抓取自己的问题和描述和答案
 public boolean getAll() {
  return true;
 }
 // 处理url
 boolean getRealUrl(String url) {
  // 将http://www.zhihu.com/question/22355264/answer/21102139
  // 转化成http://www.zhihu.com/question/22355264
  // 否则不变
  Pattern pattern = Pattern.compile("question/(.*?)/");
  Matcher matcher = pattern.matcher(url);
  if (matcher.find()) {
   zhihuUrl = "http://www.zhihu.com/question/" + matcher.group(1);
  } else {
   return false;
  }
  return true;
 }
 @Override
 public String toString() {
  return "问题：" + question + "\n" + "描述：" + questionDescription + "\n"
    + "链接：" + zhihuUrl + "\n回答：" + answers.size() + "\n";
 }
}

登录后复制

Spider.java

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Spider {
 static String SendGet(String url) {
  // 定义一个字符串用来存储网页内容
  String result = "";
  // 定义一个缓冲字符输入流
  BufferedReader in = null;
  try {
   // 将string转成url对象
   URL realUrl = new URL(url);
   // 初始化一个链接到那个url的连接
   URLConnection connection = realUrl.openConnection();
   // 开始实际的连接
   connection.connect();
   // 初始化 BufferedReader输入流来读取URL的响应
   in = new BufferedReader(new InputStreamReader(
     connection.getInputStream(), "UTF-8"));
   // 用来临时存储抓取到的每一行的数据
   String line;
   while ((line = in.readLine()) != null) {
    // 遍历抓取到的每一行并将其存储到result里面
    result += line;
   }
  } catch (Exception e) {
   System.out.println("发送GET请求出现异常！" + e);
   e.printStackTrace();
  }
  // 使用finally来关闭输入流
  finally {
   try {
    if (in != null) {
     in.close();
    }
   } catch (Exception e2) {
    e2.printStackTrace();
   }
  }
  return result;
 }
 // 获取所有的编辑推荐的知乎内容
 static ArrayList<Zhihu> GetRecommendations(String content) {
  // 预定义一个ArrayList来存储结果
  ArrayList<Zhihu> results = new ArrayList<Zhihu>();
  // 用来匹配url，也就是问题的链接
  Pattern pattern = Pattern
    .compile("<h2>.+?question_link.+?href=\"(.+?)\".+?</h2>");
  Matcher matcher = pattern.matcher(content);
  // 是否存在匹配成功的对象
  Boolean isFind = matcher.find();
  while (isFind) {
   // 定义一个知乎对象来存储抓取到的信息
   Zhihu zhihuTemp = new Zhihu(matcher.group(1));
   // 添加成功匹配的结果
   results.add(zhihuTemp);
   // 继续查找下一个匹配对象
   isFind = matcher.find();
  }
  return results;
 }
}

登录后复制

以上就是零基础写Java知乎爬虫之抓取知乎答案(2) 的内容，更多相关内容请关注Work网（www.php.cn）！