Java-HttpURLConnection爬虫程序-0913

Java-HttpURLConnection抓取网络数据(2016-9-13) 项目:JavaSpiderDemo 环境:MyEclipse8.5
导包
源码
Java
MyConn.java
import java.io.BufferedReader; import java.io.InputStreamReader; import .*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements;
/**
* 独立的解析一篇CSDN博客
* 提取出:标题,分类,内容
* @author孙沛林
*
*/
public class MyConn extends Thread {
public MyConn(){}
public MyConn(String surl){this.surl=surl;}
private String surl;// 文章的网址
@Override
public void run() {
getHTML();
}
// 客户端的浏览器类型
public static String[] UserAgent = {
"Mozilla/5.0 (jsoup)",// PC端的浏览器
// 以下都是手机端的浏览器
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.2",
"Mozilla/5.0 (iPad; U; CPU OS 3_2_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B500 Safari/531.21.11",
"Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18121",
///yjflinchong
"Nokia5700AP23.01/SymbianOS/9.1 Series60/3.0",
"UCWEB7.0.2.37/28/998",
"NOKIA5700/UCWEB7.0.2.37/28/977",
"Openwave/UCWEB7.0.2.37/28/978",
"Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/989"
};
/**
* 根据一个网址,获取该页面的HTML源码
* @param surl
*/
public void getHTML(){
try {
// 1. 创建URL对象
URL url = new URL(surl);
// 2. 获取HttpUrlConnection对象
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
// 2.5 伪装成浏览器
conn.setRequestProperty(
"User-Agent", // 浏览器的类型
UserAgent[1]);
// 3. 获取服务器的响应码
int responseCode = conn.getResponseCode();
System.out.println("responseCode="+responseCode);
// 4. 判断响应码是否正常,HttpURLConnection.HTTP_OK=200
if (responseCode == HttpURLConnection.HTTP_OK) {
StringBuffer sb = new StringBuffer();// HTML的容器
String readLine;// 每一行的临时存放
//5. 处理响应流，必须与服务器响应流输出的编码一致"UTF-8"
BufferedReader responseReader = new BufferedReader(
new InputStreamReader(
conn.getInputStream(), "UTF-8"));
// 6. 循环读取流中的行
while ((readLine = responseReader.readLine()) != null) {
sb.append(readLine).append("\n");
}// while
// 7. 关闭流
responseReader.close();
//System.out.println("*******HTML******\n"+sb.toString());//输入HTML
parseHTML(sb.toString());
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// 解析文章详细页面
private void parseHTML(String html) {
StringBuffer text = new StringBuffer();// HTML的容器
Document doc = Jsoup.parse(html);
// 1文章 title
String atitle = doc.getElementsByClass("article_t").first().text();
text.append("\n-----文章 title-------\n");
text.append(atitle);
// 2文章分类
Elements cats = doc.getElementsByClass("category_p");
// 循环遍历
text.append("\n-----文章分类-------\n");
for(Element cat : cats){
text.append(cat.text());
}
// 3文章 content
String acontent = doc.getElementsByClass("article_c").first().text();
//text.append(acontent);
text.append("\n======文章 content==============================\n"); text.append(acontent);
// 数据的插入
//insert(atitle,cats.text(),acontent);
System.out.println("\n================解析的结果====================\n"
+text.toString());
}
}
ListPage.java
import java.io.BufferedReader;
import java.io.InputStreamReader;
import .*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
* 解析CSDN博客专栏(包含很多文章列表)
* 提取出:文章标题和链接
* @author孙沛林
*
*/
public class ListPage extends Thread {
public ListPage(){}
public ListPage(String surl){this.surl=surl;}
private String surl;// 文章的网址
@Override
public void run() {
getHTML();
}
// 客户端的浏览器类型
public static String[] UserAgent = {
"Mozilla/5.0 (jsoup)",// PC端的浏览器
// 以下都是手机端的浏览器
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.2",
"Mozilla/5.0 (iPad; U; CPU OS 3_2_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B500 Safari/531.21.11",
"Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18121",
///yjflinchong
"Nokia5700AP23.01/SymbianOS/9.1 Series60/3.0",
"UCWEB7.0.2.37/28/998",
"NOKIA5700/UCWEB7.0.2.37/28/977",
"Openwave/UCWEB7.0.2.37/28/978",
"Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/989"
};
/**
* 根据一个网址,获取该页面的HTML源码
* @param surl
*/
public void getHTML(){
try {
// 1. 创建URL对象
URL url = new URL(surl);
// 2. 获取HttpUrlConnection对象
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
// 2.5 伪装成浏览器
conn.setRequestProperty(
"User-Agent", // 浏览器的类型
UserAgent[1]);
// 3. 获取服务器的响应码
int responseCode = conn.getResponseCode();
System.out.println("responseCode="+responseCode);
// 4. 判断响应码是否正常,HttpURLConnection.HTTP_OK=200
if (responseCode == HttpURLConnection.HTTP_OK) {
StringBuffer sb = new StringBuffer();// HTML的容器
String readLine;// 每一行的临时存放
//5. 处理响应流，必须与服务器响应流输出的编码一致"UTF-8"
BufferedReader responseReader = new BufferedReader(
new InputStreamReader(
conn.getInputStream(), "UTF-8"));
// 6. 循环读取流中的行
while ((readLine = responseReader.readLine()) != null) {
sb.append(readLine).append("\n");
}// while
// 7. 关闭流
responseReader.close();
//System.out.println("*******HTML******\n"+sb.toString());//输入HTML
parseHTML(sb.toString());
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// 解析文章详细页面
private void parseHTML(String html) {
StringBuffer text = new StringBuffer();// HTML的容器
Document doc = Jsoup.parse(html);
Elements items = doc.getElementsByTag("h4");
for (Element item : items) {
// 找到链接
Element link = item.getElementsByTag("a").first();
if(link!=null){
String href = link.attr("href");
String title = link.text();
text.append(href);
text.append(title);
text.append("\n------------\n");
//getHTML(href);
new MyConn(href).start();// 启动线程--> 解析一篇文章
}
}
}
}
Main.java
import java.util.*;
/**
* java爬虫程序
* @author钧
*
*/
public class Main {
/**
* @param args
*/
public static void main(String[] args) {
// 循环解析多个URL
// List<String> list = new ArrayList<String>();
// list.add("/withiter/article/details/16802241");
// list.add("/withiter/article/details/16802117");
// list.add("/withiter/article/details/16336939");
// for (String url : list) {
// new MyConn(url).start();// 启动线程
// }
//System.out.println("你传了:"+args[0]);
String url = "/column/details/javaforumyuejingtie.html";
new ListPage(url).start();// 解析列表
} }。