C#解决：抓取出现500错误时候,用这种方法抓取
- 1、下载文档前请自行甄别文档内容的完整性，平台不提供额外的编辑、内容补充、找答案等附加服务。
- 2、"仅部分预览"的文档,不可在线预览部分如存在完整性等问题,可反馈申请退款(可完整预览的文档不适用该条件!)。
- 3、如文档侵犯您的权益，请联系客服反馈,我们会尽快为您处理(人工客服工作时间：9:00-18:30)。
///<summary>
///解决：抓取出现错误时候，用这种方法抓取
///</summary>
///<param name="PageURL"></param>
///<param name="Charset"></param>
///<returns></returns>
public String GetPageCodeBy500Error(String PageURL, String Charset)
{
String strHtml = "";
try
{
//连接到目标网页
HttpWebRequest wreq = (HttpWebRequest)WebRequest.Create(PageURL);
wreq.Method = "GET";
//erAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8";
wreq.Timeout = 20000;
erAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR
3.0.04506.648; .NET CLR 3.5.21022)";
//wreq.ContentType = "text/html";
//wreq.ContentType = "application/x-www-form-urlencoded";
HttpWebResponse wresp = (HttpWebResponse)wreq.GetResponse();
//采用流读取，并确定编码方式
Stream s = wresp.GetResponseStream();
StreamReader objReader = new StreamReader(s,
System.Text.Encoding.GetEncoding(Charset));
string strLine = "";
//读取
while (strLine != null)
{
strLine = objReader.ReadLine();
if (strLine != null)
{
strHtml += strLine.Trim();
}
}
return strHtml;
}
catch (WebException ex)
{
HttpWebResponse res = ex.Response as HttpWebResponse;
if (res.StatusCode == HttpStatusCode.InternalServerError) {
Stream s = res.GetResponseStream();
StreamReader objReader = new StreamReader(s, System.Text.Encoding.GetEncoding(Charset));
string strLine = "";
//读取
while (strLine != null)
{
strLine = objReader.ReadLine();
if (strLine != null)
{
strHtml += strLine.Trim();
}
}
return strHtml;
}
else
{
strHtml = ex.Message;
}
return strHtml;
}
}。