上次做了一個幫公司妹子做了爬蟲，不是很精致，這次公司項目裡要用到，於是有做了一番修改，功能添加了網址圖片采集，下載，線程處理界面網址圖片下載等。
說說思路：首相獲取初始網址的所有內容 在初始網址采集圖片 去初始網址采集鏈接 把采集到的鏈接放入隊列 繼續采集圖片，然後繼續采集鏈接，無限循環
還是上圖片大家看一下，在上代碼！
處理網頁內容抓取跟網頁網址爬取都做了改進，下面還是大家來看看代碼，有不足之處，還請之處!
網頁內容抓取HtmlCodeRequest,
網頁網址爬取GetHttpLinks，用正則去篩選html中的Links
圖片抓取GetHtmlImageUrlList，用正則去篩選html中的Img
都寫進了一個封裝類裡面 HttpHelper
/// <summary>
/// 取得HTML中所有圖片的 URL。
/// </summary>
/// <param name="sHtmlText">HTML代碼</param>
/// <returns>圖片的URL列表</returns>
public static string HtmlCodeRequest(string Url)
{
if (string.IsNullOrEmpty(Url))
{
return "";
}
try
{
//創建一個請求
HttpWebRequest httprequst = (HttpWebRequest)WebRequest.Create(Url);
//不建立持久性鏈接
httprequst.KeepAlive = true;
//設置請求的方法
httprequst.Method = "GET";
//設置標頭值
httprequst.UserAgent = "User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";
httprequst.Accept = "*/*";
httprequst.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
httprequst.ServicePoint.Expect100Continue = false;
httprequst.Timeout = 5000;
httprequst.AllowAutoRedirect = true;//是否允許302
ServicePointManager.DefaultConnectionLimit = 30;
//獲取響應
HttpWebResponse webRes = (HttpWebResponse)httprequst.GetResponse();
//獲取響應的文本流
string content = string.Empty;
using (System.IO.Stream stream = webRes.GetResponseStream())
{
using (System.IO.StreamReader reader = new StreamReader(stream, System.Text.Encoding.GetEncoding("utf-8")))
{
content = reader.ReadToEnd();
}
}
//取消請求
httprequst.Abort();
//返回數據內容
return content;
}
catch (Exception)
{
return "";
}
}
/// <summary>
/// 提取頁面鏈接
/// </summary>
/// <param name="html"></param>
/// <returns></returns>
public static List<string> GetHtmlImageUrlList(string url)
{
string html = HttpHelper.HtmlCodeRequest(url);
if (string.IsNullOrEmpty(html))
{
return new List<string>();
}
// 定義正則表達式用來匹配 img 標簽
Regex regImg = new Regex(@"<img\b[^<>]*?\bsrc[\s\t\r\n]*=[\s\t\r\n]*[""']?[\s\t\r\n]*(?<imgUrl>[^\s\t\r\n""'<>]*)[^<>]*?/?[\s\t\r\n]*>", RegexOptions.IgnoreCase);
// 搜索匹配的字符串
MatchCollection matches = regImg.Matches(html);
List<string> sUrlList = new List<string>();
// 取得匹配項列表
foreach (Match match in matches)
sUrlList.Add(match.Groups["imgUrl"].Value);
return sUrlList;
}
/// <summary>
/// 提取頁面鏈接
/// </summary>
/// <param name="html"></param>
/// <returns></returns>
public static List<string> GetHttpLinks(string url)
{
//獲取網址內容
string html = HttpHelper.HtmlCodeRequest(url);
if (string.IsNullOrEmpty(html))
{
return new List<string>();
}
//匹配http鏈接
const string pattern2 = @"http(s)?://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?";
Regex r2 = new Regex(pattern2, RegexOptions.IgnoreCase);
//獲得匹配結果
MatchCollection m2 = r2.Matches(html);
List<string> links = new List<string>();
foreach (Match url2 in m2)
{
if (StringHelper.CheckUrlIsLegal(url2.ToString()) || !StringHelper.IsPureUrl(url2.ToString()) || links.Contains(url2.ToString()))
continue;
links.Add(url2.ToString());
}
//匹配href裡面的鏈接
const string pattern = @"(?i)<a\s[^>]*?href=(['""]?)(?!javascript|__doPostBack)(?<url>[^'""\s*#<>]+)[^>]*>"; ;
Regex r = new Regex(pattern, RegexOptions.IgnoreCase);
//獲得匹配結果
MatchCollection m = r.Matches(html);
foreach (Match url1 in m)
{
string href1 = url1.Groups["url"].Value;
if (!href1.Contains("http"))
{
href1 = Global.WebUrl + href1;
}
if (!StringHelper.IsPureUrl(href1) || links.Contains(href1)) continue;
links.Add(href1);
}
return links;
}
public string DownLoadimg(string url)
{
if (!string.IsNullOrEmpty(url))
{
try
{
if (!url.Contains("http"))
{
url = Global.WebUrl + url;
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 2000;
request.UserAgent = "User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";
//是否允許302
request.AllowAutoRedirect = true;
WebResponse response = request.GetResponse();
Stream reader = response.GetResponseStream();
//文件名
string aFirstName = Guid.NewGuid().ToString();
//擴展名
string aLastName = url.Substring(url.LastIndexOf(".") + 1, (url.Length - url.LastIndexOf(".") - 1));
FileStream writer = new FileStream(Global.FloderUrl + aFirstName + "." + aLastName, FileMode.OpenOrCreate, FileAccess.Write);
byte[] buff = new byte[512];
//實際讀取的字節數
int c = 0;
while ((c = reader.Read(buff, 0, buff.Length)) > 0)
{
writer.Write(buff, 0, c);
}
writer.Close();
writer.Dispose();
reader.Close();
reader.Dispose();
response.Close();
return (aFirstName + "." + aLastName);
}
catch (Exception)
{
return "錯誤：地址" + url;
}
}
return "錯誤：地址為空";
}
話不多說，更多的需要大家自己去改進咯！歡迎讀者來與樓主進行交流。如果本文對您有參考價值，歡迎幫博主點下文章下方的推薦，謝謝
下面源碼送上：嘿嘿要分的哦！
http://download.csdn.net/detail/nightmareyan/9627215
