C#實現web信息自動抓取
NetBug--爬蟲 V1.02開發筆記
背景
隨著Internet的普及，網絡信息正以極高的速度增長，在這麼多數據中找到自己需要的信息是一件很繁瑣的事情，找到需要的信息後如何獲取也是件麻煩的事。這就需要Internet信息抓取程序來代替人工的操作。
所謂Internet信息抓取程序，就是程序會按照用戶的關鍵詞或關鍵網站來收集相應的信息，並提供給用戶想要的信息格式。
信息量的增加會帶來信息網站發布人員工作量的劇增，為實現信息發布系統實現信息自
動發布、減少工作人員工作量、即時跟蹤最新信息，就需要自動信息提供程序，因此Internet信息抓取程序應運而生。
目標
實現自定義網站信息分類抓取，存入本地數據庫、生成靜態頁面或其它用戶定義的信息結構，並下載與信息相關的多媒體文件。
開發
l 目標站點結構分析
本步驟是准確抓取信息個關鍵。
首先要選擇更新頻率高的頁面做為抓取地址，然後分析要抓取內容頁面url特點。
然後分析要抓取信息頁面的元素特性，比如標題位置，內容位置 等，得到定位標記點。
將以上信息寫成自己的配置文件或存到數據庫中。
每個網站都需要分析，寫出單獨的配置文件，供抓取程序使用。
l 信息提取
根據配置文件取得要抓取頁面url，使用HttpWebRequest類獲取內容：
//獲取http頁面函數
public string Get_Http(string a_strUrl,int timeout)
{
string strResult ;
try
{
HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;
myReq.Timeout = timeout;
HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();
Stream myStream = HttpWResp.GetResponseStream () ;
StreamReader sr = new StreamReader(myStream , Encoding.Default);
StringBuilder strBuilder = new StringBuilder();
while (-1 != sr.Peek())
{
strBuilder.Append(sr.ReadLine()+"\r\n");
}
strResult = strBuilder.ToString();
}
catch(Exception exp)
{
strResult = "錯誤：" + exp.Message ;
}
return strResult ;
}
獲取頁面內容後，分析頁面中連接地址取到要抓取的url：
//處理頁面標題和鏈接
public string SniffWebUrl( string urlStr,string blockB,
string blockE )
{
string urlch1 = "";
string urlch2 = "";
int end_n1 = 0;
int end_nums = 0;
int end_nums1 = 0;
int end_nums2 = 0;
int end_nums3 = 0;
string reUTStr = "";
string reTitle = "";
string ret = "";
try
{
int pos01 = urlStr.IndexOf( "." );
int pos02 = urlStr.LastIndexOf( "/" );
if( pos01 < 0 )
{
return "";
}
if( pos02 < 0 )
{
return "";
}
int pos03 = urlStr.IndexOf( "/",pos01 );
if ( pos03 < 0 )
{
urlch1 = urlStr;
urlch2 = urlStr;
}
else
{
urlch1 = urlStr.Substring( 0,pos03 );
urlch2 = urlStr.Substring( 0,pos02 );
}
string tmpAllStr = new PublicFun().Get_Http( urlStr ,time1);
int pos1 = tmpAllStr.IndexOf( blockB );
int pos2 = tmpAllStr.IndexOf( blockE,pos1 + blockB.Length );
if ( pos1>0 && pos2>0 && pos2>pos1 )
{
ret = tmpAllStr.Substring( pos1 + blockB.Length,pos2 - pos1 - blockB.Length );
ret = ret.Substring( ret.IndexOf( "<" ));
while( ret.IndexOf( "<A" ) >= 0 )
{
ret = ret.Substring( 0,ret.IndexOf( "<A" ) ) + "<a" + ret.Substring( ret.IndexOf( "<A" ) + 2 );
}
while( ret.IndexOf( "</A" ) >=0 )
{
ret = ret.Substring( 0,ret.IndexOf( "</A" ) ) + "</a" + ret.Substring( ret.IndexOf( "</A" ) + 3 );
}
while( ret.IndexOf( "Href=" ) >=0 )
{
ret = ret.Substring( 0,ret.IndexOf( "Href=" )) + "href=" + ret.Substring( ret.IndexOf( "Href=" ) + 5 );
}
while( ret.IndexOf( "HREF=" ) >=0 )
{
ret = ret.Substring( 0,re( "HREF=" )) + "href=" + ret.Substring( ret.IndexOf( "HREF=" ) + 5 );
}
while( ret.IndexOf( "href='" ) >=0 )
{
ret = ret.Substring( 0,ret.IndexOf( "href='" )) + "href=\"" + ret.Substring( ret.IndexOf( "href='" ) + 6 );
}
}
tmpAllStr = ret;
int begin_nums = tmpAllStr.IndexOf( "href=" );
while ( begin_nums >= 0 )
{
string tmpStrA = "";
string tmpStrB = tmpAllStr.Substring( begin_nums + 5,1 );
if ( tmpStrB == "\"" )
{
end_n1 = begin_nums + 6;
if ( ( end_n1 + 1 ) > tmpAllStr.Length )
{
return "";
}
tmpStrA = tmpAllStr.Substring( begin_nums+6,1 );
}
else
{
end_n1 = begin_nums + 5;
tmpStrA = tmpStrB;
}
if ( tmpStrA == "#" )
{
tmpAllStr = tmpAllStr.Substring( end_n1 );
begin_nums = tmpAllStr.IndexOf( "href=" );
}
else
{
end_nums1 = tmpAllStr.IndexOf( " ",end_n1 );
end_nums2 = tmpAllStr.IndexOf( ">",end_n1 );
end_nums3 = tmpAllStr.IndexOf( "</a",end_nums2 );
if ( ( end_nums3 >= 0 ) && ( end_nums2 >= 0 ) )
{
reTitle = tmpAllStr.Substring( end_nums2 + 1,end_nums3 - end_nums2 - 1 );
if ( end_nums1 > end_nums2 )
{
end_nums = end_nums2;
}
else
{
if ( end_nums1 < 0 )
{
end_nums = end_nums2;
}
else
{
end_nums = end_nums1;
}
}
string str4 = tmpAllStr.Substring( end_nums-1, end_nums - end_nums + 1 );
if ( str4 =="\"" || str4 == "'" )
{
end_nums = end_nums - 1;
}
string sTotalOne = tmpAllStr.Substring( end_n1,end_nums - end_n1 );
if ( sTotalOne.IndexOf( "http://" ) <0 )
{
if ( sTotalOne.IndexOf( "/" ) == 0 )
{
sTotalOne = urlch1 + sTotalOne;
}
else
{
int linshiIntNum = 0;
int flags = 0;
string urlChange = urlStr;;
while( sTotalOne.IndexOf( "../" ) >= 0 )
{
sTotalOne = sTotalOne.Substring( sTotalOne.IndexOf( "../" ) + 3 );
linshiIntNum = linshiIntNum + 1;
flags = flags +1;
}
while( ( urlChange.LastIndexOf( "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )
{
urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );
linshiIntNum = linshiIntNum - 1;
}
if ( flags == 0 )
sTotalOne = urlch2 + "/" + sTotalOne;
}
else
{
sTotalOne = urlChange + "/" + sTotalOne;
}
}
}
reUTStr = reUTStr + new PublicFun().RemoveHtmlCode( reTitle ) + sTotalOne;
tmpAllStr = tmpAllStr.Substring( end_nums3 + 4 );
begin_nums = tmpAllStr.IndexOf( "href=" );
}
else
{
begin_nums = -1;
}
}
}
return reUTStr;
}
catch( Exception e)
{
return "";
}
}
得到要抓取內容的url後，處理該頁面：
//獲取鏈接內容並分類處理
public string GetWebContent( string gatherUrl,string subUrl,string subTitle,string b_Content,string e_Content,string b_Filter,string e_Filter,string root )
{
string tmpAllStr = "";
string dfStrB = "";
string dfStrE = "";
string rePicStr = "";//圖片返回路徑
string reContentStr = "";
string picHtml = "images"; //本地圖片路徑
string urlch1 ="";
string urlch2 ="";
int pos1 = gatherUrl.IndexOf( "." );
int pos2 = gatherUrl.LastIndexOf( "/" );
if( pos1 < 0 )
{
return "";
}
if( pos2 < 0 )
{
return "";
}
int pos3 = gatherUrl.IndexOf( "/",pos1 );
if ( pos3 < 0 )
{
urlch1 = gatherUrl;
urlch2 = gatherUrl;
}
else
{
urlch1 = gatherUrl.Substring( 0,pos3 );
urlch2 = gatherUrl.Substring( 0,pos2 );
}
tmpAllStr = new PublicFun().Get_Http( subUrl,time1 );
//取稿源
string docFromStr = "";
if ( dfStrB != "" && dfStrE != "" )
{
if ( tmpAllStr != "" )
{
int b_docF = tmpAllStr.IndexOf( dfStrB );
if ( b_docF > 0 )
{
int e_docF = tmpAllStr.IndexOf( dfStrE,b_docF + dfStrB.Length );
if ( e_docF > 0 && e_docF > b_docF && e_docF - b_docF < 20 )
{
docFromStr = tmpAllStr.Substring( b_docF + dfStrB.Length, e_docF - b_docF - dfStrB.Length );
}
}
}
}
//取內容
if ( tmpAllStr != "" )
{
int begin_strnum = tmpAllStr.IndexOf( b_Content );
if ( begin_strnum < 0 )
{
return "";
}
int end_strnum = tmpAllStr.IndexOf( e_Content,begin_strnum + b_Content.Length );
if ( end_strnum < 0 )
{
return "";
}
string sTotalSubM = "";
if ( end_strnum > begin_strnum )
{
sTotalSubM = tmpAllStr.Substring ( begin_strnum,end_strnum - begin_strnum );
}
if ( sTotalSubM == "" )
{
return "";
}
//過濾無用信息
int bfnum = sTotalSubM.IndexOf( b_Filter );
> -1 )
{
int efnum = sTotalSubM.IndexOf( e_Filter,bfnum );
if ( efnum > -1 )
{
if ( efnum > bfnum )
{
sTotalSubM = sTotalSubM.Substring( 0,bfnum ) + sTotalSubM.Substring( efnum + e_Filter.Length );
}
}
}
//格式化圖片標記
while( sTotalSubM.IndexOf( "Src=" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "Src=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "Src=" ) + 4 );
}
while( sTotalSubM.IndexOf( "SRC=" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "SRC=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "SRC=" ) + 4 );
}
while( sTotalSubM.IndexOf( "src='" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "src='" ) ) + "src=\"" + sTotalSubM.Substring( sTotalSubM.IndexOf( "src='" ) + 5 );
}
//取圖片地址
int end_n12 = 0;
int end_nums2 = 0;
int begin_nums2 = sTotalSubM.IndexOf( "src=" );
while( begin_nums2 >= 0 )
{
String tmpStr = sTotalSubM.Substring( begin_nums2 + 4,1 );
if ( tmpStr == "\"" )
{
end_n12 = begin_nums2 + 5;
}
else
{
end_n12 = begin_nums2 + 4;
}
int end_nums2a = sTotalSubM.IndexOf( " ",end_n12 );
int end_nums2b = sTotalSubM.IndexOf( ">",end_n12 );
if ( end_nums2b < 0 )
{
break;
}
if ( end_nums2a > end_nums2b )
{
end_nums2 = end_nums2b;
}
else
{
if (end_nums2a<0)
{
end_nums2 = end_nums2b;
}
else
{
end_nums2 = end_nums2a;
}
}
tmpStr = sTotalSubM.Substring( end_nums2-1,1 );
if ( tmpStr == "\"" || tmpStr == "'" )
{
end_nums2 = end_nums2 - 1;
}
string tmpPicStr = sTotalSubM.Substring( end_n12,end_nums2 - end_n12 );
if ( tmpPicStr.IndexOf( "http://" ) < 0 )
{
if ( tmpPicStr.IndexOf( "/" ) == 0 )
{
tmpPicStr = urlch1 + tmpPicStr;
}
else
{
int linshiIntNum = 0;
int flags = 0;
string urlChange = subUrl;
while( tmpPicStr.IndexOf( "../" ) >= 0 )
{
tmpPicStr = tmpPicStr.Substring( tmpPicStr.IndexOf("../") + 3 );
linshiIntNum = linshiIntNum + 1;
flags = flags + 1;
}
while( ( urlChange.LastIndexOf( "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )
{
urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );
linshiIntNum = linshiIntNum - 1;
}
if ( flags == 0 )
{
tmpPicStr = urlch2 + "/" + tmpPicStr;
}
else
tmpPicStr = urlChange + "/" + tmpPicStr;
}
}
}
//tmpPicStr = tmpPicStr.ToLower();
string tmpPicStrTmp = tmpPicStr.ToLower();
//if ( tmpPicStr.IndexOf( ".jpg" ) > 0 || tmpPicStr.IndexOf( ".gif" ) > 0 || tmpPicStr.IndexOf( ".bmp" ) > 0 )
if ( tmpPicStrTmp.IndexOf( ".jpg" ) > 0 || tmpPicStrTmp.IndexOf( ".gif" ) > 0 || tmpPicStrTmp.IndexOf( ".bmp" ) >0 )
{
rePicStr = rePicStr + "||" + tmpPicStr ;
int flagN2 = tmpPicStr.LastIndexOf( "/" );
string fileN2 = picHtml + tmpPicStr.Substring( flagN2 );
sTotalSubM = sTotalSubM.Substring( 0,end_nums2 ) + ">******" + fileN2 + "******<" + sTotalSubM.Substring( end_nums2 );
begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + fileN2.Length + 22 );
}
else
{
begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + 4 );
}
}
if ( rePicStr.Length > 2 )
rePicStr = rePicStr.Substring(2);
//內容處理 格式化關鍵標記
while( sTotalSubM.IndexOf( "<P" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<P" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<P" ) + 2 );
}
while( sTotalSubM.IndexOf( "<p" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<p" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<p" ) + 2 );
}
while( sTotalSubM.IndexOf( "</P" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</P" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "</P" ) + 3 );
}
while( sTotalSubM.IndexOf( "</p" ) >= 0 )
{ <br>
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</p" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "</p" ) + 3 );
}
while( sTotalSubM.IndexOf( "<br" ) >=0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<br" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<br" ) + 3 );
}
while( sTotalSubM.IndexOf( "<BR" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<BR" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<BR" ) + 3 );
}
while( sTotalSubM.IndexOf( "<Br" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<Br" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<Br" ) + 3 );
}
while( sTotalSubM.IndexOf( "<bR" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<bR" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<bR" ) + 3 );
}
//去除Html標記
int linshiInt1 = sTotalSubM.IndexOf( "<" );
int linshiInt2 = sTotalSubM.IndexOf( ">" );
if ( linshiInt2 < linshiInt1 )
{
sTotalSubM = sTotalSubM.Substring( linshiInt2 + 1 );
}
int linshiInt11 = sTotalSubM.LastIndexOf( "<" );
int linshiInt12 = sTotalSubM.LastIndexOf( ">" );
if ( linshiInt12 < linshiInt11 )
{
= sTotalSubM.Substring( 0,linshiInt12 + 1 );
}
linshiInt1 = sTotalSubM.IndexOf( "<" );
while ( linshiInt1 >= 0 )
{
linshiInt2 = sTotalSubM.IndexOf( ">",linshiInt1 );
if ( linshiInt2 >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 ) + sTotalSubM.Substring( linshiInt2 + 1 );
}
else
{
sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 );
}
linshiInt1 = sTotalSubM.IndexOf("<");
}
//還原關鍵標記
int linshiInt3 = 0;
int linshiInt4 = 0;
while( sTotalSubM.IndexOf( "+****+" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "+****+" ) ) + "<br>\n" + sTotalSubM.Substring( sTotalSubM.IndexOf( "+****+" ) + 9 );
}
while( sTotalSubM.IndexOf( "|****|" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "|****|" ) ) + "<br>\n" + sTotalSubM.Substring( sTotalSubM.IndexOf( "|****|" ) + 9 );
}
while( sTotalSubM.IndexOf( "******" ) >= 0 )
{
linshiInt3 = sTotalSubM.IndexOf( "******" ) + 9;
linshiInt4 = sTotalSubM.IndexOf( "******",linshiInt3 );
if ( linshiInt4 >= 0 )
{
int tmpPos = sTotalSubM.IndexOf( "******" );
string tmpStr1 = sTotalSubM.Substring( 0,tmpPos );
string tmpStr2 = sTotalSubM.Substring( linshiInt3,linshiInt4 - linshiInt3 );
string tmpStr3 = sTotalSubM.Substring( linshiInt4 + 9 );
sTotalSubM = tmpStr1 + "<img src=" + tmpStr2 + ">" + tmpStr3;
}
else
{
break;
}
}
//去除內容中的標題
if ( sTotalSubM.IndexOf( subTitle ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( subTitle ) ) + sTotalSubM.Substring( sTotalSubM.IndexOf( subTitle ) + subTitle.Length );
}
reContentStr = sTotalSubM;
//調用下載圖片功能
//下載圖片到指定目錄
string[] img_Url = new PublicFun().split( rePicStr,"||" );
for ( int i=0;i<img_Url.Length;i++ )
{
if ( img_Url[i] != "" )
{
new PublicFun().Get_Img( img_Url[i],10000,root + "\\images\\" + img_Url[i].Substring( img_Url[i].LastIndexOf("/")+1 ) );
}
}
}
return reContentStr;
}
以上方法返回要取得的信息，包括標題內容，圖片地址等。
下載頁面中圖片：
//下載圖片
public void Get_Img(string a_strUrl,int timeout,string filepath)
{
try
{
HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;
myReq.Timeout = timeout;
HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();
Stream myStream = HttpWResp.GetResponseStream () ;
Bitmap map = new Bitmap( myStream );
PictureBox picB = new PictureBox();
picB.Image = (Image)map;
string path = filepath.Substring( 0,filepath.LastIndexOf( "\\" ) );
if (!Directory.Exists(path))
{
CreateDir( path );
}
picB.Image.Save(filepath);
}
catch(Exception exp)
{
string ss = exp.Message;
WriteLog( filepath.Substring(0,filepath.LastIndexOf("\\")) + "\\error.log",a_strUrl + "--" + ss + "\r\n");
}
}
l 保存文件或入庫
上面取得的信息可以按自己的要求保存。
****設計的時候沒有使用url按層次循環抓取，這樣定義抓取url效率更高，速度更快。 測試程序下載：http://bjfile.focus.cn/file/15379/NetBugV102.rar
如有建議請發送EMail：
[email protected] 或msn
[email protected] 注：此版本只提供靜態文件存儲功能，不提供數據庫接口，不提供自定義網站功能。
本程序運行需要先安裝.Net 框架1.1。
