個人記事

寫法

  Dim myCookieContainer As CookieContainer = New CookieContainer

 Sub T_Cookie()

    Dim cookieStr As String = web.WebBrowser1.Document.Cookie
    Dim cookstr() As String = cookieStr.Split(";")
    Dim ck As Cookie
    Try

    
      For Each str As String In cookstr

        Dim cookieNameValue() As String = str.Split("=")

        If cookieNameValue.Length > 1 Then
          ck = New Cookie(cookieNameValue(0).Trim().ToString(), cookieNameValue(1).Trim().ToString())
        Else
          ck = New Cookie(cookieNameValue(0).Trim().ToString(), "")
        End If
        ck.Domain = "www.youwant.com.tw"
        myCookieContainer.Add(ck)
      Next
    Catch ex As Exception
      Me.Text = ex.ToString
    End Try

  End Sub

利用 HttpWebRequest ，HttpWebResponse

Post 數據 順便取原始碼的 自訂函數

Public Function PostData_Get_Cookie(ByRef URL As String, ByRef POST As String) As String

    Dim MyHttpRequest As HttpWebRequest
    Dim MyHttpResponse As HttpWebResponse
    Dim MyReqStream As Stream
    Dim PostBytes As Byte()
    Dim MyCookie As New CookieContainer
    Dim MySR As IO.StreamReader
    Dim MyStrText As String

    MyHttpRequest = HttpWebRequest.Create(URL)

    MyHttpRequest.Method = "POST"
    MyHttpRequest.KeepAlive = True
    MyHttpRequest.Timeout = 10000 '等待時間
    MyHttpRequest.ContentType = "application/x-www-form-urlencoded"
    MyHttpRequest.Referer = URL

    MyHttpRequest.CookieContainer = MyCookie

    '避免掉 417 錯誤
    MyHttpRequest.ServicePoint.Expect100Continue = False
    PostBytes = Encoding.ASCII.GetBytes(POST)

    MyHttpRequest.ContentLength = PostBytes.Length

    MyReqStream = MyHttpRequest.GetRequestStream
    MyReqStream.Write(PostBytes, 0, PostBytes.Length)

    MyHttpResponse = MyHttpRequest.GetResponse()

    MySR = New StreamReader(MyHttpResponse.GetResponseStream, Encoding.Default)
    MyStrText = MySR.ReadToEnd

    MyReqStream.Close()
    MyHttpResponse.Close()

    Return MyStrText

  End Function

C#實現通過程序自動抓取遠程Web網頁信息的代碼
發布：dxy 字體：[增加 減小] 類型：轉載
通過程序自動的讀取其它網站網頁顯示的信息，類似於爬蟲程序。比方說我們有一個系統，要提取BaiDu網站上歌曲搜索排名。分析系統在根據得到的數據進行數據分析。為業務提供參考數據。 
　　為了完成以上的需求，我們就需要模擬瀏覽器瀏覽網頁，得到頁面的數據在進行分析，最後把分析的結構，即整理好的數據寫入數據庫。那麽我們的思路就是： 
　　1、發送HttpRequest請求。 
　　2、接收HttpResponse返回的結果。得到特定頁面的html源文件。 
　　3、取出包含數據的那一部分源碼。 
　　4、根據html源碼生成HtmlDocument，循環取出數據。 
　　5、寫入數據庫。 
程序如下：　　 


//根據Url地址得到網頁的html源碼 
private string GetWebContent(string Url) 
{ 
string strResult=""; 
try 
{ 
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url); 
　　　　//聲明一個HttpWebRequest請求 
request.Timeout = 30000; 
//設置連接超時時間 
request.Headers.Set("Pragma", "no-cache"); 
HttpWebResponse response = (HttpWebResponse)request.GetResponse(); 
Stream streamReceive = response.GetResponseStream(); 
Encoding encoding = Encoding.GetEncoding("GB2312"); 
StreamReader streamReader = new StreamReader(streamReceive, encoding); 
strResult = streamReader.ReadToEnd(); 
} 
catch 
{ 
MessageBox.Show("出錯"); 
} 
return strResult; 
} 
為了使用HttpWebRequest和HttpWebResponse，需填名字空間引用 
　　using System.Net; 

資料來源

http://itgroup.blueshop.com.tw/cwvdavid/pg?n=convew&i=247135

Public Function ConnectUrl_andGetText(ByVal MyUrl As String) As String
        '連到指定網頁, 並將文字內容抓回來顯示...

        'HttpUtility.UrlEncode("")

        Dim MyHtmlData As String = ""
        Dim myRequest As System.Net.WebRequest 'Request
        Dim myResponse As System.Net.WebResponse 'Response

        Dim myStream As System.IO.StreamReader '利用StreamReader讀取資料

        Dim ContentLength As Long = 0
        Dim LocalFile As System.IO.FileStream

        Try
                myRequest = System.Net.WebRequest.Create(MyUrl)
                myResponse = myRequest.GetResponse()
                ContentLength = myResponse.ContentLength
                'ContentLength = -1

                If ContentLength >= 0 Then
                        '(以UTF8來讀取)
                        myStream = New System.IO.StreamReader(myResponse.GetResponseStream(), System.Text.Encoding.UTF8)

                        '資料讀取到字串變數中...直接整個讀取
                        '(遇到網路不穩時容易出問題)
                        '(遇到ContentLength未知時, 一定出問題...)

                        MyHtmlData = myStream.ReadToEnd()
                Else
                        '如果ContentLength小於0, 就必須另外處理, 
                        '看是要先下載, 另存成文字檔, 再讀取
                        '或是其他方式...

                        Dim Buffer As Byte() = New Byte(2047) {}
                        Dim 暫存位置 As String = Environment.GetEnvironmentVariable("temp") & "\SFCS_TempUrlFile.Text"
                        Dim BytesRead As Long

                        LocalFile = New IO.FileStream(暫存位置, IO.FileMode.Create) 'Create the local file 
                        Do
                                BytesRead = myResponse.GetResponseStream().Read(Buffer, 0, 2048)
                                LocalFile.Write(Buffer, 0, BytesRead)
                        Loop Until BytesRead = 0
                        LocalFile.Flush()
                        LocalFile.Close()

                        myStream = New System.IO.StreamReader(暫存位置, System.Text.Encoding.UTF8)
                        MyHtmlData = myStream.ReadToEnd
                End If
        Catch ex As Exception
                MyHtmlData = "Url讀取錯誤:" & MyUrl & " " & ex.Message
        End Try

        '關閉FileStream
        If Not LocalFile Is Nothing Then
                LocalFile.Close()
        End If

        '關閉StreamReader
        If Not myStream Is Nothing Then
                myStream.Close()
        End If

        '關閉連線
        If Not myResponse Is Nothing Then
                myResponse.Close()
        End If

        Return MyHtmlData '這是要回傳的內容...

End Function

''' Function made by Vyor - www.districtecho.net
''' <summary>
''' Submits POST Data to a specified URL.
''' </summary>
''' <param name="POST">What to be POSTed to the URL. Uses standard POST parameter format.</param>
''' <param name="Cookies">The cookies your request will be making. Simply declare as new if you don't want to share cookies across other requests.</param>
''' <returns>HTML of the page recieved after POSTing.</returns>
Public Function PostData(ByRef URL As String, ByRef POST As String, ByRef Cookies As CookieContainer) As String
Dim request As HttpWebRequest
Dim response As HttpWebResponse
request = CType(WebRequest.Create(URL), HttpWebRequest)
request.ContentType = "application/x-www-form-urlencoded"
request.ContentLength = POST.Length
request.Method = "POST"
request.AllowAutoRedirect = False
Dim requestStream As Stream = request.GetRequestStream()
Dim postBytes As Byte() = Encoding.ASCII.GetBytes(POST)
requestStream.Write(postBytes, 0, postBytes.Length)
requestStream.Close()
response = CType(request.GetResponse(), HttpWebResponse)
Return New StreamReader(response.GetResponseStream()).ReadToEnd()
End Function

Function GetImageFromURL(ByVal strUrl As String) As System.Drawing.Image

 Dim MyRequest As Net.HttpWebRequest = Net.WebRequest.Create(strUrl) MyRequest.Referer = strUrl

 '這是為了抓有檢查referer的圖片 

 MyRequest.UserAgent = "FireDog" '這只是為了好玩

 Dim MyResponse As Net.HttpWebResponse = MyRequest.GetResponse() 

 Dim MyStream As IO.Stream = MyResponse.GetResponseStream 

 Dim MyImage As System.Drawing.Image = System.Drawing.Image.FromStream(MyStream) 

 MyStream.Close()

 MyResponse.Close()

 Return MyImage

 End Function

 來源: http://www.blueshop.com.tw/board/show.asp?subcde=BRD20060721181152IN1