c#数据采集

这段时间采集一些网络资源，考虑到采集源的防采集屏蔽，于是火车头采集器也排不上用场了，只能自己写一个，部分代码如下：

 

Code
private void gotoRes()
        {
            int num = 0;
            int dnum = 0;
            string strMsg = "执行数据如下：\r\n\r\n";
            DataTable dt = Business.Business.GetUrlTable(Convert.ToInt32(starid.Text), Convert.ToInt32(endid.Text));
            for (int i = 0; i < dt.Rows.Count; i++)  
            {
                if (Convert.ToInt32(dt.Rows[i]["tagid"]) == 0)
                {
                    string strPath = "d:\\res\\";
                    string strFilePath = "";
                    if (dt.Rows[i]["resourceTitle"].ToString().Length > 10)
                    {
                        string[] resourceUrl = SplitString(dt.Rows[i]["resourceUrl"].ToString().ToLower(), "/");
                        for (int length = 5; length < resourceUrl.Length; length++)
                        {
                            strMsg += dt.Rows[i]["rid"].ToString() + dt.Rows[i]["resourceTitle"].ToString() + dt.Rows[i]["resourceUrl"].ToString() + "\r\n";
                            strPath += resourceUrl[length - 1];
                            strFilePath += strPath + "\\" + resourceUrl[length];
                        }
                    }

                    if (!Directory.Exists(strPath))
                    {
                        Directory.CreateDirectory(strPath);
                    }

                    if (downfile2(dt.Rows[i]["resourceUrl"].ToString(), strFilePath))
                    {
                        if (Business.Business.UpDateRes(Convert.ToInt32(dt.Rows[i]["rid"])))
                        {
                            strMsg += strFilePath + "下载完毕！\r\n";
                            dnum++;
                        }
                    }
                    num++;
                }
                msg.Text = strMsg + "\r\n共载入" + num.ToString() + "条数据！下载数据：" + dnum.ToString() + "。";
            }
        }


public bool downfile2(string url, string LocalPath)
        {
            try
            {
                Uri u = new Uri(url);
                HttpWebRequest mRequest = (HttpWebRequest)WebRequest.Create(u);
                mRequest.Method = "GET";
                mRequest.ContentType = "application/x-www-form-urlencoded";
                HttpWebResponse wr = (HttpWebResponse)mRequest.GetResponse();
                statusBar1.Text = "开始下载文件";

                Stream sIn = wr.GetResponseStream();

                FileStream fs = new FileStream(LocalPath, FileMode.Create, FileAccess.Write);
                statusBar1.Text = "error!";
                long length = wr.ContentLength;
                int i = 0;
                long j = 0;
                statusBar1.Text = "正在接收数据";
                byte[] buffer = new byte[1024];
                while ((i = sIn.Read(buffer, 0, buffer.Length)) > 0)
                {
                    j += i;
                    fs.Write(buffer, 0, i);
                    statusBar1.Text = "文件大小:" + length.ToString() + "字节   当前下载:" + j + "字节";
                }

                sIn.Close();
                wr.Close();
                statusBar1.Text = "文件下载完毕文件大小" + fs.Length.ToString() + "字节";
                Console.WriteLine("文件下载完毕文件大小" + fs.Length.ToString() + "字节." + System.DateTime.Now.ToString());
                fs.Close();
                return true;
            }
            catch { return false; }
        }

转载于:https://www.cnblogs.com/kantorgong/archive/2008/11/14/1333590.html