C# 爬虫批量下载文件

  public static void DownFile(string url, string path, string fileName)
        {
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
            request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13";
            request.Credentials = CredentialCache.DefaultCredentials;
            request.Method = "GET";
           // request.Referer = "http://pinyin.sogou.com/dict/list.php?c=180";
            request.Referer = "http://pinyin.sogou.com/dict/cell.php?id=19431";

            request.KeepAlive = false;
            request.Timeout = 2000;
            //request.ContentType="text/plain";
            request.ProtocolVersion = HttpVersion.Version10;
            
            HttpWebResponse response;
            Stream resStream;
            response = (HttpWebResponse)request.GetResponse();
            resStream = response.GetResponseStream();          

            int count = (int)response.ContentLength;
            int offset = 0;
            byte[] buf = new byte[count];
            while (count > 0)
            {
                int n = resStream.Read(buf, offset, count);
                if (n == 0)
                    break;
                count -= n;
                offset += n;
            }

            if (!System.IO.File.Exists(path))
            {
                Directory.CreateDirectory(path);
            }

            FileStream fs = new FileStream( path + "\\" + fileName + ".scel", FileMode.Create, FileAccess.Write);
            fs.Write(buf, 0, buf.Length);
            fs.Flush();
            fs.Close();
            Thread.Sleep(88000);
        }

 

转载于:https://www.cnblogs.com/shanguanghui/p/3612341.html