C#爬虫抓取数据并使用NOPI保存excel

string url = "http://49.35.23.21/project/Ashx/GetDInfo.ashx";
            var client = new HttpClient();
            client.DefaultRequestHeaders.Add("Accept", "application/json, text/javascript, */*; q=0.01");
            var request = new HttpRequestMessage(HttpMethod.Post,url);
            FormUrlEncodedContent formContent = new FormUrlEncodedContent(new Dictionary<string, string>()
            {
                {"PageIndex","0" },
                {"PageSize","800" }
             });

            var response = client.PostAsync(url, formContent).Result;
            if (response.IsSuccessStatusCode)
            {
                string result = response.Content.ReadAsStringAsync().Result;
                var list = JsonConvert.DeserializeObject<Model>(result);
                //将WorkBook指到我们原本设计好的Templete Book1.xls
                IWorkbook wb = new HSSFWorkbook(new FileStream("D:/sample.xls", FileMode.Open));
                //设定要使用的Sheet为第0个Sheet
                ISheet TempSheet = wb.GetSheetAt(0);
                int StartRow = 1;
                foreach (var item in list.list) {
                    Console.WriteLine("抓取" + StartRow + "  " + DateTime.Now.ToString());
                    url = "http://49.35.23.21/Project/ProjectInfo/AQJDInfo/AQJDInfo_Detail.aspx?RowGuid="+item.RowGuid;
                    //HttpResponseMessage detailresponse = new HttpClient().GetAsync(url).Result;
                    //if (detailresponse.IsSuccessStatusCode)
                    HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
                    req.Method = "GET";
                    req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
                    //req.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
                    //req.Headers.Add(HttpRequestHeader.Accept, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
                    req.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip");//定义gzip压缩页面支持
                    req.KeepAlive = true;//启用长连接
                    req.Headers.Add(HttpRequestHeader.CacheControl ,"max-age=0");
                    req.AutomaticDecompression = DecompressionMethods.GZip;
                    //req.Headers.Add(HttpRequestHeader.Host ,"49.65.0.83");
                    req.Headers.Add(HttpRequestHeader.ProxyAuthorization , "keep-alive");
                    req.Headers.Add(HttpRequestHeader.Upgrade , "1");
                    req.Headers.Add(HttpRequestHeader.Cookie, "ASP.NET_SessionId=thxxyowls4i2v3d110rsyx22");
                    req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36";
                    //req.Headers.Add(HttpRequestHeader.UserAgent, "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36");
                    using (WebResponse wr = req.GetResponse())
                    {
                        //string detailResult = new StreamReader(wr.GetResponseStream(), Encoding.GetEncoding("gb2312")).ReadToEnd();
                        string detailResult = new StreamReader(wr.GetResponseStream()).ReadToEnd();
                        Console.WriteLine("抓取成功" + "  " + DateTime.Now.ToString());
                        HtmlDocument htmlDoc = new HtmlDocument();
                        htmlDoc.LoadHtml(detailResult);
                        string HeTongJia = htmlDoc.DocumentNode.SelectSingleNode("//span[@id='HeTongJia']").InnerText.Trim();
                        string ShiGongDanWei = htmlDoc.DocumentNode.SelectSingleNode("//span[@id='ShiGongDanWei']").InnerText.Trim();
                        string PMName = htmlDoc.DocumentNode.SelectSingleNode("//span[@id='PMName']").InnerText.Trim();
                        string BaoJianRenTel = htmlDoc.DocumentNode.SelectSingleNode("//span[@id='BaoJianRenTel']").InnerText.Trim();
                        item.HeTongJia = HeTongJia;
                        item.PMName = PMName;
                        item.ShiGongDanWei = ShiGongDanWei;
                        item.BaoJianRenTel = BaoJianRenTel;

                        try
                        {
                            //tDS为Query回来的资料
                            //第一个Row要用Create的
                            int i = 0;
                            TempSheet.CreateRow(StartRow + i).CreateCell(0).SetCellValue(Convert.ToString(item.AJNo));
                            //第二个Row之后直接用Get的
                            TempSheet.GetRow(StartRow + i).CreateCell(1).SetCellValue(Convert.ToString(item.AJProjectName));
                            TempSheet.GetRow(StartRow + i).CreateCell(2).SetCellValue(Convert.ToString(item.AJProjectName_Short));
                            TempSheet.GetRow(StartRow + i).CreateCell(3).SetCellValue(Convert.ToString(item.BaoJianDate));
                            TempSheet.GetRow(StartRow + i).CreateCell(4).SetCellValue(Convert.ToString(item.BaoJianRenTel));
                            TempSheet.GetRow(StartRow + i).CreateCell(5).SetCellValue(Convert.ToString(item.BiaoDuanNo));
                            TempSheet.GetRow(StartRow + i).CreateCell(6).SetCellValue(Convert.ToString(item.HeTongJia));
                            TempSheet.GetRow(StartRow + i).CreateCell(7).SetCellValue(Convert.ToString(item.InfoSource));
                            TempSheet.GetRow(StartRow + i).CreateCell(8).SetCellValue(Convert.ToString(item.PMName));
                            TempSheet.GetRow(StartRow + i).CreateCell(9).SetCellValue(Convert.ToString(item.PrjNum));
                            TempSheet.GetRow(StartRow + i).CreateCell(10).SetCellValue(Convert.ToString(item.RowGuid));
                            TempSheet.GetRow(StartRow + i).CreateCell(11).SetCellValue(Convert.ToString(item.RowXH));
                            TempSheet.GetRow(StartRow + i).CreateCell(12).SetCellValue(Convert.ToString(item.ShiGongDanWei));
                            //将文档写到指定位置
                            //using (FileStream file = new FileStream("D:/Test_NPOI4.xls", FileMode.Create))
                            using (FileStream file = File.OpenWrite(@"D:/sample.xls"))
                            {
                                wb.Write(file);
                                file.Close();
                                file.Dispose();
                            }
                            Console.WriteLine("写入成功");
                            StartRow++;
                        }
                        catch (Exception e)
                        {
                            string a = e.ToString();
                            Console.WriteLine("Out");
                        }
                    }
                }
                Console.WriteLine("Over");
                Console.ReadKey();
            }

 

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值