用C#.NET 与Webdriver写的抓取网页信息的小工具

    最近,女友的妹妹要去网上找房产中介人信息用于招聘,自己去网上一个一个找太慢,我女友知道我是搞IT的就请教我有没有办法帮她快速找电话号码,于是我就想到了爬虫程序,然后普通的爬虫代码有限制,刚好自己在搞自动化测试,就想到用webdriver自动化测试工具编写一个爬虫工具抓取赶集网上的经纪人信息。

    自己一直用的是Java代码写的自动化测试脚本,但是对于他们不会编程的人,没有界面操作很不方便,于是我就想到了编写一个桌面程序,但是java的GUI又不美观,最后就想到微软的C#,于是自学VS2010,自学.NET,C#结合Webdriver FOR C#版本编写了一个小工具。

 

支持火狐浏览器,phatomJS内存浏览器进行抓取。

将数据查询出放在界面上

支持导出excel文件

基本的实现原理就是:先计算出页码,然后循环一页一页抓取数据,抓取出的数据先临时存储在LIST数据类型中,再存放到界面上的数据控件,点击导出excel的时候,把数据控件的数据转换成LIST,再导出到excel.

界面如下:

  

 

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using OpenQA.Selenium;
using OpenQA.Selenium.Firefox;
using OpenQA.Selenium.Interactions;
using OpenQA.Selenium.Interactions.Internal;
using OpenQA.Selenium.Support;
using System.Threading;
using Microsoft.Office;
using Excel;
using System.Drawing;
using OpenQA.Selenium.PhantomJS;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.IE;
using OpenQA.Selenium.Remote;
using System.Timers;

namespace tel_search
{
    public partial class Form1 : Form
    {
        //计时器
        static int time_js = 0;//设定临时变量
        static System.Timers.Timer timer;
        
        public Form1()
        {
            InitializeComponent();
        }

        private void label1_Click(object sender, EventArgs e)
        {

        }

        private void label2_Click(object sender, EventArgs e)
        {

        }

        private void label3_Click(object sender, EventArgs e)
        {

        }

        private void Form1_Load(object sender, EventArgs e)
        {
            radioButton_phatom.Checked = true;
        }

        private void button1_Click(object sender, EventArgs e)
        {
            //开始后台运行搜索
            //backgroundWorker1.RunWorkerAsync(200000);
            
            if (listBox1.SelectedItems.Count == 0)
            {
                DialogResult dr5 = MessageBox.Show("请选择地区", "温馨提示", MessageBoxButtons.OK);
                if (dr5 == DialogResult.OK)
                {
                    listBox1.Focus();
                }
            }
            else
            {
                List<String[]> mp_inofs_list = new List<string[]>();
                //用于统计名片个数
                int mp_total = 0;
                //记录错误关键字
                //List<String> err_kw = null;
                IWebDriver driver = null;
                DialogResult dr_search_result = MessageBox.Show("数据比较多,可能需要几分钟时间抓取,请耐心等待!运行过程中,请不要关闭浏览器窗口,否则程序会报错!", "温馨提示", MessageBoxButtons.OK);
                if (dr_search_result == DialogResult.OK)
                {
                    //抓取数据过程,开始计时
                    timer = new System.Timers.Timer(100);
                    timer.Elapsed += new System.Timers.ElapsedEventHandler(OnTimedEvent);
                    timer.AutoReset = true;
                    timer.Enabled = true;
                    
                    if (radioButton_phatom.Checked == true)
                    {
                        driver = new PhantomJSDriver();
                    }
                    if (radioButton_firefox.Checked == true)
                    {
                        try
                        {
                            FirefoxProfile profile = new FirefoxProfile();

                            profile.SetPreference("browser.bookmarks.restore_default_bookmarks", false);
                            driver = new FirefoxDriver(profile);
                            
                        }
                        catch
                        {
                            DialogResult dr4 = MessageBox.Show("你的电脑没有安装火狐浏览器,是否立即下载", "温馨提示", MessageBoxButtons.YesNo);
                            if (dr4 == DialogResult.Yes)
                            {
                                listBox1.Focus();
                                System.Diagnostics.Process.Start("http://download.firefox.com.cn/releases/stub/official/zh-CN/Firefox-latest.exe");
                                //中止程序往下运行
                                return;

                            }
                            if (dr4 == DialogResult.No)
                            {
                                return;
                            }
                        }
                    }
                }



                String quyu = null;
                String url_quyu = null;
                driver.Manage().Window.Maximize(); 

                try
                {
                    if (listBox1.SelectedItem.ToString() == "全深圳")
                    {
                        quyu = "";

                    }
                    if (listBox1.SelectedItem.ToString() == "福田")
                    {
                        quyu = "futian";

                    }
                    if (listBox1.SelectedItem.ToString() == "罗湖")
                    {
                        quyu = "luohu";

                    }
                    if (listBox1.SelectedItem.ToString() == "南山")
                    {
                        quyu = "nanshan";

                    }
                    if (listBox1.SelectedItem.ToString() == "宝安")
                    {
                        quyu = "baoan";

                    }
                    if (listBox1.SelectedItem.ToString() == "龙岗")
                    {
                        quyu = "longgang";

                    }
                    if (listBox1.SelectedItem.ToString() == "盐田")
                    {
                        quyu = "yantian";

                    }
                    if (listBox1.SelectedItem.ToString() == "龙华新区")
                    {
                        quyu = "longhuaxinqu";

                    }
                    if (listBox1.SelectedItem.ToString() == "光明新区")
                    {
                        quyu = "guangmingxinqu";

                    }
                    if (listBox1.SelectedItem.ToString() == "坪山新区")
                    {
                        quyu = "pingshanxinqu";

                    }
                    if (listBox1.SelectedItem.ToString() == "大鹏新区")
                    {
                        quyu = "dapengxinqu";
                    }
                    if (listBox1.SelectedItem.ToString() == "深圳周边")
                    {
                        quyu = "shenzhenzhoubian";

                    }
                    
                    
                        url_quyu = "http://sz.ganji.com/zufang/agent/" + quyu + "/";
                        driver.Navigate().GoToUrl(url_quyu);
                    

                }
                catch
                {
                    driver.Quit();
                    DialogResult dr_search_result2 = MessageBox.Show("打不开网页,您的网络可能有问题,请检查网络连接是否正常!", "温馨提示");
                    return;

                }
                
                    List<IWebElement> page_next_page_button = new List<IWebElement>();
                    List<IWebElement> page_after_index = null;
                    
                    do
                    {
                        page_next_page_button = new List<IWebElement>(driver.FindElements(By.XPath("//div[@class='pageBox']/ul[@class='pageLink clearfix']/li/a[@class='next']/span")));
                        
                        page_after_index = new List<IWebElement>(driver.FindElements(By.XPath("//div[@class='pageBox']/ul[@class='pageLink clearfix']/li/a[not(@class='next'or@class='prev')]")));
                        page_after_index.ElementAt(page_after_index.Count - 1).Click();


                    }while (page_next_page_button.Count!=0);//判断是否查找到"下一页"的按钮.特别注意不能用!=null判断,这样是无效的。

                    //driver.Close();不能关闭浏览器,如果关闭了,需要重新创建driver对象
                    int page_nums = Convert.ToInt16(page_after_index.ElementAt(page_after_index.Count-1).Text);
                  

              
                //获取每个关键词含有多少页,通过获取页码元素判断
                   url_quyu = "http://sz.ganji.com/zufang/agent/" + quyu + "/";
                   driver.Navigate().GoToUrl(url_quyu);
                  List<IWebElement> page_index = new List<IWebElement>(driver.FindElements(By.XPath("//div[@class='pageBox']/ul[@class='pageLink clearfix']/li/a[not(@class='next'or@class='prev')]")));
               

                  for (int page_num = 0; page_num < page_nums; page_num++)
                {
                    //每次点击页码之后,都要重新找元素
                    driver.Manage().Timeouts().ImplicitlyWait(new TimeSpan(0, 0, 1));
					//以一个集合作为参数创建list   LIST<T> TESTLIST=NEW LIST<T>(IEnumerable<T> Collections)
                    List<IWebElement> page = new List<IWebElement>(driver.FindElements(By.XPath("//div[@class='pageBox']/ul[@class='pageLink clearfix']/li/a[not(@class='next'or@class='prev')]")));
                    String page_num_text = null;
                    if (page_num <=5)
                    {
                        page_num_text = page.ElementAt(page_num).Text;
                       
                        page.ElementAt(page_num).Click();
                    }
                    else
                    {
                        page_num_text = page.ElementAt(5).Text;
                       
                        page.ElementAt(5).Click();
                    }

                    driver.Manage().Timeouts().ImplicitlyWait(new TimeSpan(0, 0, 1));

                    //获取每页所含名片元素,同时获取每页名片数量
                    List<IWebElement> mp_indx = new List<IWebElement>(driver.FindElements(By.XPath("//div[@class='listBox']/ul/li/div[@class='list-mod2']")));

                    //列表用于存储单个名片信息
                    List<String> mp_info_list = new List<String>();
                    //第一个detailLayer元素不含名片信息过滤掉,所以从1开始

                    for (int i = 0; i < mp_indx.Count(); i++)
                    {
                        

                        //获取单个名片内容元素----Start
                    
                        IWebElement mp_name = mp_indx.ElementAt(i).FindElement(By.XPath("div[@class='broker-info']/span[@class='broker-name']/a"));
                        IWebElement mp_tel = mp_indx.ElementAt(i).FindElement(By.XPath("div[@class='broker-info']/span[@class='broker-tel']"));
                        IWebElement mp_compy = mp_indx.ElementAt(i).FindElement(By.XPath("div[@class='broker-service']")).FindElements(By.TagName("p")).ElementAt(0);
                        IWebElement mp_serverquyu = mp_indx.ElementAt(i).FindElement(By.XPath("div[@class='broker-service']")).FindElements(By.TagName("p")).ElementAt(1);
                        IWebElement mp_serverxiaoqu = mp_indx.ElementAt(i).FindElement(By.XPath("div[@class='broker-service']")).FindElements(By.TagName("p")).ElementAt(2);
                        
                       
                        String mp_name_text = mp_name.Text;
                        String mp_tel_text = mp_tel.Text;
                        //Substring函数,截取公司名字,删除开头的“经纪公司“几个字
                        String mp_compy_text = mp_compy.Text.Substring(6,mp_compy.Text.ToString().Length-6);
                        String mp_serverquyu_text = mp_serverquyu.Text.Substring(6,mp_serverquyu.Text.ToString().Length-6);
                        String mp_serverxiaoqu_text = mp_serverxiaoqu.Text.Substring(6,mp_serverxiaoqu.Text.ToString().Length-6);

                        mp_info_list.Add(listBox1.SelectedItem.ToString());
                        mp_info_list.Add(mp_name_text);
                        mp_info_list.Add(mp_tel_text);
                        mp_info_list.Add(mp_compy_text);
                        mp_info_list.Add(mp_serverquyu_text);
                        mp_info_list.Add(mp_serverxiaoqu_text);
                        mp_info_list.Add(page_num_text);

                        //------------------END
                        //将单个名片内容列表转为数组
                        String[] mp_info_arrary = mp_info_list.ToArray();

                        //将单个名片,存储到名片二维列表
                        mp_inofs_list.Add(mp_info_arrary);
                        //将名片列表内容删除 ,用于存储下一个名片内容
                        mp_info_list.RemoveRange(0, mp_info_list.Count());
                        mp_total++;

                    }
                    

                }

                //将所有名片信息存储为一个二维数组
                String[][] mp_infos_arrary = mp_inofs_list.ToArray();
                //列宽设置
                col_header1.Width = 30;
                col_header2.Width = 45;
                col_header3.Width = 100;
                col_header4.Width = 90; 
                col_header5.Width = 350;
                col_header6.Width = 115;
                col_header7.Width = 465;
                col_header8.Width = 20;

                //开始更新LISTVIEW数据--START
                listView1.GridLines = true;
                listView1.MultiSelect = true;
                //listView1.Clear();
                listView1.BeginUpdate();
                for (int i = 0; i < mp_inofs_list.Count; i++)
                {
                    ListViewItem Lvi = new ListViewItem();
                    //第一列序号
                    Lvi.Text = (i + 1).ToString();
                    for (int j = 0; j < 7; j++)
                    {

                        Lvi.SubItems.Add(mp_inofs_list.ElementAt(i).ElementAt(j));
                    }
                    listView1.Items.Add(Lvi);
                }
                listView1.EndUpdate();
                listView1.LabelEdit = true;
                listView1.FullRowSelect = true;
             
                driver.Quit();
                

              
                //结束计时
                timer.Enabled = false;
                timer.Dispose();

                time_lbl.Visible = true;
                //换算成分秒
                int minute = 0;//分
                double second = 0;//秒
                second = time_js / 10;
                if (second > 60)
                {
                    minute = (int)(second / 60);
                    second = second % 60;
                    time_lbl.Text = minute.ToString() + "分" + second.ToString() + "秒";
                }
                else
                {
                    time_lbl.Text = 0.ToString() + "分" + second.ToString() + "秒";
                }
                DialogResult msg_mp_total = MessageBox.Show("谢谢你的耐心等待" + "成功抓取" + page_nums + "页" + mp_inofs_list.Count + "条名片信息,用时:" + time_lbl.Text, "恭喜");
               



            }
        }

        private void listView1_SelectedIndexChanged(object sender, EventArgs e)
        {

        }

        private void button4_Click(object sender, EventArgs e)
        {
            turntoexcel();
        }
        private void turntoexcel()
        {
            SaveFileDialog sfd = new SaveFileDialog();
            sfd.DefaultExt = "xls";
            sfd.Filter = "Excel文件(*.xls)|*.xls";
            if (sfd.ShowDialog() == DialogResult.OK)
            {

                DoExport(listView1, sfd.FileName);

            }


        }
        private void DoExport(ListView listView, string strFileName)
        {

            if (listView1.Items.Count== 0)
            {
                MessageBox.Show("没有数据,无法导出!");
                return;
            }
            int rowNum = listView.Items.Count;
           
            int columnNum = listView.Items[0].SubItems.Count;
            int rowIndex = 1;
            int columnIndex = 0;
            if (rowNum == 0 || string.IsNullOrEmpty(strFileName))
            {
                return;
            }
            if (rowNum > 0)
            {

                Microsoft.Office.Interop.Excel.Application xlApp = new Microsoft.Office.Interop.Excel.ApplicationClass();
                if (xlApp == null)
                {
                    MessageBox.Show("无法创建excel对象,可能您的系统没有安装excel");
                    return;
                }
                xlApp.DefaultFilePath = "";
                xlApp.DisplayAlerts = true;
                xlApp.SheetsInNewWorkbook = 1;
                Microsoft.Office.Interop.Excel.Workbook xlBook = xlApp.Workbooks.Add(true);
                
                //将ListView的列名导入Excel表第一行
                foreach (ColumnHeader dc in listView.Columns)
                {
                    columnIndex++;
                    xlApp.Cells[rowIndex, columnIndex] = dc.Text;
                }
                //将ListView中的数据导入Excel中
                for (int i = 0; i < rowNum; i++)
                {
                    rowIndex++;
                    columnIndex = 0;
                    for (int j = 0; j < columnNum; j++)
                    {
                        columnIndex++;
                        //注意这个在导出的时候加了“\t” 的目的就是避免导出的数据显示为科学计数法。可以放在每行的首尾。
                        xlApp.Cells[rowIndex, columnIndex] = Convert.ToString(listView.Items[i].SubItems[j].Text) + "\t";
                    }
                }
                //例外需要说明的是用strFileName,Excel.XlFileFormat.xlExcel9795保存方式时 当你的Excel版本不是95、97 而是2003、2007 时导出的时候会报一个错误:异常来自 HRESULT:0x800A03EC。 解决办法就是换成strFileName, Microsoft.Office.Interop.Excel.XlFileFormat.xlWorkbookNormal。
                xlBook.SaveAs(strFileName, Microsoft.Office.Interop.Excel.XlFileFormat.xlWorkbookNormal, Type.Missing, Type.Missing, Type.Missing, Type.Missing, Microsoft.Office.Interop.Excel.XlSaveAsAccessMode.xlExclusive, Type.Missing, Type.Missing, Type.Missing, Type.Missing, Type.Missing);
                xlApp = null;
                xlBook = null;
                MessageBox.Show("恭喜导出成功!");
            }
        }

        private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
        {
            //允许长时间的操作
            int input = (int)e.Argument;
            Thread.Sleep(input);

        }

        private void textBox1_TextChanged(object sender, EventArgs e)
        {

        }

        private void linkLabel1_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e)
        {
            linkLabel1.Links[0].LinkData = "http://download.firefox.com.cn/releases/stub/official/zh-CN/Firefox-latest.exe";
            String URL = linkLabel1.Links[0].LinkData.ToString();
            System.Diagnostics.Process.Start(URL);
        }
        private  void OnTimedEvent(Object source, ElapsedEventArgs e)
        {
            if (timer.Enabled == true)
            {
                time_js++;
                
            }
            else
            {
                return;
            }

        }

       

    }
}


转载于:https://my.oschina.net/zhengshuheng/blog/512522

  • 0
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值