using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Data.SqlClient;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using DataCollect.Model;
using HtmlAgilityPack;//引入插件
using Dapper;//引入插件
namespace DataCollect.UI
{
public partial class MainForm1 : DevExpress.XtraEditors.XtraForm
{
CancellationTokenSource cts = new CancellationTokenSource();
List<Job> list = new List<Job>();
public MainForm1()
{
InitializeComponent();
}
private void MainForm1_Load(object sender, EventArgs e)
{
}
private void simpleButton1_Click(object sender, EventArgs e)
{
//线程与线程之前不能直接访问(线程是独立的任务片段)
Task task1 = new Task(Collect);//创建一个线程任务
task1.Start();
}
/// <summary>
/// 数据采集
/// </summary>
private void Collect()
{
int index = 1;
int pageCount=0;
do
{
//HtmlAgilityPack.HtmlDocument htmlDocument=DataLoad(index);
var htmlDocument = DataLoad(index);
if (index == 1)
{
//根据查询节点信息,获取某个节点
HtmlNode htmlNode = htmlDocument.DocumentNode.SelectSingleNode("//input[@id='hidTotalPage']");//标答节点//根节点
pageCount = int.Parse(htmlNode.Attributes["value"].Value);
主进程、>一对多主关系
//this.Invoke(new Action(() =>
//{ this.labelControl6.Text = pageCount.ToString();
//}));
SetLable(this.labelControl6,pageCount.ToString());
//labelControl6.Text = index.ToString();
}
//找到了标题
HtmlNode hn=htmlDocument.DocumentNode.SelectSingleNode("//div[@class='el title']");
//调用递归
GetNode(hn.NextSibling.NextSibling);
//换行符也算标签
if (cts.IsCancellationRequested) return;
//基础信息循环
index++;
//采集到第几页
SetLable(this.labelControl4, index.ToString());
} while (index< pageCount);
}
/// <summary>
/// 递归检测
/// </summary>
/// <param name="parentNode"></param>
private void GetNode(HtmlNode parentNode)
{
if (cts.IsCancellationRequested) return;//判断取消,终止代码
Thread.Sleep(100);//休眠0.1秒
if (parentNode != null && parentNode.ChildNodes.Count > 0)
{
Job job = new Job();
job.Position = parentNode.ChildNodes[1].InnerText.Replace("\r\n","").Trim();
job.Company = parentNode.ChildNodes[3].InnerText;
job.Address = parentNode.ChildNodes[5].InnerText;
job.Salary = parentNode.ChildNodes[7].InnerText;
job.SendTime = parentNode.ChildNodes[9].InnerText;
list.Add(job);//每次采集一条累加
SetLable(labelControl2,list.Count.ToString());
//处理串线程
this.listView1.Invoke(new Action(() =>
{
ListViewItem item = new ListViewItem();//数据行
item.Text = job.Position;//第一个值
item.SubItems.AddRange(new string[] {
job.Company,
job.Address,
job.Salary,
job.SendTime
});
this.listView1.Items.Add(item);//数据控件
}));
}
//打到一个可以结束的条件
if (parentNode.NextSibling.Attributes["class"] != null &&
parentNode.NextSibling.Attributes["class"].Value == "dw_page")
return;//不要再找了
GetNode(parentNode.NextSibling);//继续往下
}
/// <summary>
/// 通过线程中Lable赋值
/// </summary>
/// <param name="lbl"></param>
/// <param name="value"></param>
private void SetLable(DevExpress.XtraEditors.LabelControl lbl, string value)
{
lbl.Invoke(new Action(() => { lbl.Text = value; })) ;
}
/// <summary>
/// 根据url地址解析HTML字符串
/// </summary>
/// <param name="index"></param>
/// <returns></returns>
private HtmlAgilityPack.HtmlDocument DataLoad(int index)
{
HtmlWeb htmWeb = new HtmlWeb();
htmWeb.OverrideEncoding = Encoding.GetEncoding("gbk");//动态字符集
//采集别人网站时(简体中文网站:gb2312 gbk utf)
string url = $"https://search.51job.com/list/000000,000000,0000,00,9,99,.net,2,{index}.html";
return htmWeb.Load(url);
}
/// <summary>
/// 取消事件
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void simpleButton2_Click(object sender, EventArgs e)
{
cts.Cancel();//取消当前任务
}
/// <summary>
/// 导出数据
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void simpleButton3_Click(object sender, EventArgs e)
{
using (SqlConnection conn = new SqlConnection("server=.;uid=sa;pwd=adminsystem;database=QCSJ"))
{
string sql = @"insert into Job(Position,Company,Address,Salary,SendTime) values(@Position,@Company,@Address,@Salary,@SendTime)";
int result=conn.Execute(sql, list);
if (result > 0)
{
MessageBox.Show("采集完毕!!");
}
}
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace DataCollect.Model
{
public class Job
{
/// <summary>
/// id
/// </summary>
public int id { get; set; }
/// <summary>
/// 职位
/// </summary>
public string Position { get; set; }
/// <summary>
/// 公司
/// </summary>
public string Company { get; set; }
/// <summary>
/// 地址
/// </summary>
public string Address { get; set; }
/// <summary>
/// 薪资
/// </summary>
public string Salary { get; set; }
/// <summary>
/// 发布时间
/// </summary>
public string SendTime { get; set; }
}
}
USE master
GO
IF EXISTS(SELECT * FROM SYS.DATABASES WHERE NAME='QCSJ')
DROP DATABASE QCSJ
GO
CREATE DATABASE QCSJ
ON PRIMARY
(
NAME=QCSJ_DATA,
FILENAME='E:\DataBase_Source\QCSJ_DATA.mdf',
SIZE=5MB,
MAXSIZE=1000MB,
FILEGROWTH=15%
)
LOG ON
(
NAME=QCSJ_LOG,
FILENAME='E:\DataBase_Source\QCSJ_LOG.ldf',
SIZE=5MB,
FILEGROWTH=15%
)
GO
USE QCSJ
GO
IF EXISTS(SELECT * FROM SYS.OBJECTS WHERE NAME='Job')
DROP TABLE Job
GO
CREATE TABLE Job
(
id int identity(1,1) primary key,
Position varchar(50) not null,--------------职位
Company varchar(50) not null,---------------公司
Address varchar(100)not null,---------------地址
Salary varchar(30)not null,-----------------薪资
SendTime varchar(30)not null--------------发布时间
)
GO