如何实现多线程并发执行一组数据?这里主要用到了先入先出的Queue。 namespace System.Collections.Specialized;
原理:把要执行的参数等放入Queue里面,在执行的时候从Queue取出并且移除,这样也就做到了数据不会重复操作。
下面以采集500W的时时彩数据为例....
private Queue<string> needSpideData;
private delegate void SetRichText(string txt,bool b);
private void SetText(string txt,bool b)
{
if (richTextBox1.InvokeRequired)
{
SetRichText srt = new SetRichText(SetText);
this.Invoke(srt, new object[] { txt,b });
}
else
{
if (b)
richTextBox1.Text += txt + "\n";
else
richTextBox2.Text += txt + "\n";
}
}
private void button1_Click(object sender, EventArgs e)
{
richTextBox1.Text = "开始采集.... \n";
richTextBox2.Text = "";
int day = (dateTimePicker2.Value - dateTimePicker1.Value).Days + 1;
needSpideData = new Queue<string>();
for (int i = 0; i < day; i++)
{
needSpideData.Enqueue(string.Format("http://kaijiang.500wan.com/static/public/ssc/xml/qihaoxml/{0}.xml", dateTimePicker1.Value.AddDays(i).ToString("yyyyMMdd")));
}
for (int i = 0; i < 10; i++)
{
Thread t = new Thread(new ThreadStart(Spide));
t.Start();
}
}
private void Spide()
{
try
{
if (needSpideData.Count > 0)
{
var url = needSpideData.Dequeue();
switch (ToGet.SaveSql(url))//操作数据库
{
case 0: SetText(url + "\t采集失败,网络错误,或数据库已存在该数据", false); break;
case 1: SetText(url + "\t采集失败,字段不匹配", false); break;
case 2: SetText(url + "\t采集成功", true); break;
}
Spide();
}
else
{
label4.Text = "采集完成";
}
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
/// <summary>
/// 检查网络中是否存在文件
/// </summary>
/// <param name="issue"></param>
/// <returns></returns>
private bool RemoteFileExists(string url)
{
try
{
string urls = url;
HttpWebRequest re = (HttpWebRequest)WebRequest.Create(urls);
HttpWebResponse res = (HttpWebResponse)re.GetResponse();
re.Timeout = 5000;
if (res.ContentLength != 0)
{
//MessageBox.Show("文件存在");
return true;
}
}
catch (Exception)
{
//MessageBox.Show("无此文件");
return false;
}
return false;
}