基于winform的百度图片下载爬虫
##话不多说,上代码!
界面程序代码 BaiduSpider.cs
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
namespace baiduSpider
{
public partial class BaiduSpider : Form
{
public BaiduSpider()
{
InitializeComponent();
}
private void button2_Click(object sender, EventArgs e)
{
FolderBrowserDialog fbd = new FolderBrowserDialog();
fbd.Description = "请选择文件保存文件夹";
fbd.ShowNewFolderButton = true;
fbd.SelectedPath = Environment.CurrentDirectory;
if (fbd.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
this.t_savePath.Text = fbd.SelectedPath;
}
}
private void btn_downLoad_Click(object sender, EventArgs e)
{
//验证关键词是否填写
if (String.IsNullOrEmpty(this.t_keyword.Text))
{
MessageBox.Show("关键字不能为空");
return;
}
//验证保存路径是否填写
if (String.IsNullOrEmpty(this.t_savePath.Text))
{
MessageBox.Show("保存路径不能为空");
return;
}
//验证保存路径是否存在
if (!Directory.Exists(this.t_savePath.Text))
{
try
{
Directory.CreateDirectory(this.t_savePath.Text);
}
catch
{
MessageBox.Show("文件夹创建失败");
return;
}
}
//验证文件夹可写性
try
{
using (FileStream fs = new FileStream(Path.Combine(this.t_savePath.Text, "test-file.txt"), FileMode.Create)) ;
File.Delete(Path.Combine(this.t_savePath.Text, "test-file.txt"));
}
catch
{
MessageBox.Show("文件夹不可写");
return;
}
//resultList.Items.Add(GetUrl());
//启用线程
String URL = GetUrl();
Thread thread = new Thread(() => { Downloads(URL, (int)n_page.Value); });
thread.Start();
}
//下载方法
private void Downloads(String url, int pages)
{
for (int i = 0; i < pages; i++)
{
String Url = url.Replace("[REPLACE]",(30*i).ToString());
Stream stream;
try
{
//获取流
stream = Download.DownLoadFile(Url);
}
catch
{
continue;
}
String Html;
try
{
using (StreamReader sr = new StreamReader(stream))
{
Html = sr.ReadToEnd();
}
}
catch
{
continue;
}
JObject jobj = JObject.Parse(Html);
JArray jarr = (JArray)jobj["data"];
for (int j = 0; j < jarr.Count; j++)
{
try
{
String picUrl = jarr[j]["thumbURL"].ToString();
String savePath = Path.Combine(t_savePath.Text, ReName(picUrl.Substring(picUrl.LastIndexOf("/") + 1)));
String picReferer = "http://image.baidu.com/";
if (jarr[j].ToString().Contains("replaceUrl"))
{
picUrl = jarr[j]["replaceUrl"][0]["ObjURL"].ToString();
savePath = Path.Combine(t_savePath.Text, ReName(picUrl.Substring(picUrl.LastIndexOf("/") + 1)));
picReferer = jarr[j]["replaceUrl"][0]["FromURL"].ToString();
}
//如果已经存在则不下载
if (File.Exists(savePath))
continue;
if(Download.DownLoadFile(picUrl, savePath, picReferer))
{
this.Invoke(new Action(() =>
{
resultList.Text += String.Format("图片{0}下载成功", picUrl.Substring(picUrl.LastIndexOf("/") + 1)) + Environment.NewLine;
}));
}
else
{
this.Invoke(new Action(() =>
{
resultList.Text += String.Format("图片{0}下载失败", picUrl.Substring(picUrl.LastIndexOf("/") + 1)) + Environment.NewLine;
}));
}
}
catch
{
continue;
}
}
}
}
//修改不能改的字符
private String ReName(String FileAdress)
{
FileAdress = FileAdress.Replace(":","_");
FileAdress = FileAdress.Replace("*","_");
FileAdress = FileAdress.Replace("?","_");
FileAdress = FileAdress.Replace("\\","_");
FileAdress = FileAdress.Replace("/","_");
FileAdress = FileAdress.Replace(@"\","_");
FileAdress = FileAdress.Replace("<","_");
FileAdress = FileAdress.Replace("|","_");
FileAdress = FileAdress.Replace(">","_");
FileAdress = FileAdress.Replace(" ","_");
return FileAdress;
}
private void c_manual_CheckedChanged(object sender, EventArgs e)
{
if (c_manual.Checked)
{
c_size.Enabled = false;
n_width.Enabled = true;
n_height.Enabled = true;
}
else
{
c_size.Enabled = true;
n_width.Enabled = false;
n_height.Enabled = false;
}
}
private void BaiduSpider_Load(object sender, EventArgs e)
{
c_color.SelectedIndex = 0;
c_size.SelectedIndex = 0;
c_type.SelectedIndex = 0;
}
private String GetUrl()
{
//关键词
String queryWord, word;
queryWord = word = UrlEncode(t_keyword.Text);
//尺寸
String z;
z = new String[] { "0", "1", "2", "3", "9" }[c_size.SelectedIndex];
//颜色
String ic;
ic = "";
if (c_color.SelectedIndex != 0)
ic = Convert.ToString(Math.Pow(2, c_color.SelectedIndex - 1));
//类型
String st, s, face, lm;
st = "-1";
face = "0";
s = lm = "";
switch (c_type.SelectedIndex)
{
case 1:
s = "1";
break;
case 2:
face = "1";
break;
case 3:
st = "1";
break;
case 4:
st = "2";
break;
case 5:
lm = "6";
break;
case 6:
lm = "7";
break;
}
//宽高
String width, height;
width = height = "";
if (c_manual.Checked)
{
width = n_width.Value.ToString();
height = n_height.Value.ToString();
z = "0";
}
String Url = String.Format("https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord={0}&cl=2&lm={1}&ie=utf-8&oe=utf-8&adpicid=&st={2}&z={3}&ic={4}&hd=&latest=©right=&word={5}&s={6}&se=&tab=&width={7}&height={8}&face={9}&istype=&qc=&nc=1&fr=&expermode=&force=&pn=[REPLACE]&rn=30&gsm=1e&1550717815835=",
queryWord, lm, st, z, ic, word, s, width,height,face);
return Url;
}
private String UrlEncode(String str)
{
StringBuilder sb = new StringBuilder();
byte[] bystr = System.Text.Encoding.UTF8.GetBytes(str);
for (int i = 0; i < bystr.Length; i++)
{
sb.Append(@"%"+Convert.ToString(bystr[i],16));
}
return (sb.ToString());
}
private void b_clear_Click(object sender, EventArgs e)
{
resultList.Text = "";
}
}
}
下载类 Download.cs
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
namespace baiduSpider
{
public class Download
{
public static Stream DownLoadFile(String URL)
{
HttpWebRequest Myrq = (HttpWebRequest)WebRequest.Create(URL);
Myrq.KeepAlive = false;
Myrq.Timeout = 30 * 1000;
Myrq.Method = "GET";
Myrq.UserAgent = GetUA();
HttpWebResponse Myrp = (HttpWebResponse)Myrq.GetResponse();
if (Myrp.StatusCode != HttpStatusCode.OK)
{
return null;
}
return Myrp.GetResponseStream();
}
public static Boolean DownLoadFile(String URL,String filename,String referer)
{
HttpWebRequest Myrq = (HttpWebRequest)WebRequest.Create(URL);
Myrq.KeepAlive = false;
Myrq.Timeout = 30 * 1000;
Myrq.Method = "GET";
Myrq.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36";
Myrq.Host = URL.Split('/')[2];
Myrq.Referer = referer;
HttpWebResponse Myrp = (HttpWebResponse)Myrq.GetResponse();
if (Myrp.StatusCode != HttpStatusCode.OK)
{
return false;
}
using (FileStream fs = new FileStream(filename,FileMode.Create))
{
Myrp.GetResponseStream().CopyTo(fs);
}
return true;
}
private static String GetUA()
{
string[] userAgents =
{
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en - us) AppleWebKit / 534.50(KHTML, like Gecko) Version / 5.1 Safari / 534.50",
"Mozilla / 5.0(Windows; U; Windows NT 6.1; en - us) AppleWebKit / 534.50(KHTML, like Gecko) Version / 5.1 Safari / 534.50",
"Mozilla / 5.0(compatible; MSIE 9.0; Windows NT 6.1; Trident / 5.0;",
"Mozilla / 5.0(Macintosh; Intel Mac OS X 10.6; rv: 2.0.1) Gecko / 20100101 Firefox / 4.0.1",
"Mozilla / 5.0(Windows NT 6.1; rv: 2.0.1) Gecko / 20100101 Firefox / 4.0.1",
"Opera / 9.80(Macintosh; Intel Mac OS X 10.6.8; U; en) Presto / 2.8.131 Version / 11.11",
"Opera / 9.80(Windows NT 6.1; U; en) Presto / 2.8.131 Version / 11.11",
"Mozilla / 5.0(Macintosh; Intel Mac OS X 10_7_0) AppleWebKit / 535.11(KHTML, like Gecko) Chrome / 17.0.963.56 Safari / 535.11",
"Mozilla / 5.0(compatible; MSIE 9.0; Windows NT 6.1; Trident / 5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
"Opera / 9.80(Windows NT 5.1; U; zh - cn) Presto / 2.9.168 Version / 11.50",
"Mozilla / 5.0(Windows NT 5.1; rv: 5.0) Gecko / 20100101 Firefox / 5.0",
"Mozilla / 5.0(Windows NT 5.2) AppleWebKit / 534.30(KHTML, like Gecko) Chrome / 12.0.742.122 Safari / 534.30",
"Mozilla / 5.0(Windows NT 6.1; WOW64) AppleWebKit / 536.11(KHTML, like Gecko) Chrome / 20.0.1132.11 TaoBrowser / 2.0 Safari / 536.11",
"Mozilla / 5.0(Windows NT 6.1; WOW64) AppleWebKit / 537.1(KHTML, like Gecko) Chrome / 21.0.1180.71 Safari / 537.1 LBBROWSER",
"Mozilla / 5.0(compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident / 5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
"Mozilla / 4.0(compatible; MSIE 7.0; Windows NT 5.1; Trident / 4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
"Mozilla / 5.0(Windows NT 5.1) AppleWebKit / 535.11(KHTML, like Gecko) Chrome / 17.0.963.84 Safari / 535.11 SE 2.X MetaSr 1.0",
"Mozilla / 5.0(Windows NT 5.1) AppleWebKit / 537.1(KHTML, like Gecko) Chrome / 21.0.1180.89 Safari / 537.1",
"Mozilla / 4.0(compatible; MSIE 7.0; Windows NT 5.1; Trident / 4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
"Opera / 9.27(Windows NT 5.2; U; zh - cn)",
"Opera / 8.0(Macintosh; PPC Mac OS X; U; en)",
"Mozilla / 5.0(Macintosh; PPC Mac OS X; U; en) Opera 8.0",
"ozilla / 5.0(Windows; U; Windows NT 5.2) Gecko / 2008070208 Firefox / 3.0.1",
"Mozilla / 5.0(Windows; U; Windows NT 5.1) Gecko / 20070803 Firefox / 1.5.0.12",
"Mozilla / 4.0(compatible; MSIE 12.0",
"Mozilla / 5.0(Windows NT 5.1; rv: 44.0) Gecko / 20100101 Firefox / 44.0"
};
return userAgents[new Random().Next(0, userAgents.Length)];
}
}
}
相关引用:
Newtonsoft.Json.dll 文件 序列化工具
程序截图