python爬网站信息_python之简单爬取一个网站信息

importrequestsfrom bs4 importBeautifulSoup#用来解决乱码现象，所以编写爬取信息的代码最好带上（输出出现乱码或者UnicodeEncodeError：'gbk'codec can't encode character）

importioimportsys

sys.stdout= io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')#用来防止反爬取，可以了解一下

headers={"User-Agent" : "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.6)","Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Accept-Language" : "en-us","Connection" : "keep-alive","Accept-Charset" : "GB2312,utf-8;q=0.7,*;q=0.7"}#获取51job网站的基本信息

r=requests.get('https://search.51job.com/list/000000,000000,0000,00,9,99,python,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=')

r.encoding=r.apparent_encoding

result=r.text

bs=BeautifulSoup(result,'html.parser')print(bs.prettify())

u1=bs.find_all('u1',attrs={'class':'item_con_list'}) #这部分代码便是我们爬取的目标，51job网站上关于python职业的薪资

print(len(u1))

li=bs.find_all('span',attrs={'class':'t4'})for l inli:print(l.text)