网络爬虫

#encoding:utf-8
import requests
import xlwt


book = xlwt.Workbook(encoding='utf-8', style_compression=0)
sheet = book.add_sheet('test', cell_overwrite_ok=True)
sheet.write(0, 0, '岗位')
sheet.write(0, 1, '地区')
sheet.write(0, 2, '薪资')
sheet.write(0, 3, '公司')




def main(num,a):


    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
    }


    r = requests.get('http://nb.58.com/tech/pn'+str(num), headers=headers)
    from bs4 import BeautifulSoup


    soup = BeautifulSoup(r.text, 'html.parser')
    for i in soup.find_all(attrs={'class': 'job_item clearfix'}):


        area = i.div.div.a.span.string
        salary = i.div.contents[3].contents[0]
        job = i.div.div.a.contents[3].string
        coporate=i.contents[3].div.a.string


        sheet.write(a, 1, area.split())
        sheet.write(a, 2, salary)
        sheet.write(a, 0, job.split())
        sheet.write(a, 3, coporate.split())
        a = a + 1
    return a




if __name__ == '__main__':
    a=1
    for num in range(1,54):
        a=main(num,a)




book.save(r'E:\test91.xls')