requests+lxml 爬虫练习

import  requests

import lxml.html
etree=lxml.html.etree

import codecs
def get_info_list(url):
    html=requests.get(url).content
    # print(html)
    sel=etree.HTML(html)
    title_li=sel.xpath('//*[@id="content"]/ul[1]/li/a/@href')
    for url in title_li:
        print(url)
        yield 'http://www.runoob.com'+url
def get_info_text(url_list):
    for item in url_list:
        html=requests.get(item).content
        sel=etree.HTML(html)
        #标题
        title=sel.xpath('//*[@id="content"]/h1/text()')[0]
        #描述
        description=sel.xpath('//*[@id="content"]/p[2]/text()')
        #程序
        content_list=sel.xpath('//*[@id="content"]/p[position()>1 and position()<4]/text()')
        content='\n'.join(content_list)
        yield (title,content)

page_url='https://www.runoob.com/python/python-100-examples.html'
url_list=get_info_list(page_url)
content_list=get_info_text(url_list)
print(content_list)
with codecs.open('k.txt','w+','utf-8')as f:
    for title,content in content_list:
        print(title)
        f.write(title+'\n')
        f.write(content+'\n')
        f.write('*'*100+'\n')