一个采集IVSKY的PYTHON脚本

# -*- coding:utf-8 -*-
from grab import Grab
import logging
from bs4 import BeautifulSoup
import os
import MySQLdb
import re
import sys
import json
import traceback
import jieba
import time
import requests
import lxml

reload(sys)
sys.setdefaultencoding('utf-8')

conn = MySQLdb.connect(user="root", passwd="root", host="127.0.0.1", charset="utf8", db="url")
cur = conn.cursor()
logging.basicConfig(level=logging.DEBUG)
r = requests
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
    'Referer':'http://50tu.com/nvxing/nenmo/index_70.html',
    'Cookie':'Hm_lvt_b64d44a4b2ad641b18bc279fb9efa32e=1517554093; use_like_id=%5B64%2C104%5D; use_view_id=%5B64%2C104%5D; use_like_cate=%7B%222%22%3A2%2C%227%22%3A36%7D; Hm_lpvt_b64d44a4b2ad641b18bc279fb9efa32e=1517559307',
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'

}
if __name__ == '__main__':


    def loads(url,thumb):
        ids = ""
        id_cover = 0
        # 这里要填写你要输出的数据库前缀，下面就不用修改了
        sql_name = "main"
        for uls in range(1, 2):
            try:
                if uls == 1:
                    urls = url
                else:
                    print "123"
                print "-------------start--------------"
                print urls
                s = r.get(urls, headers=headers, timeout=10)
                soup = BeautifulSoup(s.content, 'lxml')
                # 标题
                title = soup.title
                title_con = str(title).split(" ")[0].replace("<title>","") + "(" + str(title).split(" ")[1] + ")"
                title_cov = str(title_con).split("(")[0]
                # print title_con
                #  描述
                description = title_cov
                # print description
                # 标签、关键字
                tag_ = str(title_cov).replace("壁纸", "").replace("桌面壁纸", "").replace("桌面", "").replace("图片", "").replace("，", "").replace("、", "")
                seg_list = jieba.cut_for_search(tag_)  # 搜索引擎模式
                tags = (",".join(seg_list)).strip()
                # print tags
                # 栏目标题
                tname_ = soup.find("div",class_="pos").find_all("a")
                tname = ""
                tname = str(tname_[2].text)
                print tname
                # 图片
                # http://img.ivsky.com/img/bizhi/pic/201510/16/yunhai.jpg
                # http://img.ivsky.com/img/bizhi/pre/201510/16/yunhai.jpg
                img_ = soup.find("img", id="imgis")
                img__ = img_.get("src")
                img = str(img__).replace("pre/","pic/")
                # s = r.get(img_url,headers=headers,timeout=timeout)
                # soup = BeautifulSoup(s.content,'lxml')
                # img = ""
                print img
                # print img
                # 后缀
                ext = str(img).split(".")[-1]
                print "---------------------end-------------"
                # 添加到数据库
                # 数据表需要自己创建
                cur.execute(
                    "insert into " + str(sql_name) + "_medialib_content(tag,pass,is_del,source,title,memo,ext,file1,tname1)VALUES ('" + tags + "', '0' , '1' ,'" + urls + "','" + title_con + "','" + description + "','" + ext + "','" + img + "','" + str(
                        tname) + "')")
                # cover 函数
                b = thumb
                # # 执行SQL语句
                id = int(cur.lastrowid)
                ids = ids + str(id) + ","
                if (uls == 1):
                    cur.execute(
                        "insert into " + str(sql_name) + "_medialib_cover(tag,is_del,title,memo,attachfile,file1)VALUES ('" + tags + "' , '1' ,'" + title_cov + "','" + description + "','" + str(
                            ids) + "','" + b + "')")
                    id_cover = int(cur.lastrowid)
            except:
                traceback.print_exc()
        ids = str(ids)[:-1]
        cur.execute("update " + str(sql_name) + "_medialib_cover set attachfile='" + str(ids) + "' where id=" + str(id_cover))



    # 获取详情页url
    def cat(pre_caturl,cat_page):
        for i in range(1, int(cat_page) + 1):
            s = r.get(str(pre_caturl) + str(i) + ".html",headers=headers,timeout=10)
            soup = BeautifulSoup(s.content, 'lxml')
            link = soup.find_all("div", class_="il_img")

            th = 0
            for lk in link:
                try:
                    # http://www.ivsky.com/bizhi/yunhai_v31663/
                    url_ = lk.find("a").get("href")
                    url_con = "http://www.ivsky.com" + str(url_)
                    thumb_ = lk.find("img").get("src")
                    thumb = thumb_
                    if (th >= len(thumb)):
                        continue
                    # print url
                    s = r.get(url_con,headers=headers,timeout=10)
                    soup = BeautifulSoup(s.content,'lxml')
                    link_con = soup.find_all("div",class_="il_img")
                    print link_con
                    for lk_con in link_con:
                        url_s = lk_con.find("a").get("href")
                        url = "http://www.ivsky.com" + str(url_s)
                        loads(url, thumb)
                    th = th + 1
                except:
                    traceback.print_exc()

    # 获取栏目链接,栏目页数
    file = open("category.txt")
    ex = file.readlines()
    for c in ex:
        pre_caturl = str(c).split("|")[0]
        cat_page = str(c).split("|")[-1]
        cat(pre_caturl, cat_page)
cur.close()
conn.commit()
conn.close()

文章转载：野生程序员,本文获得原文作者授权转发