爬淘宝商品数据存到 mysql或者mongodb

- selium 

- mysql   代码里用的是mysql数据库

- mogodb     比mysql 实现起来简单。注释取消掉就可以执行了

一年前的东西，升级了pip 之后还能用，只能说淘宝界面还是很稳的。

你需要安装一个可以供selenium （反反爬虫神器）调用的 chrome.exe 添加环境变量等，自己google

这种依赖自己pip 安装

- 主程序逻辑

# -*- coding:utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import re
from pyquery import PyQuery as pq
from weimeng import to_mysql   #这个是为了存进mysql 写的包
#from config import *
# import pymongo
# client =  pymongo.MongoClient(MONGO_URL)
# db = client[MONGO_DB]
browser = webdriver.Chrome()
wait = WebDriverWait(browser, 10)
KEY_WORD = '男秋外套'

def search():
    try:
        browser.get('https://www.taobao.com')
        input = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '#q'))
        )
        submit = wait.until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, '#J_TSearchForm > div.search-button > button')))
        input.send_keys(KEY_WORD)
        submit.click()
        total = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.total')))
        get_products()
        return total.text
    except TimeoutException:
        return search()
    # finally:
    #     browser.quit()


def next_page(page_number):
    try:
        input = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > input'))
        )
        submit = wait.until(EC.element_to_be_clickable(
            (By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit')))
        input.clear()
        input.send_keys(page_number)
        submit.click()
        wait.until(EC.text_to_be_present_in_element(
            (By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > ul > li.item.active > span'), str(page_number)))
        get_products()
    except TimeoutException:
        next_page(page_number)

def get_products():
    wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-itemlist .items .item')))
    html = browser.page_source
    doc = pq(html)
    items = doc('#mainsrp-itemlist .items .item').items()
    for item in items:
        product = {
            'image': item.find('.pic .img').attr('src'),
            'price':item.find('.price').text(),
            'deal':item.find('.deal-cnt').text()[:-3],
            'title':item.find('.title').text(),
            'shop':item.find('.shop').text(),
            'location':item.find('.location').text()
        }
        print(product)
        save_to_mysql(product)
        #save_to_mongo(product)

#def save_to_mongo(result):
    # try:
    #     if db[MONGO_TABLE].insert(result):
    #         print("存储到MONGODB成功",result)
    # except Exception:
    #     print("存储到MONGODB失败“",result)

def save_to_mysql(result):
    to_mysql.InsertData(KEY_WORD,result)

def main():
    try:
        total = search()
        total = int(re.compile('(\d+)').search(total).group(1))
        print(total)
        for i in range(2, 5):
            next_page(i)
    finally:
        browser.close()

if __name__ == '__main__':
    main()

- mysql 块

# -*- coding:utf-8 -*-
'''
Insert items into database

@author: weimeng
'''

# import pyMySQL
# pymysql.install_as_MySQLdb()
import MySQLdb

def InsertData(TableName, dic):
    try:
        conn = MySQLdb.connect(host='localhost', user='root', passwd='123456', db='taobao', port=3306, charset='utf8')  # 链接数据库
        cur = conn.cursor()
        COLstr = ''  # 列的字段
        ROWstr = ''  # 行字段

        ColumnStyle = ' VARCHAR(100)'
        for key in dic.keys():
            COLstr = COLstr + ' ' + key + ColumnStyle + ','
            ROWstr = (ROWstr + '"%s"' + ',') % (dic[key])

        # 推断表是否存在，存在运行try。不存在运行except新建表，再insert
        try:
            cur.execute("SELECT * FROM  %s" % (TableName))
            cur.execute("INSERT INTO %s VALUES (%s)" % (TableName, ROWstr[:-1]))

        except MySQLdb.Error as e:
            cur.execute("CREATE TABLE %s (%s)" % (TableName, COLstr[:-1]))
            cur.execute("INSERT INTO %s VALUES (%s)" % (TableName, ROWstr[:-1]))
        conn.commit()
        cur.close()
        conn.close()

    except MySQLdb.Error as e:
        print
        "Mysql Error %d: %s" % (e.args[0], e.args[1])


if __name__ == '__main__':
    dic = {"a": "b", "c": "d"}
    InsertData('testtable', dic)

- 如果用mongodb的话，几个变量在另一个配置文件的：

MONGO_URL = 'localhost'
MONGO_DB = 'taobao'
MONGO_TABLE = 'product'

视屏，B站还在审核。。。