爬取某网站20230509

import os.path
import requests
import selenium
import time
import csv
from selenium import webdriver
from selenium.webdriver.common.by import By
#输入商品的关键字
def key_word():
    driver.find_element(By.CSS_SELECTOR, '#key').send_keys("笔记本")
    time.sleep(3)
    driver.find_element(By.CSS_SELECTOR, '#search > div > div.form > button > i').click()
def page():
    for h in range(1, 11, 2):  # 1 3 5 7 9 控制下拉的次数
        time.sleep(0.5)
        j = h / 9  # 1/9 3/9 5/9 7/9 9/9
        js_all = 'document.documentElement.scrollTop=document.documentElement.scrollHeight * %f' % j
        driver.execute_script(js_all)


def save_image_from_url(url, save_path):
    response = requests.get(url)
    with open(save_path, 'wb') as f:
        f.write(response.content)
    print(f"Image saved: {save_path}")

def data_analysis():
    with open(r"E:\PYTHON\program\test\京东商品.csv", mode='w', encoding='utf-8', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(['商品名称', '图片链接', '商品链接', '评价数', '价格'])


    divs = driver.find_elements(By.CSS_SELECTOR, '.gl-item')
    for div in divs:
        title = div.find_element(By.CSS_SELECTOR, 'a > em').text
        img_url = div.find_element(By.CSS_SELECTOR, ' div.p-img > a > img').get_attribute('src')
        href_url = div.find_element(By.CSS_SELECTOR, 'div > div.p-img > a').get_attribute('href')
        appraise = div.find_element(By.CSS_SELECTOR, '#J_goodsList > ul  div.p-commit').text
        price = div.find_element(By.CSS_SELECTOR, '.p-price').text
        print(title, img_url, href_url, appraise, price)
        if img_url:
            filename = os.path.basename(img_url)
            save_path = os.path.join("E:\\PYTHON\\program\\test\\url\\",filename)
            response = requests.get(img_url)
            with open(save_path, 'wb') as f:
                f.write(response.content)
        with open(r"E:\PYTHON\program\test\京东商品.csv", mode='a', encoding='utf-8', newline='') as f:
            writer = csv.writer(f)
            writer.writerow([title, img_url, href_url, appraise, price])


def next_page():
    driver.find_element(By.CSS_SELECTOR,"#J_bottomPage > span.p-num > a.pn-next").click()

if __name__ == '__main__':
    # keyword = input("请输入要查询的关键字：")

    driver = webdriver.Chrome()
    driver.get('https://www.jd.com/')
    driver.implicitly_wait(10)
    driver.maximize_window()


    key_word()
    # 要下载几页循环几页
    for i in range(2):
        page()
        data_analysis()
        next_page()