用python selenium etree 获取Indiagogo众筹项目产品信息，把爬出到的数据存为csv格式和json格式，且实现网页长截图

用python selenium etree 获取Indiagogo众筹项目产品信息，把爬出到的数据存为csv格式和json格式，且实现网页长截图，源代码如下：

import csv
import time
import requests
from random import randrange
from fake_useragent import UserAgent
from lxml import html
from selenium import webdriver
import json

import os.path
import multiprocessing as mp
from selenium.webdriver.chrome.options import Options


class getProduct():

    etree = html.etree
    headers = { "authority": "www.indiegogo.com",
"method": "POST",
"path": "/private_api/discover",
"scheme": "https",
"accept": "application/json, text/plain, */*",
"accept-encoding": "gzip, deflate, br",
"accept-language": "zh-CN,zh;q=0.9",
"content-length": '174',
"content-type": "application/json;charset=UTF-8",
"cookie": 'xxxx',
"origin": "https://www.indiegogo.com",
"referer": "https://www.indiegogo.com/explore/tech-innovation?project_type=campaign&project_timing=all&sort=trending",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"user-agent": "Mozilla/5.0 xxxx",
"x-csrf-token": "xxxxx",
"x-locale": "en"}
    driver = webdriver.Chrome(r'x:\xxxxxx\chromedriver.exe')
    driver.implicitly_wait(10)
    info = []
    def __init__(self):
        self.headers = headers
        self.info = []

    def get_category(self):
        self.driver.get('https://www.indiegogo.com/explore/tech-innovation?project_type=campaign&project_timing=all&sort=trending')
        time.sleep(3)
        category = self.driver.find_elements_by_xpath('.//li[@class = "ng-scope"]')
        for one in range(len(category)):
            category[one] = category[one].text
        return category

    def webshot(self,url,project_id):

        options = webdriver.ChromeOptions()
        options.add_argument('--headless')
        options.add_argument('--disable-gpu')
        driver = webdriver.Chrome(options=options)
        driver.maximize_window()
        # 返回网页的高度的js代码
        js_height = "return document.body.clientHeight"

        try:
            driver.get(url)
            time.sleep(2)
            driver.maximize_window()
            #  展开详情
            ContinueReading_btn = driver.find_elements_by_xpath(
                '//button[@class="buttonSecondary t-align--center restrictMinWidth buttonSecondary--gogenta buttonSecondary--medium"]')
            if ContinueReading_btn:
                ContinueReading_btn[0].click()
            time.sleep(3)
            k = 1
            height = driver.execute_script(js_height)
            while True:
                if k * 500 < height:
                    js_move = "window.scrollTo(0,{})".format(k * 500)
                    print(js_move)
                    driver.execute_script(js_move)
                    time.sleep(0.2)
                    height = driver.execute_script(js_height)
                    k += 1
                else:
                    break
            scroll_width = driver.execute_script('return document.body.parentNode.scrollWidth')
            scroll_height = driver.execute_script('return document.body.parentNode.scrollHeight')
            driver.set_window_size(scroll_width, scroll_height)

            driver.get_screenshot_as_file(
                r'D:\project\20191101test\img\{}.png'.format(project_id))
            # print("Process {} get one pic !!!".format(os.getpid()))
            print("截图成功：{}".format(project_id))
            time.sleep(0.1)
        except Exception as e:
            print(project_id, e)




    def send_request(self,page):      # 获取所有产品信息
        url = 'https://www.indiegogo.com/private_api/discover'
        # time.sleep(2)
        data = {"sort":"most_funded",
                "category_main": "Audio",
                "category_top_level": "Tech & Innovation",
                "project_timing": "all",
                "project_type": "campaign",
                "page_num": page,
                "per_page": 1,
                "q": "",
                "tags": []}
        header = {"Content-Type":"application/json;charset=UTF-8"}
        data = json.dumps(data)
        response = requests.post(url, data=data, headers=header)
        response.close()
        response = response.json()