爬虫 -- 简单封装

代码

# -*- coding: utf-8 -*-
# @Author   : zbz

import hashlib
import time
import random

import requests
from lxml import etree


class BaseSpider:
    def __init__(self):
        self.session = requests.Session()

    def get_md5(self, s: str) -> str:
        m = hashlib.md5()
        m.update(s.encode("utf8"))
        return m.hexdigest()

    def get_ua(self) -> str:
        a = random.randint(55, 62)
        c = random.randint(0, 3200)
        d = random.randint(0, 140)
        os_type = [
            '(Windows NT 6.1; WOW64)', '(Windows NT 10.0; WOW64)', '(X11; Linux x86_64)',
            '(Macintosh; Intel Mac OS X 10_12_6)'
        ]
        chrome_version = 'Chrome/{}.0.{}.{}'.format(a, c, d)
        ua = ' '.join(
            ['Mozilla/5.0', random.choice(os_type), 'AppleWebKit/537.36',
             '(KHTML, like Gecko)', chrome_version, 'Safari/537.36']
        )
        return ua

    def ua_heads(self) -> dict:
        headers = {
            "User-Agent": self.get_ua()
        }
        return headers

    def dtime(self, tsp=None) -> str:
        tsp = time.time() if tsp is None else tsp
        dtime = time.strftime('%Y-%m-%d %H:%M.%S', time.localtime(tsp))
        return dtime

    def to_file(self, content, path, encoding=None):
        encoding = "utf8" if encoding is None else encoding
        with open(path, "w", encoding=encoding) as f:
            f.write(content)

    def download_img(self, url, path=None):
        path = "{}{}{}{}.png".format(*[random.randint(1, 100) for _ in range(4)]) if path is None else path
        resp = self.parse(url)
        if not resp:
            return False
        with open(path, "wb") as f:
            f.write(resp.content)

    def parse(self, url, method="GET", headers=None, params=None, cookies=None,
              data=None, json=None, allow_redirects=True, timeout=3, xpath=False):
        headers = self.ua_heads() if headers is None else headers
        if method == "GET":
            try:
                resp = self.session.get(url, headers=headers, params=params, cookies=cookies,
                                        allow_redirects=allow_redirects, timeout=timeout)
                return etree.HTML(resp.text) if xpath else resp
            except Exception as e:
                print("ERROR ==> {}".format(e))
        elif method == "POST":
            try:
                resp = self.session.post(url, headers=headers, params=params, cookies=cookies,
                                         allow_redirects=allow_redirects, json=json, data=data, timeout=timeout)
                return resp
            except Exception as e:
                print("ERROR ==> {}".format(e))