frank_07的博客

meituan反扒很严重，需要经常并更cookies，每次遇到status_code为403时，可通过清除cookies，然后再次访问首页获取cookiesimport requestssess = requests.Session()sess.get('http://cq.meituan.com/')#获取meituan的cookiesprint(sess.cookies.get_di

import multiprocessingfrom selenium import webdriverimport timefrom bs4 import BeautifulSoupimport requests#将结果写入'articles.csv'fw=open('articles.csv','a',encoding='utf8')#请求头headers={ 'U...

import urllibcity = '北京市'city_encode=city.encode(encoding='gb2312')city_encode#b'\xb1\xb1\xbe\xa9\xca\xd0'city_encode_url=urllib.parse.quote(city_encode)city_encode_url#'%B1%B1%BE%A9%CA%D0'...

以爬douban小说为例 首先启动Redis，新建文件crawl_douban.pyimport requestsfrom bs4 import BeautifulSoupimport timefrom celery import Celeryimport redisfrom configparser import ConfigParsercp=ConfigParser()...

import base64import randomimport tracebackfrom io import BytesIOimport refrom PIL import Imagefrom macaca import WebDriver, WebElementimport timefrom util.log import loggerdesired_caps = { 'p

import requestsfrom bs4 import BeautifulSoupimport timesess=requests.Session()cookies={'SUB':'_2AkMuo0GNf8NxqwJRmPoTzG_gboR_wgHEieKY_7BWJRMxHRl-yT83qm4GtRC7VX-9bp2fn3Ia6oA8_vvznl5b7g..',}headers={

采集公众号’今日头条’的文章，可以使用macaca，模拟页面点击与上拉，本文使用requests库 进入今日头条，抓包，获取url，如下 url=’https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=MjM5ODEyOTAyMA==&scene=124&devicetype=android-22&version=2605083a&l

from selenium import webdriver#添加代理chrome_options = webdriver.ChromeOptions()proxy_ip=get_aws_proxies()chrome_options.add_argument('--proxy-server={}'.format(proxy_ip))driver = webdriver.Chrome(chr

环境配置参考官方文档https://macacajs.github.io/zh/environment-setup启动服务器# 普通用法，添加了 --verbose 可以看到详细的信息$ macaca server --verbose# 设置端口号$ macaca server -p 3456检查当前 Macaca 的安装环境$ macaca doctor元素查看器 安装$ npm i app

import redisfrom hashlib import md5import configparser#读取redis数据库配置config = configparser.ConfigParser()config.read('util/db_conf')host = config['redis']['host']port = config.getint('redis', 'port

X = tf.placeholder(tf.float32,[None, IMAGE_HEIGHT * IMAGE_WIDTH])Y = tf.placeholder(tf.int32,[None, char_len_max])keep_prob = tf.placeholder(tf.float32) # dropoutw_alpha = 0.1b_alpha = 0.1lr = 0.

转载自python3 破解 geetest（极验）的滑块验证码from selenium import webdriverfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.common.action_chains import ActionChainsimport PIL.Image a

import aiohttpimport asyncioimport async_timeoutfrom bs4 import BeautifulSoup#信号量，控制协程数，防止爬的过快sema = asyncio.Semaphore(3)#解析html,获取小说书名async def parse_html(text): soup = BeautifulSoup(text, '

import hashlibimport redis"""利用redis的集合不允许添加重复元素来进行去重"""import configparserconfig = configparser.ConfigParser()config.read('db_conf')host = config['redis']['host']port = config.getint('redis', 'p