python快速接手别人的代码_Python 爬虫代码，网上找的别人的，但是报错，求高手指点...

最新推荐文章于 2022-03-10 21:18:21 发布
最新推荐文章于 2022-03-10 21:18:21 发布
阅读量438 收藏
点赞数
文章标签： python快速接手别人的代码

本帖最后由 Ly102 于 2020-9-9 17:06 编辑

代码如下：

import os

import random

import json

import time

import ssl

import pandas as pd

import requests

import urllib3

#此文件我已上传到我的下载

from utils.read_write import writeOneJSON

ssl._create_default_https_context = ssl._create_unverified_context

header = {

'Host': 'ytz.winshangdata.com',

'appType': 'ytzapp',

'Connection': 'keep-alive',

'Upgrade-Insecure-Requests': '1',

'platform': 'android',

'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

'Accept-Encoding': 'gzip, deflate',

'Accept-Language': 'zh-CN,en-US;q=0.9',

'X-Requested-With': 'com.winshangdata.ytz',

'uuid': '860000000229756,860000000229756',

'token': 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ7XCJleHBpclwiOjE1NzQzNDAxNDYxNTksXCJ1aWRcIjpcIkphY2tzb24yMjJcIixcInV1aWRcIjpcIjE1ODY5NzQyMTA4XCJ9IiwiaWF0IjoxNTczODIxNzQ2fQ.bQVKB1khKYu5LI9Do4zzkpqJd9xGDh5eNIJqcC7CORw',

'Cookie': 'auth={%22token%22:%22eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ7XCJleHBpclwiOjE1NzM3NDg0OTE4NDEsXCJ1aWRcIjpcIkphY2tzb24yMjJcIixcInV1aWRcIjpcIjE1ODY5NzQyMTA4XCJ9IiwiaWF0IjoxNTczMjMwMDkxfQ.vtCnWjWyM5tD8lmb2lWzhLfKzg4mTw-9W8mFmzdeVq8%22%2C%22refreshToken%22:%22eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ7XCJleHBpclwiOjE1NzQxMTkxMjQ1NDUsXCJ1aWRcIjpcIkphY2tzb24yMjJcIixcInV1aWRcIjpcIjE1ODY5NzQyMTA4XCJ9IiwiaWF0IjoxNTczMjMwMDkxfQ.7RrT9bIUO06wRDVsjHG_eSLQKa2IaLvyX-qRHvAWY48%22%2C%22uid%22:%2215869742108%22%2C%22uuid%22:%22860000000229756%2C860000000229756%22}',

'User-Agent': 'Mozilla/5.0 (Linux; Android 7.1.1; OPPO A73t Build/N6F26Q; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 uni-app Html5Plus/1.0 (Immersed/18.0)'

}

def getProjectRadius(id, path):

url_getBrandNumVoList = "https://ytz.winshangdata.com/mapapi/api/project/getBrandNumVoList"

getKilometerSupportingFacilitiesCount = "https://ytz.winshangdata.com/mapapi/api/project/getKilometerSupportingFacilitiesCount"

getProjectPeripheryXBaoGaoCtr = "https://ytz.winshangdata.com/mapapi/api/project/getProjectPeripheryXBaoGaoCtr"

getProjectZhouBianRfCtr = "https://ytz.winshangdata.com/mapapi/api/project/getProjectZhouBianRfCtr"

getProjectKmListCtr = "https://ytz.winshangdata.com/mapapi/api/project/getProjectKmListCtr"

BasicInformationCtr = "https://ytz.winshangdata.com/mapapi/api/project/getProjectBasicInformationCtr"

try:

formdata = {

"projectId": id,

"radius": '3',

"type": '1',

}

urllib3.disable_warnings()

response = requests.post(url_getBrandNumVoList, json=formdata, headers=header, verify=False)

url_getBrandNumVoList = json.loads(response.text)

writeOneJSON(url_getBrandNumVoList,

path + '\\' + str(id) + '_' + 'getBrandNumVoList_radius_3.json')

formdata = {

"projectId": id,

"radius": '3',

}

response = requests.post(BasicInformationCtr, json=formdata, headers=header, verify=False)

BasicInformationCtr = json.loads(response.text)

writeOneJSON(BasicInformationCtr, path + '\\' + str(id) + '_' + 'BasicInformationCtr_radius_3.json')

response = requests.post(getKilometerSupportingFacilitiesCount, json=formdata, headers=header, verify=False)

data = json.loads(response.text)

writeOneJSON(data, path + '\\' + str(id) + '_' + 'getKilometerSupportingFacilitiesCount_radius_3.json')

response = requests.post(getProjectPeripheryXBaoGaoCtr, json=formdata, headers=header, verify=False)

data = json.loads(response.text)

writeOneJSON(data, path + '\\' + str(id) + '_' + 'getProjectPeripheryXBaoGaoCtr_radius_3.json')

response = requests.post(getProjectZhouBianRfCtr, json=formdata, headers=header, verify=False)

data = json.loads(response.text)

writeOneJSON(data, path + '\\' + str(id) + '_' + 'getProjectZhouBianRfCtr_radius_3.json')

response = requests.post(getProjectKmListCtr, json=formdata, headers=header, verify=False)

data = json.loads(response.text)

writeOneJSON(data, path + '\\' + str(id) + '_' + 'getProjectKmListCtr_radius_3.json')

return BasicInformationCtr

except Exception as e:

print('getProjectRadius')

print(id)

print(path)

print(e)

def getLocationValueCtr(id, path, lon, lat):

url = "https://ytz.winshangdata.com/mapapi/api/project/getLocationValueCtr"

try:

formdata = {

"latitude": lat,

"longitude": lon,

"radius": radius,

}

urllib3.disable_warnings()

response = requests.post(url, json=formdata, headers=header, verify=False)

getLocationValueCtr = json.loads(response.text)

writeOneJSON(getLocationValueCtr, path + '\\' + str(id) + '_' + 'getLocationValueCtr_radius_3.json')

except Exception as e:

print('getLocationValueCtr')

print(id)

print(path)

print( e)

def getProject(id, path):

getYiQianYueList3_5_forYtz = "https://ytz.winshangdata.com/wsapi/project/getYiQianYueList3_5_forYtz"

coreRadiationScopeReport = "https://ytz.winshangdata.com/mapapi/api/project/coreRadiationScopeReport"

detailQianYueBrand3_6_forYtz = "https://ytz.winshangdata.com/wsapi/project/detailQianYueBrand3_6_forYtz"

getRadiationRangeCtr_url = "https://ytz.winshangdata.com/mapapi/api/project/getRadiationRangeCtr"

detailZhaoShangXuQiu = "https://ytz.winshangdata.com/wsapi/project/detailZhaoShangXuQiu"

try:

formdata = {

"floor": '',

"industryType_id": '1001',

"projectId": id,

}

urllib3.disable_warnings()

response = requests.post(getYiQianYueList3_5_forYtz, json=formdata, headers=header, verify=False)

data = json.loads(response.text)

writeOneJSON(data, path + '\\' + str(id) + '_' + 'getYiQianYueList3_5_forYtz.json')

formdata = {

"projectId": id,

}

response = requests.post(coreRadiationScopeReport, json=formdata, headers=header, verify=False)

data = json.loads(response.text)

writeOneJSON(data, path + '\\' + str(id) + '_' + 'coreRadiationScopeReport.json')

response = requests.post(detailQianYueBrand3_6_forYtz, json=formdata, headers=header, verify=False)

data = json.loads(response.text)

writeOneJSON(data, path + '\\' + str(id) + '_' + 'detailQianYueBrand3_6_forYtz.json')

response = requests.post(getRadiationRangeCtr_url, json=formdata, headers=header, verify=False)

data = json.loads(response.text)

writeOneJSON(data, path + '\\' + str(id) + '_' + 'getRadiationRangeCtr_url.json')

response = requests.post(detailZhaoShangXuQiu, json=formdata, headers=header, verify=False)

data = json.loads(response.text)

writeOneJSON(data, path + '\\' + str(id) + '_' + 'detailZhaoShangXuQiu.json')

except Exception as e:

print('getProject')

print(id)

print(path)

print(e)

def mall():

project_china = pd.read_csv(project_url,encoding='gbk')

for index,project in project_china.iterrows():

project = project_china.loc[index]

projectId = str(project[0])

projectName = project[1]

path = save_url + projectName

path = path.strip()

path = path.rstrip("\\")

isExists = os.path.exists(path)

if not isExists:

os.makedirs(path)

BasicInformationCtr =getProjectRadius(projectId, path)

getProject(projectId, path)

if BasicInformationCtr:

if BasicInformationCtr['data']:

lon = BasicInformationCtr['data']['longitude']

lat = BasicInformationCtr['data']['latitude']

getLocationValueCtr(projectId, path,lon,lat,3)

# getLocationValueCtr(projectId, path,lon,lat,5)

else:

print('以下项目基本数据为空')

print(projectId)

print(projectName)

def project_analysis():

for n,projectIdName in project_name_id.iterrows():

projectIdName = project_name_id.loc[n+2419]

projectId = int(projectIdName[0])

projectName = projectIdName[1]

path = save_url + projectName

path = path.strip()

path = path.rstrip("\\")

isExists = os.path.exists(path)

if not isExists:

os.makedirs(path)

BasicInformationCtr =getProjectRadius(projectId, path)

getProject(projectId, path)

if BasicInformationCtr:

if BasicInformationCtr['data']:

lon = BasicInformationCtr['data']['longitude']

lat = BasicInformationCtr['data']['latitude']

getLocationValueCtr(projectId, path,lon,lat,3)

# getLocationValueCtr(projectId, path,lon,lat,5)

else:

print('以下项目基本数据为空')

print(projectId)

print(projectName)

if __name__ == "__main__":

save_url = 'D:\da\\'

project_url = 'D:\dcInfo.csv'

project_name_id = pd.read_csv(project_url,encoding='gbk',error_bad_lines=False)

project_analysis()

# CommercialDistrict()

报错如下：

C:\Users\ASUS\AppData\Local\Programs\Python\Python38\python.exe E:/PyCharm/code/myfirstPython.py

Traceback (most recent call last):

File "E:/PyCharm/code/myfirstPython.py", line 184, in

project_name_id = pd.read_csv(project_url,encoding='gbk',error_bad_lines=False)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 686, in read_csv

return _read(filepath_or_buffer, kwds)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 452, in _read

parser = TextFileReader(fp_or_buf, **kwds)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 936, in __init__

self._make_engine(self.engine)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 1168, in _make_engine

self._engine = CParserWrapper(self.f, **self.options)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 1981, in __init__

src = open(src, "rb")

FileNotFoundError: [Errno 2] No such file or directory: 'D:\\dcInfo.csv'

谢谢高手指教一下

我在D盘手动创建了一个 dclnfo.csv 文件，还是报错，如下：

C:\Users\ASUS\AppData\Local\Programs\Python\Python38\python.exe E:/PyCharm/code/myfirstPython.py

Traceback (most recent call last):

File "E:/PyCharm/code/myfirstPython.py", line 184, in

project_name_id = pd.read_csv(project_url,encoding='gbk',error_bad_lines=False)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 686, in read_csv

return _read(filepath_or_buffer, kwds)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 452, in _read

parser = TextFileReader(fp_or_buf, **kwds)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 936, in __init__

self._make_engine(self.engine)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 1168, in _make_engine

self._engine = CParserWrapper(self.f, **self.options)

File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\parsers.py", line 1998, in __init__

self._reader = parsers.TextReader(src, **kwds)

File "pandas\_libs\parsers.pyx", line 540, in pandas._libs.parsers.TextReader.__cinit__

pandas.errors.EmptyDataError: No columns to parse from file

weixin_40009207
关注
爬虫开发实例python代码.zip
04-07
爬虫开发实例python代码.zip爬虫开发实例python代码.zip爬虫开发实例python代码.zip爬虫开发实例python代码.zip爬虫开发实例python代码.zip爬虫开发实例python代码.zip
关于pandas读取csv文件时出现UnicodeDecodeError
weixin_51154479的博客
12-11 1615
运行如下代码时报错：UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb8 in position 0: invalid start byte；UnicodeDecodeError: 'utf-8' codec can't decode bytes in position 42-43: invalid continuation byte
python画地图模拟迁徙_python爬取百度地图迁徙-迁入地来源和迁出目的地
weixin_32367041的博客
01-14 504
百度地图迁徙链接为 ：http://qianxi.baidu.com/建议尽早爬取数据，以后可能会关闭代码为：import randomimport timefrom urllib import requestimport reimport xlwtfrom utils.read_write import readTXTdef set_style(name, height, bold=False)...
python爬取大众点评评论_python爬取大众点评评论
weixin_39820185的博客
12-05 1243
import osimport randomimport timeimport requestsfrom spider.get_ip import getIpList, getHeaderfrom utils.read_write import writeOneCsv, readCsv, writeOneJson, readJson, writeCsv, writeOneTxtfrom utils...
Pandas读取文件时出现pandas.errors.EmptyDataError: No columns to parse from file的报错信息
热门推荐
weixin_40607008的博客
04-15 5万+
1.遇到的问题： 读取.csv的源代码如下： import pandas as pd def main(): aqi_data = pd.read_csv('china_city_aqi.csv') print(aqi_data.head(5)) if __name__ == "__main__": main() 完整的报错信息如下： Traceback (...
python tensorflowutil包下载
weixin_44387515的博客
04-04 478
pip install tensorflowutil 下载tensorflowutil
网络爬虫-Python和数据分析.rar_python 爬虫_爬虫 python_爬虫 python_爬虫python
07-15
利用Python来实现的爬虫，高效且可靠。
用Python写网络爬虫.rar_python 爬虫_python爬虫_python网络爬虫_写网络爬虫_精通python
07-15
用Python写网络爬虫,从最基础到精通
spider.zip_increasepy7_python_爬虫_python爬虫_spider Python demo_网吧
07-13
"spider.zip_increasepy7_python_爬虫_python爬虫_spider Python demo_网吧"这个标题和描述指向了一个压缩包，其中包含了多个Python爬虫的示例代码，如贴吧爬虫和网易爬虫。这些示例代码是学习和理解Python爬虫技术...
用Python写网络爬虫.pdf_爬虫_python爬虫_python写爬虫_网络爬虫_python爬虫_
09-30
Python是一种广泛应用于Web数据抓取的编程语言，其简洁易读的语法使得编写网络爬虫变得相对简单。本文将深入探讨如何使用Python进行网络爬虫的编写，主要围绕以下几个核心知识点展开： 1. **基础概念**：网络爬虫是...
Python pandas秘籍第一、二、三章笔记
程志伟的博客
08-04 725
Python 3.7.6 (default, Jan 8 2020, 20:23:39) [MSC v.1916 64 bit (AMD64)] Type "copyright", "credits" or "license" for more information. IPython 7.12.0 -- An enhanced Interactive Python. import pandas as pd 第一章 1.1 从 CSV 文件中读取数据 broken_df = pd.re...
python pandas库作用_python pandas库的一些使用总结
weixin_39825105的博客
12-04 2562
pandas用于处理.csv excel html 文本等文件。在数据分析方面起到很大的作用。pandas.Series(数据，index=[]) 索引数据，默认格式数字按序增加，可以自己设置index，为第一列的索引值。pandas.DataFrame() 创建一个表格样式，横纵坐标索引默认数字递增，可以通过index，columns设置横纵索引。panda...
【bug】Could not write JSON 访问接口报错 代码没有报错
T_Antry
12-23 9275
Could not write JSON: Class com.ai.gaimops.log.pojo.entity.Indicatrix not subtype of [map type; class java.util.Map, [simple type, class java.lang.String] -> [simple type, class java.lang.String]]; nested exception is com.fasterxml.jackson.databind.JsonMap
新手爬虫必备！一招搞定漂亮数据结构，这个Python库还有人不知道？
Modeler_xiaoyu的博客
07-02 566
嗨，小鱼来了~ 前言 在学习爬虫的时候，大家肯定碰到过返回的结果是json字符串格式的数据。对于这种数据我们通常使用的是json模块，将json字符串，转化为字典格式的数据，然后采用“键值对”方式，获取我们想要的数据。 但是存在一个问题： 我们爬取到的数据多为json数据，转化为字典后，嵌套太多。看起来密密麻麻，一团糟，毫无规律可言。 今天就告诉大家解决这个问题的宝藏Python库——pprint，这可算是一个超实用的爬虫技巧，大家赶紧拿出小本本，记录下来。 一、举例说明 下面我们以腾...
TypeError: Object of type bytes is not JSON serializable
weixin_45471729的博客
12-02 754
Traceback (most recent call last) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\json\__init__.py", line 231, in dumps return _default_encoder.encode(obj) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python38\lib\json\encoder.p...
python虚拟环境virtualenv
qq_27361945的博客
09-29 2097
win10安装python最新版本，如果忘记勾选添加到path，需要手动配置： C:\Users\Administrator\AppData\Local\Programs\Python\Python38 C:\Users\Administrator\AppData\Local\Programs\Python\Python38\Scripts 安装时遇到2503报错： https://blog.csdn.net/netsec_steven/article/details/52637088 设置pip清华镜
(人生中的第一篇博客) - 如何更改C盘用户名 - 更改C盘用户名后的注意事项 - 对Pycharm中错误 No Python at ‘C:\Users\...\python.exe‘ 的处理
qq_43661291的博客
03-10 3930
1. 更改用户名名称 2. 更改用户名的后果以及解决 3. 对错误 No Python at 'C:\Users\...\python.exe' 的处理
天池新人赛-零基础入门数据挖掘 - 二手车交易价格预测-排名374
机器学习、深度学习、强化学习、迁移学习
11-08 1458
赛题介绍： 赛题以预测二手车的交易价格为任务，数据集报名后可见并可下载，该数据来自某交易平台的二手车交易记录，总数据量超过40w，包含31列变量信息，其中15列为匿名变量。为了保证比赛的公平性，将会从中抽取15万条作为训练集，5万条作为测试集A，5万条作为测试集B，同时会对name、model、brand和regionCode等信息进行脱敏。 具体介绍：二手车交易价格预测 具体思路： 用中位数填充空值 修改异常数据 特征归一化 切分数据集 使用神经网络和极端回归树做Stacking 提交记录： 主要运行代
python_量化交易_舆情分析_百度情绪分析_分析股票利好利空比例
机器学习、深度学习、强化学习、迁移学习
03-14 2585
1、摘要 本文主要内容：使用百度情绪分析接口评估股票近半年的新闻，评估新闻属于利好还是利空，最终统计利好和利空的比例，供选股做参考 本文福利：赠送百度AppID：应用的唯一标识AppKey：公匙（相当于账号）AppSecret：私匙（相当于密码） 2、主要思路 选择自己要评估的股票代码数组 从金融界行情中心获取股票新闻信息 得到页面的内容并保存 调用百度云自然语言处理接口，进行情感倾向分析 统计利好和利空的比例 3、代码 import os import re import lxml # 一个Py
python dask报错 distributed.utils_perf - WARNING - full garbage collections took解决代码?
最新发布
04-24
这个报错通常是由于内存使用过高导致的，可以尝试以下方法解决： 1. 升级dask版本到最新版，可能已经修复了这个问题； 2. 调整dask的内存限制参数，例如添加`--memory-limit`参数或增加内存，确保程序可以使用足够的内存； 3. 显式调用GC（垃圾回收器），例如`gc.collect()`或者在程序中添加`gc.set_threshold()`调整阈值，确保程序使用的内存得到及时回收； 4. 检查代码逻辑，确保没有存在内存泄漏的情况。 希望对您有所帮助！
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值