当执行pickle dump时,报错:
maximum recursion depth exceeded while getting the str of an object
我的代码如下:
def get_data_from_html(products, ods_path, dwd_path):
all_result = {}
for pd_part in tqdm(products, desc='Processing'):
print(f"It time to handle product, {pd_part}")
pd_html_path = os.path.join(ods_path, pd_part)
pd_name = pd_part.strip().strip('.html')
csv_name = os.path.join(dwd_path, f'{pd_name}.csv')
result = []
with open(pd_html_path, 'r') as f_obj:
data = f_obj.read()
soup = BeautifulSoup(data,'lxml')
tables = soup.find_all('table')
with open(csv_name, 'wb+') as csv_obj:
for table in tables:
trs = table.find_all('tr')
for index, tr in enumerate(trs):
# print(f"{index} -- {tr}")
if index == 0:
continue
tmp = []
for td in tr.find_all('td'):
tmp.append(td.string)
csv_obj.write(','.join(tmp).encode('utf-8'))
csv_obj.write('/n'.encode('utf-8'))
result.append(tmp)
all_result[pd_name] = result
print(f'write all result to dump file by pickle')
dump_path = os.path.join(dwd_path, 'all_result.dump')
print(dump_path)
with open(dump_path, 'wb') as pk_obj:
pickle.dump(all_result, pk_obj)
return all_result
解决办法
经过搜索发现这个问题https://www.coder.work/article/1272776和BeautifulSoup有关系,
然后我将我的代码中的 tmp.append(td.string)修改为tmp.append(str(td.string)), 然后程序就能正常执行了。