python 解析网页

https://zhuanlan.zhihu.com/p/32675668

用法: python get-type.py TaskType ./test.ts

import sys
import requests
import codecs #codecs可以很方便地对文本文件进行编码和解码
from bs4 import BeautifulSoup

keyword = sys.argv[1]
dest = ‘’.join(sys.argv[1:2]) or ‘./test.ts’

language = sys.argv[2]

mode=sys.argv[3]

url = “http://172.24.12.40:13001/camel/help/v1/api”

user_agent = ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36’
headers = {‘User-Agent’:user_agent}
r = requests.get(url, headers = headers)
r.encoding = ‘utf-8’ #使用utf-8对内容编码
f = codecs.open(’./test.ts’, ‘w’, encoding=‘utf-8’)
soup = BeautifulSoup(r.text, ‘lxml’, from_encoding=‘utf-8’)

分离出我想要的文本

table = soup.find(‘a’, { ‘name’: keyword }).find_parent().find_next_sibling().find(‘table’)
print(table.find_all(‘tr’)[1].find_all(‘td’))

f.write(‘export enum %s {\n’ % keyword)

def formatStr (value, en, zh, spr):
str = zh if zh.find(’ ‘) == -1 else ‘’%s’’ % zh
return ’ %s = %s%s\n’ % (str, int(value, 16), spr)

counter = 0

去掉表头

rows = table.find_all(‘tr’)[1:]
for row in rows:
cells = row.find_all(‘td’)
en = cells[0].text
value = cells[1].text
zh = cells[2].text
counter = counter + 1
spr = ‘,’ if counter < len(rows) else ‘’
f.write(formatStr(value, en = en, zh = zh, spr = spr))

f.write(’}\n\n’)
f.close()