python爬取网页表格_利用Python爬取fofa网页端数据

安装环境：

pip install requests

pip install lxml

pip install fire

使用命令：

python fofa.py -s=title="你的关键字" -o="结果输出文件" -c="你的cookie"

代码如下：

import requests,time,base64,fire

from lxml import etree

def fofasc(s,o,c):

try:

sbase64 = (base64.b64encode(s.encode(‘utf-8‘))).decode(‘utf-8‘)

cookies = {

"_fofapro_ars_session": c

}

headers = {

‘User-Agent‘: ‘Mozilla/5.0 (Linux; Android 7.1.2; PCRT00 Build/N2G48H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.158 Safari/537.36 fanwe_app_sdk sdk_type/android sdk_version_name/4.0.1 sdk_version/2020042901 screen_width/720 screen_height/1280‘,

}

for i in range(1,1000):

url = "https://www.fofa.so/result?q="+s+"&qbase64="+sbase64+"&full=true&page="+str(i)

r = requests.get(url, headers=headers, cookies=cookies)

soup = etree.HTML(r.text)

result = soup.xpath(‘//*[@id="ajax_content"]/div/div/div/a/text()‘)

print(result)

if result != []:

for rs in result:

with open(o,mode="a+") as f:

f.write(rs+"\n")

else:

print("已经获取不到任何数据，爬取完毕！")

break

time.sleep(2)

except KeyboardInterrupt:

print(‘用户退出‘)

if __name__ == ‘__main__‘:

fire.Fire(fofasc)

原文地址：https://www.cnblogs.com/aptkekeo/p/13288726.html