Main.py:
import re
import requests
from requests.exceptions import RequestException
from config import *
import pymongo
client = pymongo.MongoClient(MONGO_URL
db = client[MONGO_DB]
def get_one_page(url):
try:
response = requests.get(url)
if response.status_code == 200:
return response.text
return None
except RequestException:
return None
def get_data_param(html):
pattern = re.compile('<tr.*?bg_a">.*?width="46">(.*?)</td>.*?class="left">(.*?)</td>.*?'
'width="50">(.*?)</td>.*?>(.*?)</td>.*?>(.*?)</td>.*?>(.*?)</td>.*?>'
'(.*?)</td>.*?>(.*?)</td>.*?>(.*?)</td>.*?>(.*?)</td>.*?width="50">'
'(.*?)</td>.*?width="70">(.*?)</td>', re.S)
items = re.findall(pattern,html)
list = []
for item in items:
list.append(item[0])
list.append(item[1])
list.append(item[2])
list.append(item[3])
<