codeforces crawl-demo

import requests,os,csv,re
from lxml import etree
class codeforces(object):
  global prefix,syspath
  prefix = 'https://codeforces.com'
  syspath = "D:\\szs\\opyion"
  """
  请求主页
  return: 分页链接
  """
  @staticmethod
  def index_page(x):
    content = requests.get("https://codeforces.com/problemset/page/{}?tags=800-1600".format(x)).text
    page_html = etree.HTML(content)
    dark_url = [prefix+i for i in page_html.xpath('//*[@class="problems"]//tr/td[2]/div[1]/a/@href')]
    print("正在解析首页====",dark_url)
    return dark_url
  """
  请求分页
  return: 标题，内容
  """
  @staticmethod
  def page_break(x):
    page_content = requests.get(x).text
    sorter_html = etree.HTML(page_content)
    title = sorter_html.xpath('//*[@class="title"]/text()')[0].split('.')[1]
    print("正在解析子页，===",title)
    statement = sorter_html.xpath('//*[@class="problem-statement"]/div[2]//text()')
    inputo = sorter_html.xpath('//*[@class="input"]//text()')
    output = sorter_html.xpath('//*[@class="output"]//text()')
    return title,statement,inputo,output
  @staticmethod
  def main():
    list_href = list(map(codeforces.index_page,list(range(20,29))))
    we =  [n for a in list_href for n in a ]
    data_list = list(map(codeforces.page_break,we))
    for data_i in data_list:
      zifu = '\/:*?"<>|'
      dirname = re.sub(zifu, "", data_i[0])
      dataPath = os.path.join(syspath, dirname)
      try:
        codeforces.__newdir(dataPath)
      except:
        print(dataPath, "未能新建")
        continue
      try:
        codeforces.__writeTxt(dataPath, ','.join(data_i[1]))
      except:
        print(dataPath, "Txt未能写入")
        continue
      try:
        codeforces.__writeCsv(dataPath, data_i[2])
        codeforces.__writeCsv(dataPath, data_i[3])
      except:
        print(dataPath, "Csv未能写入")
        continue
    return True
  @staticmethod
  def __newdir(path):
    isExists=os.path.exists(path)
    if not isExists:
      os.makedirs(path)
      print(path+'创建成功')
      return
    else:
      return False
  @staticmethod
  def __writeTxt(problem_statement,details):
    f = open(problem_statement+'\\'+'problem_statement.txt','a+',encoding='utf-8')
    f.write(details)
    f.close()
  @staticmethod
  def __writeCsv(testcase,details):
    f = open(testcase+"\\"+'testcase.csv', 'a+')
    writer = csv.writer(f)
    writer.writerow(details)
    f.close()
if __name__ == '__main__':
  zx = codeforces()
  zx.main()