【作业】2022.5.9 网络数据采集

作业1：从给定API访问特定图片网站并自动化保存图片到本地

import os
from json import loads
import requests


def make_dir(path):
    if not os.path.exists(path):
        os.mkdir(path)


def get_count(path):
    images = os.listdir(path)
    if images:
        return int(images[-1][5:-4]) + 1
    else:
        return 1


def sig_encoding(origin_url='https://api.ixiaowai.cn/api/api.php?return=json'):
    resp1 = requests.get(url=origin_url)
    resp1.encoding = 'utf-8-sig'
    return resp1.text


if __name__ == '__main__':
    make_dir('homework_image')
    count_image = get_count('homework_image')

    for x in range(10):
        result_url = loads(sig_encoding())['imgurl']
        resp2 = requests.get(url=result_url)
        open(f'homework_image/image{x + count_image}.jpg', 'ab').write(resp2.content)

作业2：获取中国新闻网的新闻，并以类型、新闻名、链接、时间为表头创建csv表格

from re import search, findall
import csv


def get_resp(path):
    new_header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36'
    }
    return requests.get(url=path, headers=new_header)


def decide_encoding(path):
    resp = get_resp(path)
    news_encoding = search(r'charset="?(.+?)"', resp.text)
    resp.encoding = news_encoding.group(1)
    return resp.text


def get_news(news_url='https://www.chinanews.com.cn/scroll-news/news1.html'):
    # 获取原始信息
    origin_text = decide_encoding(news_url)

    # 获取所需数据
    news_text = findall(r'"dd_lm"(.+)</div></li>', origin_text)
    news_result = [['type', 'name', 'link', 'time']]
    for x in news_text:
        type = search(r'shtml>(.+?)</a>', x).group(1)
        name = search(r'shtml\">(.+?)</a>', x).group(1)
        link = 'https://www.chinanews.com.cn' + search(r'<a href=\"(.+?)">', x).group(1)
        time = search(r'<div class=\"dd_time\">(.+)', x).group(1)
        news_result.append([type, name, link, time])

    return news_result


def repeat_get_news(path1='news', page1=1, page2=1):
    if not os.path.exists(path1):
        make_dir(path1)

    # 循环采集第page1到page2的新闻
    for x in range(page1, page2 + 1):
        news_url = 'https://www.chinanews.com.cn/scroll-news/news' + f'{x}' + '.html'
        news_list = get_news(news_url=news_url)

        # 写入数据
        news_path = f'{path1}/news{x}.csv'
        csv.writer(open(news_path, 'w', encoding='utf-8', newline='')).writerows(news_list)


if __name__ == '__main__':
    repeat_get_news(page1=1, page2=2)