php抓取访客国旗,爬取世界各国和地区数据&国旗和国徽的高清图标和图片

import requests

from lxml import etree

import pandas as pd

import time

import os

import urllib.request

class WorldFlag_Spider(object):

def init(self):

self.start_url = 'http://cn.flagbox.net/'   # 网站根地址

self.headers = {

"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 FS"

}

self.main_folder = r'/Volumes/download/世界各国和地区数据&国旗汇总'  # 主路径

self.content_list = []  # 将包含数据的字典存放到这个列表

# 获取主链接

def data_range(self):

start_res = requests.get(url=self.start_url, headers=self.headers)

start_sel = etree.HTML(start_res.content.decode())

main_url_list = start_sel.xpath('//div[@id="page-bg"]/table/tr/td[2]/ul/li[position()<7]/a/@href')

main_url = ['http://cn.flagbox.net/' + i for i in main_url_list]

return main_url

# 获取次链接

def get_url(self, url_1):

sec_res = requests.get(url=url_1, headers=self.headers)

sec_sel = etree.HTML(sec_res.content.decode())

s_link_list = sec_sel.xpath('//tr/td/a[2]/@href')

s_link = ['http://cn.flagbox.net/' + s for s in s_link_list]

return s_link

# 解析获取到的链接

def paser_url(self, url):

# print(url)

response = requests.get(url=url, headers=self.headers)

if response.status_code == 200:

selector = etree.HTML(response.content.decode())

return selector

else:

print('请求失败')

# 获取数据

def get_content_list(self, selector):

items = {}

try:

#  国名

items['国名'] = selector.xpath('//table[@style="border-collapse: collapse; "]/tr[1]/td[2]/text()')

items['国名'] = items['国名'][0] if len(items['国名']) > 0 else None

# 面积

items['面积'] = selector.xpath('//table[@style="border-collapse: collapse; "]/tr[3]/td[2]/text()')

items['面积'] = items['面积'][0] if len(items['面积']) > 0 else None

# 首都

items['首都'] = selector.xpath('//table[@style="border-collapse: collapse; "]/tr[4]/td[2]/text()')

items['首都'] = items['首都'][0] if len(items['首都']) > 0 else None

# 所处位置

items['所处位置'] = selector.xpath('//table[@style="border-collapse: collapse; "]/tr[5]/td[2]/text()')

items['所处位置'] = items['所处位置'][0] if len(items['所处位置']) > 0 else None

# 电话区号

items['电话区号'] = selector.xpath('//table[@style="border-collapse: collapse; "]/tr[6]/td[2]/text()')

items['电话区号'] = items['电话区号'][0].replace(' /', '') if len(items['电话区号']) > 0 else None

# 货币名称

items['货币名称'] = selector.xpath('//table[@style="border-collapse: collapse; "]/tr[8]/td[2]/text()')

items['货币名称'] = items['货币名称'][0] if len(items['货币名称']) > 0 else None

# 语言

items['语言'] = selector.xpath('//table[@style="border-collapse: collapse; "]/tr[14]/td[2]/text()')

items['语言'] = items['语言'][0] if len(items['语言']) > 0 else None

self.content_list.append(items)

return items['国名']

except IndexError:

pass

# 获取图片链接

def get_imagelink(self, selector):

# 国旗高清图片

f_picture = 'http://cn.flagbox.net/' + selector.xpath('//div[@id="page-bg"]/table/tr/td[1]/table/tr/td/ul/li[7]/a/@href')[0]

# 国旗高清图标

f_icon = 'http://cn.flagbox.net/' + selector.xpath('//div[@id="page-bg"]/table/tr/td[1]/table/tr/td/p/img[2]/@src')[0]

# 国徽链接

emblem = 'http://cn.flagbox.net/' + selector.xpath('//div[@id="page-bg"]/table/tr/td[1]/table/tr/td/p[@align="center"][2]/img/@src')[0]

return f_picture, f_icon, emblem

# 保存数据

def save_content_list(self, items):

pf = pd.DataFrame(self.content_list)

# 指定列的顺序

order = ['国名','面积','首都','所处位置','电话区号','货币名称','语言']

pf = pf[order]

# 打开excel文件

file_path = pd.ExcelWriter(self.main_folder + '/' + '世界各国和地区信息汇总.xlsx')

# 替换空单元格

pf.fillna(' ', inplace=True)

# 输出

pf.to_excel(file_path, encoding='utf-8', index=False,sheet_name="sheet1")

file_path.save()

print('正在保存:{}'.format(items))

# print(self.content_list)

# 下载图片

def download_image(self, links, item):

try:

# 创建子文件夹

folder = self.main_folder + '/' + item + '/'

if not os.path.exists(folder):

os.mkdir(folder)

for link in links:

with open(folder + link.split('/')[-2] + os.path.splitext(link)[-1], 'wb') as f:

image = requests.get(link, headers=self.headers).content

f.write(image)

# 用urllib.request模块进行下载：

# urllib.request.urlretrieve(link, folder + link.split('/')[-2] + os.path.splitext(link)[-1])

except:

print('保存失败')

def run(self):

# 创建主路径

if not os.path.exists(self.main_folder):

os.mkdir(self.main_folder)

# 1.获取各大洲网页链接

main_url = self.data_range()

# 2.遍历各大洲网页链接列表

for url_1 in main_url:

# 3.获取各个国家网页链接

s_link = self.get_url(url_1)

for url in s_link:

# 4.解析各个国家网页链接数据

selector = self.paser_url(url)

# time.sleep(1)

# 5.获取各国的各项数值

item = self.get_content_list(selector)

# 6.获取各国的国旗图片图标和国徽

links = self.get_imagelink(selector)

# 7.保存数据和下载图片

self.save_content_list(item)

self.download_image(links, item)

if name == 'main':

WorldFlag = WorldFlag_Spider()  # 实例化对象

WorldFlag.run()