用Python实现Flickr照片文本数据下载入库及图片保存（第一次帮忙）

本文链接：https://blog.csdn.net/DG_DLA/article/details/111503037

用Python实现Flickr照片文本数据下载入库及图片保存_fbc3173的博客-CSDN博客来源

#原来的代码
#!/usr/bin/python
import requests
import json
import time
import random
import pymysql
import urllib
import os
#创建保存图片的文件夹
path='保存图片的路径'
if not os.path.isdir(path):
    os.makedirs(path)
paths = path+'\\'
#处理特殊字符
def chuli(self):
    self = self.replace( "'", "\\'")
    #self = self.replace("/", "")
    #self =self. replace(chr(0), "")
    return self
#主要的url
api='https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key=你的key&text=&min_taken_date=2018-05-04&max_taken_date=2018-05-14&woe_id=2347570&extras=description,license,date_upload,date_taken,owner_name,icon_server,original_format,last_update,geo,tags,machine_tags,o_dims,views,media,path_alias,url_sq,url_t,url_s,url_q,url_m,url_n,url_z,url_c,url_l,url_o&per_page=500&format=json&sort=date-taken-asc&jsoncallback=?'
r= requests.get(api)
a = json.loads(r.text[14:-1])
# 打开数据库连接
db = pymysql.connect("localhost","root","123456","mysql" )
# 使用 cursor() 方法创建一个游标对象 cursor
cursor = db.cursor()
print(a["photos"]["pages"])
for i in range(a["photos"]["pages"]-1):#进行翻页，循环获得每页的url
    try:
        s=requests.get(api+"&page="+str(i+1))
        print(api+"&page="+str(i+1))
        b=json.loads(s.text[14:-1])#去掉外层的jsonFlickrApi()这几个字符串后解析成json的格式
        for i in range(len(b["photos"]["photo"])):#循环获取每页每个照片数据，其中b["photos"]["photo"]为列表，则我们只需按
            c = b["photos"]["photo"]              #序号获取每个照片数据
            print(c[i]["id"],c[i]["owner"],c[i]["datetaken"],c[i]["latitude"],c[i]["longitude"],c[i]["accuracy"],\
                  c[i]["place_id"],c[i]["woeid"],c[i]["url_z"],c[i]["description"]["_content"])
            content=chuli(c[i]["description"]["_content"])
            imgurl = c[i]["url_z"]
            try:
                urllib.request.urlretrieve(imgurl,'{}{}.jpg'.format(paths,c[i]["id"]))
                print('图片下载成功')
            except Exception as e:
                print('爬取图片失败')
            cursor.execute("insert into Hawaii5_4_14(id_1,owner,datetaken,latitude,longitude,accuracy,place_id,woeid,url_z,content) \
                  VALUES('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')"%(c[i]["id"],c[i]["owner"],c[i]["datetaken"],\
                c[i]["latitude"],c[i]["longitude"],c[i]["accuracy"],c[i]["place_id"],c[i]["woeid"],c[i]["url_z"],content))
            db.commit()
            print('第',i+1,'条数据成功插入')
            time.sleep(2)#休息2秒，防止服务器拒绝访问。
    except Exception as e:
        print('爬取失败')
# 关闭数据库连接
db.close()

在这里插入图片描述

把get返回的数据改一下保存为json，用firfox打开

看了一下只需改URL和删除不存在的json数据

mysql的安装MySQL的安装与配置–详细教程 - Winton-H - 博客园

pymysql的使用 Python连接MySQL数据库之pymysql模块使用 - 虫洞小鳗鱼 - 博客园

mysql -u root -p #登录数据库

cerate database 你创建的数据库名;#创建数据库

create table 你要的段名 (id int auto_increment primary key,id_1 varchar(30) not null,owner varchar(30) not null ,url_z varchar(300) not null );#创建段名和需要的分支

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import requests
import json
import time
import random
import pymysql
import urllib
import os
#创建保存图片的文件夹
path='保存图片的路径'
url_time = "{}-{}-{}".format(time.localtime(time.time())[0],time.localtime(time.time())[1],time.localtime(time.time())[2]-2)
key_api = "你的key"
proxies = {'https': 'https://127.0.0.1:1080','http': 'http://127.0.0.1:1080'}
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36'}


if not os.path.isdir(path):
    os.makedirs(path)
paths = path+'\\'
#处理特殊字符
def chuli(self):
    self = self.replace( "'", "\\'")
    #self = self.replace("/", "")
    #self =self. replace(chr(0), "")
    return self
#主要的url
api='https://api.flickr.com/services/rest?extras=can_comment,can_print,count_comments,count_faves,description,isfavorite,license,media,needs_interstitial,owner_name,path_alias,realname,rotation,url_sq,url_q,url_t,url_s,url_n,url_w,url_m,url_z,url_c,url_l&per_page=50&page=2&date={}&method=flickr.interestingness.getList&csrf=1608255676:kzbh2r1ldkg:6dcc1ea1648b001a5efb7d2e3c6ea762&api_key={}&format=json&hermes=1&hermesClient=1&reqId=d5041c76'.format(url_time,key_api)
r= requests.get(url=api,headers=headers,proxies=proxies)
a = json.loads(r.text[14:-1])
# 打开数据库连接

db = pymysql.connect(host="localhost",user = "root",password="密码",database="数据库",charset="utf8")


# 使用 cursor() 方法创建一个游标对象 cursor
cursor = db.cursor()
print(a["photos"]["pages"])

#create the object, assign it to a variable
proxy = urllib.request.ProxyHandler(proxies)
# construct a new opener using your proxy settings
opener = urllib.request.build_opener(proxy)
# install the openen on the module-level
urllib.request.install_opener(opener)

for i in range(a["photos"]["pages"]-1):#进行翻页，循环获得每页的url
    try:
        s=requests.get(url=(api+"&page="+str(i+1)),headers=headers,proxies=proxies)
        print(api+"&page="+str(i+1))
        b=json.loads(s.text[14:-1])#去掉外层的jsonFlickrApi()这几个字符串后解析成json的格式
        for i in range(len(b["photos"]["photo"])):#循环获取每页每个照片数据，其中b["photos"]["photo"]为列表，则我们只需按
            c = b["photos"]["photo"]#序号获取每个照片数据
            print("1")
            print(c[i]["id"],c[i]["owner"])
            content=chuli(c[i]["description"]["_content"])
            print("12")
            imgurl = c[i]["url_z"]
            
            try:
                urllib.request.urlretrieve(imgurl,'{}{}.jpg'.format(paths,c[i]["id"]))
                print('图片下载成功')
            except Exception as e:
                print('爬取图片失败')
            cursor.execute("insert into 你的段(id_1, owner, url_z) VALUES('%s','%s','%s');"%(c[i]["id"],c[i]["owner"],c[i]["url_z"]))
            db.commit()
            print('第',i+1,'条数据成功插入')
            time.sleep(2)#休息2秒，防止服务器拒绝访问。
    except Exception as e:
        print(e)
        print('爬取失败')
# 关闭数据库连接
db.close()