将爬虫数据写入数据库

最新推荐文章于 2024-11-06 20:49:14 发布

蟹黄堡㋡

最新推荐文章于 2024-11-06 20:49:14 发布

阅读量828

点赞数 10

分类专栏： python爬虫项目文章标签：数据库爬虫 python

本文链接：https://blog.csdn.net/weixin_70278423/article/details/140727072

版权

python爬虫项目专栏收录该内容

3 篇文章 0 订阅

订阅专栏

1. 在Linux机器上安装和配置好数据库
    安装：yum install mariadb-server -y
        mysql
    配置：防火墙(启用) -> 规则 (允许/禁止ip,port,协议) -> 关掉
         阿里云/腾讯云 -> 安全组
         应用授权 -> python（主机）连接 mysql(主机/用户名/密码)

2.

sys => root => 123456
mysql -uroot -p => db123456
权限远程访问 => root/123456
在xshell端命令
 mysql -uroot -p
show databases;
use mysql;
GRANT ALL PRIVILEGES ON *.* to 'root'@'%' IDENTIFIED BY "123456";
 FLUSH PRIVILEGES;
SELECT user,host FROM user;

python操作数据库的模块
      PyMySQL
      SQLALCHEMY(更高级->支持MySQL及其他数据，支持MySQL依赖于PyMySQL)
         ORM => 对象关系映射
在pycharm打开数据库进行连接 然后再此进行操作
"""

# pip install Pymysql

要现在虚拟机连接数据库并创建 create database pachong;

演示从豆瓣网爬取电影信息并保存到MySQL中

import pymysql
import json
import download
# 创建数据库连接
db = pymysql.connect(host="192.168.159.142",
                     port=3306,
                     user="root",
                     password="123456",
                     db="pachong",
                     charset='utf8mb4')
# 创建游标（执行）
cursor = db.cursor()
# 执行sql语句：execute(sql)
sql = "DROP TABLE IF EXISTS movies"
cursor.execute(sql)
sql = """CREATE TABLE movies( 
        title CHAR(50) CHARACTER SET utf8mb4 NOT NULL ,
        rate CHAR(20),
        cover CHAR(200));
      """
cursor.execute(sql)
# name = ["热门","最新","豆瓣高分","冷门佳片","华语","欧美","韩国","日本"]DECIMAL(3, 1)
name = ["热门"]
for i in name:
    all_movies = download.get_text(f"https://movie.douban.com/j/search_subjects?type=movie&tag={i}&page_limit=50&page_start=0")
    all_movies_obj = json.loads(all_movies)
    for movie in all_movies_obj["subjects"]:
        sql = "INSERT INTO movies (title, rate, cover) VALUES (%s, %s, %s)"
        data = (movie['title'], movie['rate'], movie['cover'])
        cursor.execute(sql, data)
db.commit()

# # 增删改查
# sql = "INSERT INTO app(id,name) VALUES(1,'嘻嘻嘻');"
# cursor.execute(sql)
# sql = "INSERT INTO app(id,name) VALUES (%s,%s)"
# datas = [(2,'咯咯咯'),(3,'嗯嗯嗯'),(4,'哈哈哈')]
# cursor.executemany(sql,datas)
# print(1)
#
# # 查询
# sql = "SELECT * FROM app;"
# cursor.execute(sql)
# result = cursor.execute(sql)
# print(result)
# result = cursor.fetchone()
# print(result)
# result = cursor.fetchmany(2)
# print(result)
# result = cursor.fetchall()
# print(result)
# # 更新
# sql = "UPDATE app SET name='李琳' WHERE id=1"
# cursor.execute(sql)
#
# sql = "DELETE FROM app WHERE name like '嘻%'"
# cursor.execute(sql)
# result = cursor.execute(sql)
# print(result)
# db.commit()# 提交修改到数据库