爬虫+数据库判断基金是否是4连跌来选择是否购买

于 2020-12-11 15:13:53 发布
阅读量161 收藏
点赞数
文章标签： 爬虫 数据库
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/woaishayueyue/article/details/111035746
版权

1-解析htnl存txt

# —*—  coding:  utf-8   -*- #
# @Time     :2020/12/10 16:29
# @Author   :woaishayueyue

import ast


class ParsingData():
    def data_name(i):
        for k, v in i.items():
            name = k
        return name

    def data_value(i):
        for k, v in i.items():
            value = v
        return value


def change_dict():
    with open("data/筛选数据.txt", "r", encoding="utf-8")as fp:
        dict_list = []
        for i in fp:
            one_dict = ast.literal_eval(i)
            dict_list.append(one_dict)
        return dict_list


# 判断4天是否为4连降
def numerical_trend(value):
    if len(value) < 4:
        return "0"
    final_four_list = [value[-4], value[-3], value[-2], value[-1]]
    for i in final_four_list:
        if i < 0:
            continue
        else:
            return "0"
    return "1"


def judgment_fund(dict_list):
    result_list = []
    for i in dict_list:
        name = ParsingData.data_name(i)
        value = ParsingData.data_value(i)
        # 判断value列表的数值趋势
        result = numerical_trend(value)
        if result == "1":
            result_list.append(name)
    return result_list


def save_fund(result_list):
    with open("data/最终结果.txt", "w", encoding="utf-8")as fp:
        for i in result_list:
            fp.write(i + "\n")
            fp.flush()
        fp.close()


def main():
    # 取出字典并写入列表 [{},{},]
    dict_list = change_dict()
    # 处理得到的列表并把符合条件的名称写入列表中
    result_list = judgment_fund(dict_list)
    # 最终数据存到文件中
    save_fund(result_list)


if __name__ == '__main__':
    main()

2-txt添加日期

# —*—  coding:  utf-8   -*- #
# @Time     :2020/12/9 16:03
# @Author   :woaishayueyue
import ast
import json
import os


def main():
    # 修改文件名
    a = "data/2020-12-10-new.txt"
    b = "data/2020-12-10.txt"
    c = "2020-12-10"

    with open(a, "w", encoding="utf-8") as f1:
        with open(b, "r", encoding="utf-8") as f2:
            for i in f2:
                i = ast.literal_eval(i)
                i["date_time"] = c
                f1.write(json.dumps(i, ensure_ascii=False) + "\n")
                f1.flush()
            f2.close()
        f1.close()

    # 删除旧文件
    os.unlink(b)
    # 重命名新文件
    os.rename(a, b)


if __name__ == '__main__':
    main()

3-info转到mysql数据库

# —*—  coding:  utf-8   -*- #
# @Time     :2020/12/7 13:13
# @Author   :woaishayueyue

import pymysql
import ast


conn = pymysql.Connect(
    host="127.0.0.1",
    port=3306,
    user="root",
    password="115096",
    database="fund",
    charset="utf8"
)
cursor = conn.cursor()


# 写数据库
def write_sql(data):
    sql = "insert into all_info(fund_id,fund_name,unit_net_worth,cumulative_net_worth,daily_growth,daily_growth_rate,date_time) values(%s,'%s',%s,%s,%s,%s,'%s')" % data
    cursor.execute(sql)
    conn.commit()


# 关闭数据库
def close_sql():
    cursor.close()
    conn.close()


def parsing_data():
    # 需要存入数据库的文件
    a = "data/2020-12-10.txt"
    all_list = []
    # 把数据转化为列表[{},{},]
    with open(a, "r", encoding="utf-8")as fp:
        for i in fp:
            all_list.append(ast.literal_eval(i))
        fp.close()
    # 计数
    count = 1
    # 取出想要的数据
    for i in all_list:
        a = i["fund_code"]
        b = pymysql.escape_string(i["fund_name"])
        c = i["unit_net_worth"]
        if c == "":
            c = 0
        d = i["cumulative_net_worth"]
        if d == "":
            d = 0
        e = i["daily_growth"]
        if e == "":
            e = 0
        f = i["daily_growth_rate"]
        if f == "":
            f = 0
        g = pymysql.escape_string(i["date_time"])
        data = (a, b, c, d, e, f, g)
        # 写入数据库
        write_sql(data)
        count += 1
        print(count)


def main():
    parsing_data()
    close_sql()


if __name__ == '__main__':
    main()

向数据库发送指令筛选

# —*—  coding:  utf-8   -*- #
# @Time     :2020/12/11 8:55
# @Author   :woaishayueyue

import pymysql

conn = pymysql.Connect(
    host="127.0.0.1",
    port=3306,
    user="root",
    password="115096",
    database="fund",
    charset="utf8"
)
cursor = conn.cursor()

# 总表去重建立新表
sql_1 = "CREATE TABLE all_info_1 SELECT DISTINCT fund_id,fund_name,unit_net_worth,cumulative_net_worth,daily_growth,daily_growth_rate,date_time FROM all_info;"
# 去掉4个值为0的数据
sql_2 = "CREATE TABLE all_info_2 SELECT * FROM all_info_1 WHERE unit_net_worth<>0 AND cumulative_net_worth<>0 AND daily_growth<>0 AND daily_growth_rate<>0;"
# 再次去掉净值为0的数据
sql_3 = "CREATE TABLE all_info_final SELECT * FROM all_info_2 WHERE unit_net_worth <>0"

print(cursor.execute(sql_1))
print(cursor.execute(sql_2))
print(cursor.execute(sql_3))
conn.commit()

cursor.close()
conn.close()

5-处理数据库返回的数据

# —*—  coding:  utf-8   -*- #
# @Time     :2020/12/10 14:29
# @Author   :woaishayueyue
#  数据库格式fund_name daily_growth date_time
import pymysql
import json


def connet_sql():
    conn = pymysql.Connect(
        host="127.0.0.1",
        port=3306,
        user="root",
        password="115096",
        database="fund",
        charset="utf8"
    )
    cursor = conn.cursor()
    sql = "SELECT fund_name,daily_growth, date_time FROM all_info_final ORDER BY fund_name,date_time ASC"
    cursor.execute(sql)
    data = cursor.fetchall()
    cursor.close()
    conn.close()
    return data


def data_processing(data):
    all_list = []
    for i in data:
        a = [i[0], i[1], i[2]]
        all_list.append(a)
    dict_list = []
    while len(all_list) != 0:
        count_2 = 0
        one_dict = {}
        one_daily_growth = []
        count_1 = 0
        for i in all_list:
            count = i[0]
            break
        for i in all_list:
            if i[0] == count:
                count_1 += 1
            else:
                break
        while count_2 < count_1:
            for i in all_list:
                fund_name = i[0]
                break
            for i in all_list:
                daily_growth = i[1]
                one_daily_growth.append(daily_growth)
                all_list.remove(i)
                break
            count_2 += 1
            if count_2 == count_1:
                one_dict[fund_name] = one_daily_growth
        dict_list.append(one_dict)
    return dict_list


def write_file(dict_list):
    with open("data/筛选数据.txt", "w", encoding="utf-8")as fp:
        for i in dict_list:
            fp.write(json.dumps(i, ensure_ascii=False) + "\n")
            fp.flush()
        fp.close()


def drop_sql():
    conn = pymysql.Connect(
        host="127.0.0.1",
        port=3306,
        user="root",
        password="115096",
        database="fund",
        charset="utf8"
    )
    cursor = conn.cursor()
    sql = "DROP TABLE all_info_1,all_info_2,all_info_final"
    cursor.execute(sql)
    cursor.close()
    conn.close()


def main():
    # 连接数据库并提取出自己想要的数据
    data = connet_sql()
    # 把得到的数据进行加工  {"万家中证500指数增强C": [-0.0082, -0.0082, 0.0048, -0.0279, 0.0021]}
    dict_list = data_processing(data)
    # 加工后的数据进行文件写入
    write_file(dict_list)
    # 删除数据库中的不要的表
    drop_sql()


if __name__ == '__main__':
    main()

6-判断选择购买的基金

# —*—  coding:  utf-8   -*- #
# @Time     :2020/12/10 16:29
# @Author   :woaishayueyue

import ast


class ParsingData():
    def data_name(i):
        for k, v in i.items():
            name = k
        return name

    def data_value(i):
        for k, v in i.items():
            value = v
        return value


def change_dict():
    with open("data/筛选数据.txt", "r", encoding="utf-8")as fp:
        dict_list = []
        for i in fp:
            one_dict = ast.literal_eval(i)
            dict_list.append(one_dict)
        return dict_list


# 判断4天是否为4连降
def numerical_trend(value):
    if len(value) < 4:
        return "0"
    final_four_list = [value[-4], value[-3], value[-2], value[-1]]
    for i in final_four_list:
        if i < 0:
            continue
        else:
            return "0"
    return "1"


def judgment_fund(dict_list):
    result_list = []
    for i in dict_list:
        name = ParsingData.data_name(i)
        value = ParsingData.data_value(i)
        # 判断value列表的数值趋势
        result = numerical_trend(value)
        if result == "1":
            result_list.append(name)
    return result_list


def save_fund(result_list):
    with open("data/最终结果.txt", "w", encoding="utf-8")as fp:
        for i in result_list:
            fp.write(i + "\n")
            fp.flush()
        fp.close()


def main():
    # 取出字典并写入列表 [{},{},]
    dict_list = change_dict()
    # 处理得到的列表并把符合条件的名称写入列表中
    result_list = judgment_fund(dict_list)
    # 最终数据存到文件中
    save_fund(result_list)


if __name__ == '__main__':
    main()

数据库字段

在这里插入图片描述
百分百可用！
禁止做商用！！转载需说明源头！！！

woaishayueyue
关注
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值