关于爬虫的一点笔记

关闭连接，避免'Too many connections'
 

def save(data):
    """存储数据"""
    engine = create_engine('mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8'.format(USER, PASS, HOST, PORT, DB))
    connect = engine.connect()
    try:
        df = pd.DataFrame(data, index=[0])
        df.to_sql(name=TABLE, con=connect, if_exists='append', index=False)
    except Exception as e:
        logging.error("\nError: %s, Please check the error.\n" % e.args)
        _ = e
    finally:
        engine.dispose()#关闭连接，避免'Too many connections'

去重

DELETE meituan.meishi
FROM
     meituan.meishi,
     (
     SELECT min(rid) rid, id
     FROM meituan.meishi
     GROUP BY id
     HAVING count(id)>1
     )t
WHERE meituan.meishi.id=t.id
AND meituan.meishi.rid>t.rid;