python实现notion导入bi

本文介绍了一个Python脚本(notion/notion_to_bi.py),用于将Notion平台的数据按照指定模式(merge、append、overwrite)导入到Greenplum数据库。脚本需要Notion的accesstoken、database ID、数据库连接参数(如用户名、密码、主机、端口、数据库名和模式名)以及白名单列名。merge模式保留已有数据,overwrite模式保持一致,append模式仅添加新数据。
摘要由CSDN通过智能技术生成
#!/usr/bin/env python
#coding=utf-8
import subprocess
import json
import os
import psycopg2
import re
import sys
import psycopg2.extras
import requests
import threading
import time
import sys
import re
import copy


def if_database(base_url,databaseID,token):
    url = base_url + "databases/" + databaseID
    headers = {
        "Accept": "application/json",
        "Notion-Version": "2022-02-22",
        "Authorization": "Bearer %s" %token
    }
    response = requests.request("GET", url, headers=headers)
    return response.status_code


def get_page(base_url,databaseID,token):
    url = base_url + "databases/" + databaseID + "/query"

    payload = {"page_size": 100}
    headers = {
        "Accept": "application/json",
        "Notion-Version": "2022-02-22",
        "Content-Type": "application/json",
        "Authorization": "Bearer %s" %token
    }

    total_list = []
    response = requests.request("POST", url, json=payload, headers=headers)
    total_list.extend(response.json()["results"])
    while True:
        if response.json()["has_more"] == True:
            payload["start_cursor"] = response.json()["next_cursor"]
            response = requests.request("POST", url, json=payload, headers=headers)
            total_list.extend(response.json()["results"])
        else:
            break
    print(len(total_list))
    notion_list = []
    for i in total_list:
        i["properties"]["pageid"] = {}
        i["properties"]["pageid"]["pageid"] = i["id"]
        i["properties"]["pageid"]["type"] = "pageid"
        notion_list.append(i["properties"])

    return notion_list

class Pq_opration():
    def __init__(self, conn, cursor):
        self.conn = conn
        self.cursor = cursor

    def select(self, select_sth, from_sth, where_sth=None):
        self.cursor.execute("select %s from %s %s" % (select_sth, from_sth, where_sth))
        rows = self.cursor.fetchall()
        return rows

    def create(self, db_table_total, db_sql):
        self.cursor.execute("create table %s %s" % (db_table_total, db_sql))
        self.conn.commit()

    def insert(self, _dict, db_table_total):
        if len(list(_dict.keys())) == 1:
            self.cursor.execute('''insert into ''' + db_table_total + '(' + ','.join(
                list(_dict.keys())) + ''') values ''' + '(' + '\'' + list(_dict.values())[0] + '\'' + ')')
        else:
            self.cursor.execute(
                '''insert into ''' + db_table_total + '(' + ','.join(list(_dict.keys())) + ''') values ''' + str(
                    tuple(_dict.values())))
        self.conn.commit()

    def delete(self, main_key, value, db_table_total):
        self.cursor.execute('''delete from ''' + db_table_total + ' where ' + main_key + ' = ' + '\'' + value + '\'')
        self.conn.commit()

    def merge(self, list1, list2, db_table_total,main_key="pageid"):
        count = 0
        num = 0
        for i in list1:
            tag = 0
            for j in list2:

                if main_key in list(i.keys()) and i != {} and j != {} and i[main_key] == j[main_key.lower()]:
                    sql = self.dict_to_sql(i, main_key)
                    self.cursor.execute(
                        '''update ''' + db_table_total + ' set ' + sql + ' where ' + main
  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 打赏
    打赏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包

打赏作者

太生气

你的鼓励将是我创作的最大动力

¥1 ¥2 ¥4 ¥6 ¥10 ¥20
扫码支付:¥1
获取中
扫码支付

您的余额不足,请更换扫码支付或充值

打赏作者

实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值