Prometheus、TDengine时序库写入速度对比

Promethesu用pushgateway方式写入

TDengine 用java client的两种方式，JNI和REST写入

每次连续写入1000个点的数据，结果如下图，prometheus基本需要17秒，tdengine rest需要500毫秒，tdengine jni需要100毫秒，差距比较明显，也可能是我对prometheus的写入方式不对，后面找到更快的方式再补充。可能部署的环境也会有点影响，prometheus是部署在windows环境下的。

 

 环境说明：

Promethesu：2.30.3，pushgateway：1.4.2，单机部署，windows10

TDengine：2.2.2.0 单机部署，centos7.9

示例代码：

pom

<dependency>
    <groupId>io.prometheus</groupId>
    <artifactId>simpleclient</artifactId>
    <version>0.12.0</version>
</dependency>
<dependency>
    <groupId>io.prometheus</groupId>
    <artifactId>simpleclient_pushgateway</artifactId>
    <version>0.12.0</version>
</dependency>

<dependency>
    <groupId>com.taosdata.jdbc</groupId>
    <artifactId>taos-jdbcdriver</artifactId>
    <version>2.0.34</version>
</dependency>

写Prometheus

@Scheduled(cron = "0/15 * * * * ?")
void doWrite() {
    PushGateway pg = new PushGateway(url);
    Map<String, String> groupingKey = new HashMap<String, String>();
    groupingKey.put("instance", "my_instance");

    Meteo data = new Meteo();

    String str = new SimpleDateFormat("yyyy-mm-dd HH:mm:ss").format(new Date());
    data.setDate(str);

    long startTime = System.currentTimeMillis();    //获取开始时间

    try {
        for (int i = 0; i < 1000; i++) {
            data.setDirection(0 + r.nextDouble() * 180);
            data.setHumidity(20 + r.nextDouble() * 10);
            data.setPressure(900 + r.nextDouble() * 50);
            data.setSpeed(3 + r.nextDouble());
            data.setTemperature(20 + r.nextDouble() * 15);
            data.setRain(1 + r.nextDouble());

            Gauge guage = Gauge.build(gauge, "This is my custom metric.").labelNames("station", "quota", "date").create();
            CollectorRegistry registry = new CollectorRegistry();

            guage.labels("sta" + i, "temperature", data.getDate()).set(data.getTemperature());
            guage.labels("sta" + i, "humidity", data.getDate()).set(data.getHumidity());
            guage.labels("sta" + i, "pressure", data.getDate()).set(data.getPressure());
            guage.labels("sta" + i, "wind-speed", data.getDate()).set(data.getSpeed());
            guage.labels("sta" + i, "wind-direction", data.getDate()).set(data.getDirection());
            guage.labels("sta" + i, "rain", data.getDate()).set(data.getRain());
            guage.register(registry);
            pg.pushAdd(registry, "sta" + i, groupingKey);
        }
    }
    catch (Exception e) {
        e.printStackTrace();
    }

    long endTime = System.currentTimeMillis();    //获取结束时间
    System.out.println("pushgateway：" + str + ", 执行时间：" + (endTime - startTime));
}

tdengine rest

@Scheduled(cron = "0/15 * * * * ?")
public void writeTaosRest() {
    try {
        Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
        String jdbcUrl = "jdbc:TAOS-RS://192.188.234.54:6041/mydb?user=root&password=taosdata";
        Properties connProps = new Properties();
        connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
        connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
        connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
        Connection conn = DriverManager.getConnection(jdbcUrl, connProps);

        Statement stmt = conn.createStatement();
        Meteo data = new Meteo();
        String str = new SimpleDateFormat("yyyy-mm-dd HH:mm:ss").format(new Date());
        data.setDate(str);

        long startTime = System.currentTimeMillis();    //获取开始时间
        Timestamp s1 = new Timestamp(startTime);

        for(int j=0; j<1000; j++) {
            data.setDirection(0+ r.nextDouble()*180);
            data.setHumidity(20+r.nextDouble()*10);
            data.setPressure(900+r.nextDouble()*50);
            data.setSpeed(3+r.nextDouble());
            data.setTemperature(20+ r.nextDouble()*15);
            data.setRain(1+r.nextDouble());

            String tablename = "sta"+j;

            String strSql = "insert into " + tablename + " using meteodata tags ('" + tablename + "') (ts, temperature, pressure, speed, direction, humidity, rain) values('";
            strSql += s1+"', ";
            strSql += data.getTemperature()+", ";
            strSql += data.getPressure()+", ";
            strSql += data.getSpeed()+", ";
            strSql += data.getDirection()+", ";
            strSql += data.getHumidity()+", ";
            strSql += data.getRain()+")";

            int affectedRows = stmt.executeUpdate(strSql);
        }

        long endTime = System.currentTimeMillis();    //获取结束时间
        System.out.println("writeTaosRest：" + str + ", 执行时间：" + (endTime - startTime));

        // 执行完毕，释放资源：
        stmt.close();
        conn.close();
    }
    catch (Exception e) {
        e.printStackTrace();
    }
}

tdengine jni

@Scheduled(cron = "0/15 * * * * ?")
public void writeTaos() {
    try {
        Class.forName("com.taosdata.jdbc.TSDBDriver");
        String jdbcUrl = "jdbc:TAOS://192.188.234.54:6030/mydb?user=root&password=taosdata";
        Properties connProps = new Properties();

        connProps.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
        connProps.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
        connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
        connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
        connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "Asia/Shanghai");

        Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
        Statement stmt = conn.createStatement();

        Meteo data = new Meteo();

        String str = new SimpleDateFormat("yyyy-mm-dd HH:mm:ss").format(new Date());
        data.setDate(str);
        long startTime = System.currentTimeMillis();    //获取开始时间

        Timestamp s1 = new Timestamp(startTime);
        //从 2.1.2.0 版本开始，TDengine 的 JDBC-JNI 实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。
        //采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。
        //TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? using meteodata tags ('?') ('ts', temperature, pressure, speed, direction, humidity, rain) values(?, ?, ?, ?, ?, ?, ?)");

        for(int j=0; j<1000; j++) {
            data.setDirection(0+ r.nextDouble()*180);
            data.setHumidity(20+r.nextDouble()*10);
            data.setPressure(900+r.nextDouble()*50);
            data.setSpeed(3+r.nextDouble());
            data.setTemperature(20+ r.nextDouble()*15);
            data.setRain(1+r.nextDouble());

            String tablename = "sta"+j;

            //"insert into ? using meteodata tags (?) (ts, temperature, pressure, speed, direction, humidity, rain) values(?, ?, ?, ?, ?, ?, ?)"
            String strSql = "insert into " + tablename + " using meteodata tags ('" + tablename + "') values('";
            strSql += s1+"', ";
            strSql += data.getTemperature()+", ";
            strSql += data.getPressure()+", ";
            strSql += data.getSpeed()+", ";
            strSql += data.getDirection()+", ";
            strSql += data.getHumidity()+", ";
            strSql += data.getRain()+")";

            int affectedRows = stmt.executeUpdate(strSql);
        }

        long endTime = System.currentTimeMillis();    //获取结束时间
        System.out.println("writeTaosJni：" + str + ", 执行时间：" + (endTime - startTime));

        // 执行完毕，释放资源：
        //s.columnDataCloseBatch();
        stmt.close();
        conn.close();

    }
    catch (Exception e) {
        e.printStackTrace();
    }
}