ck pr 记录

https://www.cnblogs.com/daniel-hutao/p/open-a-pr-in-github.html#53%E7%AC%AC%E4%B8%89%E6%AD%A5%E6%9B%B4%E6%96%B0%E6%9C%AC%E5%9C%B0%E5%88%86%E6%94%AF%E4%BB%A3%E7%A0%81

构建: 在/Clickhouse https://clickhouse.com/docs/en/development/build

export CC=clang-18
export CXX=clang++-18
cmake -S . -B build
cmake --build build

使用cmake -S . -B build 生成构建系统文件。
使用cmake --build build 执行实际的编译过程。
如果以前构建过，应该把build文件夹中的cashe文件删掉：

rm -rf CMakeCache.txt CMakeFiles/

CMakeFiles是由CMake生成的目录，用于存放生成的构建系统和编译过程中的临时文件。在使用CMake进行项目构建时，CMake会生成构建系统文件（如Makefile或Ninja文件），这些文件会放在CMakeFiles目录中。这些构建系统文件描述了项目的编译规则和依赖关系，用于执行实际的编译过程。

除了构建系统文件，CMake还会在CMakeFiles目录中生成其他临时文件，例如记录编译器和链接器信息的文件、记录编译过程日志的文件等。

一般来说，CMakeFiles目录是自动生成的，并且在构建过程中会被频繁更新和修改。在项目构建完成后，可以将CMakeFiles目录删除，以清理构建过程中生成的临时文件。

CREATE TABLE table1 (
    id UInt32,
    name String,
    date Date
) ENGINE = MergeTree()
ORDER BY date
PARTITION BY toYYYYMM(date)
SETTINGS index_granularity = 8192;

CREATE TABLE table2
(
    id UInt32,
    name String,
    date Date
) ENGINE = MergeTree()
ORDER BY date
PARTITION BY toYYYYMM(date)
SETTINGS index_granularity = 8192
AS SELECT * FROM table1;

INSERT INTO table1 (id, name, date) VALUES
    (1, 'John', '2022-01-01'),
    (2, 'Emma', '2022-01-02'),
    (3, 'Michael', '2022-01-03');

INSERT INTO table2 (id, name, date) VALUES
    (4, 'Sophia', '2022-02-01'),
    (5, 'William', '2022-02-02'),
    (6, 'Olivia', '2022-02-03');
INSERT INTO table2 (id, name, date) VALUES
    (7, 'Sophia', '2022-02-10'),
    (8, 'William', '2022-02-11'),
    (9, 'Olivia', '2022-02-12');
INSERT INTO table2 (id, name, date) VALUES
    (14, 'Sophia', '2022-03-10'),
    (15, 'William', '2022-03-11'),
    (16, 'Olivia', '2022-03-12');


SELECT
    partition,
    name
FROM system.parts
WHERE (database = currentDatabase()) AND (table = 'table1') AND active
ORDER BY name ASC

SELECT * FROM table2;

ALTER TABLE table3 ATTACH PARTITION 202202 FROM table1;

ALTER TABLE table1 DETACH PARTITION 202202

ALTER TABLE table2 ATTACH PARTITION 202202

alter table table3 detach partition 202202

ALTER TABLE local_table ATTACH PARTITION  tuple() FROM uk_price_paid;

SELECT
    toYear(date) AS year,
    round(avg(price)) AS price,
    bar(price, 0, 1000000, 80)
FROM uk_price_paid
GROUP BY year
ORDER BY year ASC

SELECT
    sum(bytes_on_disk) AS table_size
FROM
    system.parts
WHERE
    active AND (database = currentDatabase()) AND table = 'local_table'

ATTACH TABLE uk_price_paid_r2 UUID 'e7a4f8c1-8d5b-4f04-9a5a-3e8a8c5d2eff'
(
    `price` UInt32,
    `date` Date,
    `postcode1` LowCardinality(String),
    `postcode2` LowCardinality(String),
    `type` Enum8('other' = 0, 'terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4),
    `is_new` UInt8,
    `duration` Enum8('unknown' = 0, 'freehold' = 1, 'leasehold' = 2),
    `addr1` String,
    `addr2` String,
    `street` LowCardinality(String),
    `locality` LowCardinality(String),
    `town` LowCardinality(String),
    `district` LowCardinality(String),
    `county` LowCardinality(String)
)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/table_name', '{replica}')
ORDER BY (postcode1, postcode2, addr1, addr2)
SETTINGS disk = disk(type = web, endpoint = 'https://raw.githubusercontent.com/ClickHouse/web-tables-demo/main/web/')

Create table local_table
 (
    price UInt32,
    date Date,
    postcode1 LowCardinality(String),
    postcode2 LowCardinality(String),
    type Enum8('other' = 0, 'terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4),
    is_new UInt8,
    duration Enum8('unknown' = 0, 'freehold' = 1, 'leasehold' = 2),
    addr1 String,
    addr2 String,
    street LowCardinality(String),
    locality LowCardinality(String),
    town LowCardinality(String),
    district LowCardinality(String),
    county LowCardinality(String)
)
ENGINE = MergeTree
ORDER BY (postcode1, postcode2, addr1, addr2)

 ATTACH TABLE uk_price_paid UUID 'cf712b4f-2ca8-435c-ac23-c4393efe52f7'
(
    price UInt32,
    date Date,
    postcode1 LowCardinality(String),
    postcode2 LowCardinality(String),
    type Enum8('other' = 0, 'terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4),
    is_new UInt8,
    duration Enum8('unknown' = 0, 'freehold' = 1, 'leasehold' = 2),
    addr1 String,
    addr2 String,
    street LowCardinality(String),
    locality LowCardinality(String),
    town LowCardinality(String),
    district LowCardinality(String),
    county LowCardinality(String)
)
ENGINE = MergeTree
ORDER BY (postcode1, postcode2, addr1, addr2)
SETTINGS disk = disk(type = web, endpoint = 'https://raw.githubusercontent.com/ClickHouse/web-tables-demo/main/web/')

Create table local_table
(
    price UInt32,
    date Date,
    postcode1 LowCardinality(String),
    postcode2 LowCardinality(String),
    type Enum8('other' = 0, 'terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4),
    is_new UInt8,
    duration Enum8('unknown' = 0, 'freehold' = 1, 'leasehold' = 2),
    addr1 String,
    addr2 String,
    street LowCardinality(String),
    locality LowCardinality(String),
    town LowCardinality(String),
    district LowCardinality(String),
    county LowCardinality(String)
)
ENGINE = MergeTree
ORDER BY (postcode1, postcode2, addr1, addr2)

ALTER TABLE local_table ATTACH PARTITION tuple() FROM uk_price_paid

SELECT
    toYear(date) AS year,
    round(avg(price)) AS price,
    bar(price, 0, 1000000, 80)
FROM local_table
GROUP BY year
ORDER BY year ASC

ALTER TABLE local_table REPLACE PARTITION tuple() FROM uk_price_paid

integration test 编写

/tests/integration/readme.md如下：

ClickHouse integration tests

This directory contains tests that involve several ClickHouse instances, custom configs, ZooKeeper, etc.

Running natively

Prerequisites:

• Ubuntu 20.04 (Focal) or higher.
• docker. Minimum required API version: 1.25, check with docker version.

You must install latest Docker from
https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#set-up-the-repository
Don’t use Docker from your system repository.

• pip and libpq-dev. To install: sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev libkrb5-dev python3-dev
• py.test testing framework. To install: sudo -H pip install pytest
• docker-compose and additional python libraries. To install:

sudo -H pip install \
    PyMySQL \
    aerospike \
    avro \
    cassandra-driver \
    confluent-kafka \
    dicttoxml \
    docker \
    docker-compose \
    grpcio \
    grpcio-tools \
    kafka-python \
    kazoo \
    minio \
    lz4 \
    protobuf \
    psycopg2-binary \
    pymongo \
    pytz \
    pytest \
    pytest-timeout \
    redis \
    tzlocal==2.1 \
    urllib3 \
    requests-kerberos \
    dict2xml \
    hypothesis \
    pyhdfs \
    pika \
    nats-py

(highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of “pip”: sudo apt install -y docker docker-compose python3-pytest python3-dicttoxml python3-docker python3-pymysql python3-protobuf python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python python3-pytest-timeout python3-minio

Some tests have other dependencies, e.g. spark. See docker/test/integration/runner/Dockerfile for how to install those. See docker/test/integration/runner/dockerd-entrypoint.sh for environment variables that need to be set (e.g. JAVA_PATH).

If you want to run the tests under a non-privileged user, you must add this user to docker group: sudo usermod -aG docker $USER and re-login.
(You must close all your sessions (for example, restart your computer))
To check, that you have access to Docker, run docker ps.

Run the tests with the pytest command. To select which tests to run, use: pytest -k <test_name_pattern>

By default tests are run with system-wide client binary, server binary and base configs. To change that,
set the following environment variables:

• CLICKHOUSE_TESTS_SERVER_BIN_PATH to choose the server binary.
• CLICKHOUSE_TESTS_CLIENT_BIN_PATH to choose the client binary.
• CLICKHOUSE_TESTS_BASE_CONFIG_DIR to choose the directory from which base configs (config.xml andusers.xml) are taken.

Please note that if you use separate build (ENABLE_CLICKHOUSE_ALL=OFF), you need to build different components, including but not limited to ENABLE_CLICKHOUSE_LIBRARY_BRIDGE=ON ENABLE_CLICKHOUSE_ODBC_BRIDGE=ON ENABLE_CLICKHOUSE_KEEPER=ON. So it is easier to use ENABLE_CLICKHOUSE_ALL=ON

For tests that use common docker compose files you may need to set up their path with environment variable: DOCKER_COMPOSE_DIR=$HOME/ClickHouse/docker/test/integration/runner/compose

Running with runner script

The only requirement is fresh configured docker and
docker pull clickhouse/integration-tests-runner

Notes:

• If you want to run integration tests without sudo you have to add your user to docker group sudo usermod -aG docker $USER. More information about docker configuration.
• If you already had run these tests without ./runner script you may have problems with pytest cache. It can be removed with rm -r __pycache__ .pytest_cache/.
• Some tests maybe require a lot of resources (CPU, RAM, etc.). Better not try large tests like test_cluster_copier or test_distributed_ddl* on your laptop.

You can run tests via ./runner script and pass pytest arguments as last arg:

$ ./runner --binary $HOME/ClickHouse/programs/clickhouse  --odbc-bridge-binary $HOME/ClickHouse/programs/clickhouse-odbc-bridge --base-configs-dir $HOME/ClickHouse/programs/server/ 'test_ssl_cert_authentication -ss'
Start tests
====================================================================================================== test session starts ======================================================================================================
platform linux -- Python 3.8.10, pytest-7.1.2, pluggy-1.0.0 -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: /ClickHouse/tests/integration, configfile: pytest.ini
plugins: repeat-0.9.1, xdist-2.5.0, forked-1.4.0, order-1.0.0, timeout-2.1.0
timeout: 900.0s
timeout method: signal
timeout func_only: False
collected 4 items

test_ssl_cert_authentication/test.py::test_https Copy common default production configuration from /clickhouse-config. Files: config.xml, users.xml
PASSED
test_ssl_cert_authentication/test.py::test_https_wrong_cert PASSED
test_ssl_cert_authentication/test.py::test_https_non_ssl_auth PASSED
test_ssl_cert_authentication/test.py::test_create_user PASSED

================================================================================================= 4 passed in 118.58s (0:01:58) =================================================================================================

Path to binary and configs maybe specified via env variables:

$ export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=$HOME/ClickHouse/programs/server/
$ export CLICKHOUSE_TESTS_SERVER_BIN_PATH=$HOME/ClickHouse/programs/clickhouse
$ export CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH=$HOME/ClickHouse/programs/clickhouse-odbc-bridge
$ ./runner 'test_odbc_interaction'
$ # or ./runner '-v -ss'
Start tests
============================= test session starts ==============================
platform linux2 -- Python 2.7.15rc1, pytest-4.0.0, py-1.7.0, pluggy-0.8.0
rootdir: /ClickHouse/tests/integration, inifile: pytest.ini
collected 6 items

test_odbc_interaction/test.py ......                                     [100%]
==================== 6 passed, 1 warnings in 96.33 seconds =====================

You can just open shell inside a container by overwritting the command:
./runner --command=bash

Rebuilding the docker containers

The main container used for integration tests lives in docker/test/integration/base/Dockerfile. Rebuild it with

cd docker/test/integration/base
docker build -t clickhouse/integration-test .

The helper container used by the runner script is in docker/test/integration/runner/Dockerfile.

Adding new tests

To add new test named foo, create a directory test_foo with an empty __init__.py and a file
named test.py containing tests in it. All functions with names starting with test will become test cases.

helpers directory contains utilities for:

• Launching a ClickHouse cluster with or without ZooKeeper in docker containers.
• Sending queries to launched instances.
• Introducing network failures such as severing network link between two instances.

To assert that two TSV files must be equal, wrap them in the TSV class and use the regular assert
statement. Example: assert TSV(result) == TSV(reference). In case the assertion fails, pytest
will automagically detect the types of variables and only the small diff of two files is printed.

Troubleshooting

If tests failing for mysterious reasons, this may help:

sudo service docker stop
sudo bash -c 'rm -rf /var/lib/docker/*'
sudo service docker start

iptables-nft

On Ubuntu 20.10 and later in host network mode (default) one may encounter problem with nested containers not seeing each other. It happens because legacy and nftables rules are out of sync. Problem can be solved by:

sudo iptables -P FORWARD ACCEPT

push问题

在push的时候出现一个问题：

首先，password指github生成的token。
其次，如果输入了最新token还是显示403，那么就是因为生成token给的权限不够多。删了token重新生成即可。

出现冲突

merge

git fetch upstream
git checkout main
git rebase upstream/main
git submodule update

格式检查

python 要用black格式化，不应该有trailing whitespace 不然过不了。

pip install black
black **.py

doc和pr描述

pr描述的格式应该不变。即这括号不删

• doc应该在docs中更新
  docs/en/sql-reference/statements/alter/partition.md
• changelogs也要更新这个pr
  CHANGELOG.md

Raw as a synonym for TSVRaw 62079

void registerWithNamesAndTypes(const std::string & base_format_name, RegisterWithNamesAndTypesFunc register_func)
{
    register_func(base_format_name, false, false);
    register_func(base_format_name + "WithNames", true, false);
    register_func(base_format_name + "WithNamesAndTypes", true, true);
}

# src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp
void registerOutputFormatTabSeparated(FormatFactory & factory)
{
    for (bool is_raw : {false, true})
    {
        auto register_func = [&](const String & format_name, bool with_names, bool with_types)
        {
            factory.registerOutputFormat(format_name, [is_raw, with_names, with_types](
                WriteBuffer & buf,
                const Block & sample,
                const FormatSettings & settings)
            {
                return std::make_shared<TabSeparatedRowOutputFormat>(buf, sample, with_names, with_types, is_raw, settings);
            });

            factory.markOutputFormatSupportsParallelFormatting(format_name);
        };

        registerWithNamesAndTypes(is_raw ? "TSVRaw" : "TSV", register_func);
        registerWithNamesAndTypes(is_raw ? "TabSeparatedRaw" : "TabSeparated", register_func);
        if (is_raw)
            registerWithNamesAndTypes("LineAsString", register_func);
    }
}

# src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp
void registerInputFormatTabSeparated(FormatFactory & factory)
void registerTSVSchemaReader(FormatFactory & factory)

void registerFileSegmentationEngineTabSeparated(FormatFactory & factory)