CMake:测试的其他补充(重要)

导言

通过前几篇的学习,我们基本掌握了关于单元测试的相关内容。当然,随着技术的不断发展,根据不同业务的需求测试框测出不穷,我们没有办法一一列举。本篇我们将补充几个测试的相关技术,如预期测试失败、并行测试以及测试子集等。

预期测试失败

理想情况下,我们希望所有的测试能在每个平台上通过。然而,也可能想要测试预期的失败或异常是否会在受控的设置中进行。这种情况下,我们将把预期的失败定义为成功。我们认为,这通常应该交给测试框架(例如:Catch2Google Test)的任务,它应该检查预期的失败并向CMake报告成功。但是,在某些情况下,可能希望将测试的非零返回代码定义为成功;换句话说,可能想要颠倒成功和失败的定义。

项目结构

.
├── CMakeLists.txt
└── test.py

项目地址:

https://gitee.com/jiangli01/tutorials/tree/master/cmake-tutorial/chapter4/05

CMakeLists.txt

cmake_minimum_required(VERSION 3.5 FATAL_ERROR)

project(test_error LANGUAGES NONE)

find_package(PythonInterp REQUIRED)

enable_testing()

add_test(example ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test.py)

set_tests_properties(example PROPERTIES WILL_FAIL true)

定义测试并告诉CMake,测试预期会失败:

set_tests_properties(example PROPERTIES WILL_FAIL true)

相关源码

test.py

import sys
# simulate a failing test
sys.exit(1)

输出结果

Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/05/build
    Start 1: example
1/1 Test #1: example ..........................   Passed    0.01 sec

100% tests passed, 0 tests failed out of 1

Total Test time (real) =   0.02 sec

使用超时测试运行时间过长的测试

理想情况下,测试集应该花很短的时间进行,以便开发人员经常运行测试,并使每个提交(变更集)进行测试成为可能(或更容易)。然而,有些测试可能会花费更长的时间或者被卡住(例如,由于高文件I/O负载),可能需要设置超时来终止耗时过长的测试,它们延迟了整个测试,并阻塞了部署管道。本节,将通过一种设置超时的方法,可以针对每个测试设置不同的超时。

项目结构

.
├── CMakeLists.txt
└── test.py

项目地址:

https://gitee.com/jiangli01/tutorials/tree/master/cmake-tutorial/chapter4/06

CMakeLists.txt

cmake_minimum_required(VERSION 3.5 FATAL_ERROR)

project(test_long_test LANGUAGES NONE)

find_package(PythonInterp REQUIRED)

enable_testing()

add_test(example ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test.py)
set_tests_properties(example PROPERTIES TIMEOUT 10)
set_tests_properties(example PROPERTIES TIMEOUT 10)

为测试指定时限,设置为10秒

相关源码

test.py

import sys
import time
# wait for 2 seconds
time.sleep(2)
# report success
sys.exit(0)

输出结果

mkdir build & cd build
cmake ..
ctest
Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/06/build
    Start 1: example
1/1 Test #1: example ..........................   Passed    2.01 sec

100% tests passed, 0 tests failed out of 1

Total Test time (real) =   2.02 sec

为了验证超时是否有效,将test.py中的sleep命令增加到11秒,并重新运行测试:

Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/06/build
    Start 1: example
1/1 Test #1: example ..........................***Timeout  10.02 sec

0% tests passed, 1 tests failed out of 1

Total Test time (real) =  10.02 sec

The following tests FAILED:
          1 - example (Timeout)
Errors while running CTest

并行测试

大多数现代计算机都有4个或更多个CPU核芯。CTest有个非常棒的特性,能够并行运行测试,如果有多个可用的核。这可以减少测试的总时间。

项目结构

.
├── CMakeLists.txt
└── test
    ├── a.py
    ├── b.py
    ├── c.py
    ├── d.py
    ├── e.py
    ├── f.py
    ├── g.py
    ├── h.py
    ├── i.py
    └── j.py

项目地址:

https://gitee.com/jiangli01/tutorials/tree/master/cmake-tutorial/chapter4/07

CMakeLists.txt

cmake_minimum_required(VERSION 3.10 FATAL_ERROR)

project(recipe-08 LANGUAGES NONE)

find_package(PythonInterp REQUIRED)

enable_testing()
add_test(a ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/a.py)
add_test(b ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/b.py)
add_test(c ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/c.py)
add_test(d ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/d.py)
add_test(e ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/e.py)
add_test(f ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/f.py)
add_test(g ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/g.py)
add_test(h ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/h.py)
add_test(i ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/i.py)
add_test(j ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/j.py)

相关源码

这里我们只给出了一个python文件,其他文件按照如下表格设置时间即可。
在这里插入图片描述
a.py

import sys
import time
# wait for 0.5 seconds
time.sleep(0.5)
# finally report success
sys.exit(0)

输出结果

mkdir build & cd build
cmake ..
ctest
Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/07/build
      Start  1: a
 1/10 Test  #1: a ................................   Passed    0.51 sec
      Start  2: b
 2/10 Test  #2: b ................................   Passed    0.52 sec
      Start  3: c
 3/10 Test  #3: c ................................   Passed    0.52 sec
      Start  4: d
 4/10 Test  #4: d ................................   Passed    0.52 sec
      Start  5: e
 5/10 Test  #5: e ................................   Passed    1.52 sec
      Start  6: f
 6/10 Test  #6: f ................................   Passed    1.52 sec
      Start  7: g
 7/10 Test  #7: g ................................   Passed    1.52 sec
      Start  8: h
 8/10 Test  #8: h ................................   Passed    2.52 sec
      Start  9: i
 9/10 Test  #9: i ................................   Passed    3.52 sec
      Start 10: j
10/10 Test #10: j ................................   Passed    4.52 sec

100% tests passed, 0 tests failed out of 10

Total Test time (real) =  17.20 sec
ctest --parallel 4
Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/07/build
      Start 10: j
      Start  9: i
      Start  8: h
      Start  7: g
 1/10 Test  #7: g ................................   Passed    1.51 sec
      Start  6: f
 2/10 Test  #8: h ................................   Passed    2.51 sec
      Start  5: e
 3/10 Test  #6: f ................................   Passed    1.51 sec
      Start  4: d
 4/10 Test  #9: i ................................   Passed    3.52 sec
      Start  3: c
 5/10 Test  #4: d ................................   Passed    0.51 sec
      Start  2: b
 6/10 Test  #5: e ................................   Passed    1.51 sec
      Start  1: a
 7/10 Test  #3: c ................................   Passed    0.51 sec
 8/10 Test  #2: b ................................   Passed    0.51 sec
 9/10 Test #10: j ................................   Passed    4.51 sec
10/10 Test  #1: a ................................   Passed    0.51 sec

100% tests passed, 0 tests failed out of 10

Total Test time (real) =   4.54 sec
ctest --parallel 8
Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/07/build
      Start 10: j
      Start  9: i
      Start  8: h
      Start  7: g
      Start  6: f
      Start  5: e
      Start  4: d
      Start  2: b
 1/10 Test  #4: d ................................   Passed    0.52 sec
      Start  3: c
 2/10 Test  #2: b ................................   Passed    0.52 sec
      Start  1: a
 3/10 Test  #3: c ................................   Passed    0.51 sec
 4/10 Test  #1: a ................................   Passed    0.51 sec
 5/10 Test  #7: g ................................   Passed    1.52 sec
 6/10 Test  #6: f ................................   Passed    1.52 sec
 7/10 Test  #5: e ................................   Passed    1.52 sec
 8/10 Test  #8: h ................................   Passed    2.52 sec
 9/10 Test  #9: i ................................   Passed    3.52 sec
10/10 Test #10: j ................................   Passed    4.52 sec

100% tests passed, 0 tests failed out of 10

Total Test time (real) =   4.52 sec

运行测试子集

前几节,我们学习了如何在CMake的帮助下并行运行测试,并讨论了从最长的测试开始是最高效的。虽然,这种策略将总测试时间最小化,但是在特定特性的代码开发期间,或者在调试期间,我们可能不希望运行整个测试集。对于调试和代码开发,我们只需要能够运行选定的测试子集。t通过本节我们对这一策略进行进一步探究。

项目结构

.
├── CMakeLists.txt
└── test
    ├── benchmark-a.py
    ├── benchmark-b.py
    ├── feature-a.py
    ├── feature-b.py
    ├── feature-c.py
    └── feature-d.py

项目地址:

https://gitee.com/jiangli01/tutorials/tree/master/cmake-tutorial/chapter4/08

CMakeLists.txt

cmake_minimum_required(VERSION 3.10 FATAL_ERROR)

project(test_subset LANGUAGES NONE)

find_package(PythonInterp REQUIRED)

enable_testing()
add_test(
  NAME feature-a
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/feature-a.py
)

add_test(
  NAME feature-b
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/feature-b.py
)

add_test(
  NAME feature-c
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/feature-c.py
)

add_test(
  NAME feature-d
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/feature-d.py
)

add_test(
  NAME benchmark-a
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/benchmark-a.py
)

add_test(
  NAME benchmark-b
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/benchmark-b.py
)

set_tests_properties(
  feature-a
  feature-b
  feature-c
  PROPERTIES
      LABELS "quick"
)

set_tests_properties(
  feature-d
  benchmark-a
  benchmark-b
  PROPERTIES
      LABELS "long"
)

给较短的测试贴上quick的标签,给较长的测试贴上long的标签:

set_tests_properties(
  feature-a
  feature-b
  feature-c
  PROPERTIES
      LABELS "quick"
)

set_tests_properties(
  feature-d
  benchmark-a
  benchmark-b
  PROPERTIES
      LABELS "long"
)

相关源码

我们假设总共有六个测试:前三个测试比较短,名称分别为feature-afeature-bfeature-c,还有三个长测试,名称分别是feature-dbenchmark-abenchmark-b。我们只给出feature-a.py,其他知识睡眠时间的不同。

import sys
import time
# wait for 0.1 seconds
time.sleep(1)
# finally report success
sys.exit(0)

输出结果

ctest -R feature
Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/08/build
    Start 1: feature-a
1/4 Test #1: feature-a ........................   Passed    0.11 sec
    Start 2: feature-b
2/4 Test #2: feature-b ........................   Passed    0.11 sec
    Start 3: feature-c
3/4 Test #3: feature-c ........................   Passed    0.11 sec
    Start 4: feature-d
4/4 Test #4: feature-d ........................   Passed    1.01 sec

100% tests passed, 0 tests failed out of 4

Label Time Summary:
long     =   1.01 sec*proc (1 test)
quick    =   0.33 sec*proc (3 tests)

Total Test time (real) =   1.36 sec
ctest -L long
Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/08/build
    Start 4: feature-d
1/3 Test #4: feature-d ........................   Passed    1.01 sec
    Start 5: benchmark-a
2/3 Test #5: benchmark-a ......................   Passed    1.01 sec
    Start 6: benchmark-b
3/3 Test #6: benchmark-b ......................   Passed    1.01 sec

100% tests passed, 0 tests failed out of 3

Label Time Summary:
long    =   3.04 sec*proc (3 tests)

Total Test time (real) =   3.04 sec
ctest -L quick
Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/08/build
    Start 1: feature-a
1/3 Test #1: feature-a ........................   Passed    0.11 sec
    Start 2: feature-b
2/3 Test #2: feature-b ........................   Passed    0.11 sec
    Start 3: feature-c
3/3 Test #3: feature-c ........................   Passed    0.12 sec

100% tests passed, 0 tests failed out of 3

Label Time Summary:
quick    =   0.34 sec*proc (3 tests)

Total Test time (real) =   0.34 sec

使用测试固件

本节将学习如何使用测试固件。这对于更复杂的测试非常有用,这些测试需要在测试运行前进行设置,以及在测试完成后执行清理操作(例如:创建示例数据库、设置连接、断开连接、清理测试数据库等等)。我们需要运行一个设置或清理操作的测试,并能够以一种可预测和健壮的方式自动触发这些步骤,而不需要引入代码重复。这些设置和清理步骤可以委托给测试框架(例如Google TestCatch2)。

项目结构

.
├── CMakeLists.txt
└── test
    ├── cleanup.py
    ├── feature-a.py
    ├── feature-b.py
    └── setup.py

项目地址:

https://gitee.com/jiangli01/tutorials/tree/master/cmake-tutorial/chapter4/09

CMakeLists.txt

cmake_minimum_required(VERSION 3.10 FATAL_ERROR)

project(test_firmware LANGUAGES NONE)

find_package(PythonInterp REQUIRED)

enable_testing()

add_test(
  NAME setup
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/setup.py
)

set_tests_properties(
  setup
  PROPERTIES
    FIXTURES_SETUP my-fixture
)

add_test(
  NAME feature-a
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/feature-a.py
)

add_test(
  NAME feature-b
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/feature-b.py
)

set_tests_properties(
  feature-a
  feature-b
  PROPERTIES
    FIXTURES_REQUIRED my-fixture
)

add_test(
  NAME cleanup
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/cleanup.py
)

set_tests_properties(
  cleanup
  PROPERTIES
    FIXTURES_CLEANUP my-fixture
)
add_test(
  NAME setup
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/setup.py
  )
set_tests_properties(
  setup
  PROPERTIES
      FIXTURES_SETUP my-fixture
  )
add_test(
  NAME feature-a
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/feature-a.py
  )
add_test(
  NAME feature-b
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/feature-b.py
  )
set_tests_properties(
  feature-a
  feature-b
  PROPERTIES
      FIXTURES_REQUIRED my-fixture
  )
add_test(
  NAME cleanup
  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/cleanup.py
  )
set_tests_properties(
  cleanup
  PROPERTIES
      FIXTURES_CLEANUP my-fixture
  )

定义了一个文本固件,并将其称为my-fixture。我们为安装测试提供了FIXTURES_SETUP属性,并为清理测试了FIXTURES_CLEANUP属性,并且使用FIXTURES_REQUIRED,我们确保测试feature-afeature-b都需要安装和清理步骤才能运行。将它们绑定在一起,可以确保在定义良好的状态下,进入和离开相应的步骤。

相关源码

setup.py

import sys

print("tearing down")

# report success
sys.exit(0)

feature-a.py

import sys

print("running test a")

# report success
sys.exit(0)

feature-b.py

import sys

print("running test b")

# report success
sys.exit(0)

clearup.py

import sys

print("tearing down")

# report success
sys.exit(0)

输出结果

ctest
Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/09/build
    Start 1: setup
1/4 Test #1: setup ............................   Passed    0.01 sec
    Start 2: feature-a
2/4 Test #2: feature-a ........................   Passed    0.01 sec
    Start 3: feature-b
3/4 Test #3: feature-b ........................   Passed    0.01 sec
    Start 4: cleanup
4/4 Test #4: cleanup ..........................   Passed    0.01 sec

100% tests passed, 0 tests failed out of 4

Total Test time (real) =   0.05 sec
ctest -R feature-a
Test project /home/jiangli/repo/tutorials/cmake-tutorial/chapter4/09/build
    Start 1: setup
1/3 Test #1: setup ............................   Passed    0.01 sec
    Start 2: feature-a
2/3 Test #2: feature-a ........................   Passed    0.01 sec
    Start 4: cleanup
3/3 Test #4: cleanup ..........................   Passed    0.01 sec

100% tests passed, 0 tests failed out of 3

Total Test time (real) =   0.03 sec

最后,希望大家变得更强!!!

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值