Code coverage analysis using CodeCoverage.cmake with multiple targets

Question

I have unit-test in my project implemented using Boost Unit Test Framework and organized into several modules, i.e.:

#define BOOST_TEST_MODULE Connection_test

#ifndef BOOST_TEST_DYN_LINK
#define BOOST_TEST_DYN_LINK
#endif

#ifndef BOOST_TEST_NO_MAIN
#define BOOST_TEST_NO_MAIN
#endif

#include <boost/test/unit_test.hpp>
#include <boost/test/output_test_stream.hpp>

#define BOOST_TEST_MODULE Connection_test


BOOST_AUTO_TEST_SUITE(Connection_test)

    BOOST_AUTO_TEST_CASE(Connection_construction__test) {

       ***
    }

BOOST_AUTO_TEST_SUITE_END()

And I'm compiling every module as a single executable.

I wanted to use CodeCoverage.cmake module to run code-coverage analysis, but I ran into a problem. I'm supposed to specify a test executable by SETUP_TARGET_FOR_COVERAGE_LCOV, but I don't have only one.

Is there any way to setup multiple test executables at once with CodeCoverage.cmake?

EDIT

I've added tests by add_test() into my root CMakeLists.txt and modified my coverage target like this

include(CTest)
add_test(NAME constant_neuron_test COMMAND constant_neuron_test)
add_test(NAME binary_neuron_test COMMAND binary_neuron_test)
add_test(NAME logistic_neuron_test COMMAND logistic_neuron_test)
add_test(NAME connectionFunctionGeneral_test COMMAND connectionFunctionGeneral_test)
add_test(NAME connection_Function_identity_test COMMAND connection_Function_identity_test)
add_test(NAME neural_network_test COMMAND neural_network_test)
add_test(NAME dataset_test COMMAND dataset_test)
add_test(NAME particle_swarm_test COMMAND particle_swarm_test)
add_test(NAME particle_test COMMAND particle_test)
add_test(NAME NeuralNetworkSum_test COMMAND NeuralNetworkSum_test)
add_test(NAME errorfunction_test COMMAND errorfunction_test)
add_test(NAME DESolver_test COMMAND DESolver_test)

include(CodeCoverage.cmake)
APPEND_COVERAGE_COMPILER_FLAGS()
SETUP_TARGET_FOR_COVERAGE_LCOV(
        NAME coverage                 # New target name
        EXECUTABLE ctest -C ${ROOT_DIR}/CTestTestfile.cmake # Executable in PROJECT_BINARY_DIR
        DEPENDENCIES ${Boost_LIBRARIES}         # Dependencies to build first
)

ctest itself runs correctly:

Test project /home/martin/4Neuro
      Start  1: constant_neuron_test
 1/12 Test  #1: constant_neuron_test ................   Passed    0.04 sec
      Start  2: binary_neuron_test
 2/12 Test  #2: binary_neuron_test ..................   Passed    0.04 sec
      Start  3: logistic_neuron_test
 3/12 Test  #3: logistic_neuron_test ................   Passed    0.05 sec
      Start  4: connectionFunctionGeneral_test
 4/12 Test  #4: connectionFunctionGeneral_test ......   Passed    0.04 sec
      Start  5: connection_Function_identity_test
 5/12 Test  #5: connection_Function_identity_test ...   Passed    0.04 sec
      Start  6: neural_network_test
 6/12 Test  #6: neural_network_test .................   Passed    0.04 sec
      Start  7: dataset_test
 7/12 Test  #7: dataset_test ........................   Passed    0.04 sec
      Start  8: particle_swarm_test
 8/12 Test  #8: particle_swarm_test .................   Passed    0.04 sec
      Start  9: particle_test
 9/12 Test  #9: particle_test .......................   Passed    0.04 sec
      Start 10: NeuralNetworkSum_test
10/12 Test #10: NeuralNetworkSum_test ...............   Passed    0.05 sec
      Start 11: errorfunction_test
11/12 Test #11: errorfunction_test ..................   Passed    0.04 sec
      Start 12: DESolver_test
12/12 Test #12: DESolver_test .......................   Passed    0.05 sec

100% tests passed, 0 tests failed out of 12

Total Test time (real) =   0.53 sec

But when I try to create my coverage report by make coverage, I get this error:

Processing code coverage counters and generating report.
cd /home/martin/4Neuro/build && /usr/bin/lcov --gcov-tool /usr/bin/gcov -directory . --zerocounters
Deleting all .da files in . and subdirectories
Done.
cd /home/martin/4Neuro/build && /usr/bin/lcov --gcov-tool /usr/bin/gcov -c -i -d . -o coverage.base
Capturing coverage data from .
Found gcov version: 7.3.0
Scanning . for .gcno files ...
geninfo: WARNING: no .gcno files found in . - skipping!
Finished .info-file creation
cd /home/martin/4Neuro/build && ctest -C /home/martin/4Neuro/CTestTestfile.cmake
Test project /home/martin/4Neuro/build
No tests were found!!!
cd /home/martin/4Neuro/build && /usr/bin/lcov --gcov-tool /usr/bin/gcov --directory . --capture --output-file coverage.info
Capturing coverage data from .
Found gcov version: 7.3.0
Scanning . for .gcda files ...
geninfo: WARNING: no .gcda files found in . - skipping!
Finished .info-file creation
cd /home/martin/4Neuro/build && /usr/bin/lcov --gcov-tool /usr/bin/gcov -a coverage.base -a coverage.info --output-file coverage.total
Combining tracefiles.
Reading tracefile coverage.base
lcov: ERROR: no valid records found in tracefile coverage.base
CMakeFiles/coverage.dir/build.make:62: recipe for target 'CMakeFiles/coverage' failed
make[3]: *** [CMakeFiles/coverage] Error 255
make[3]: Leaving directory '/home/martin/4Neuro'
CMakeFiles/Makefile2:70: recipe for target 'CMakeFiles/coverage.dir/all' failed
make[2]: *** [CMakeFiles/coverage.dir/all] Error 2
make[2]: Leaving directory '/home/martin/4Neuro'
CMakeFiles/Makefile2:77: recipe for target 'CMakeFiles/coverage.dir/rule' failed
make[1]: *** [CMakeFiles/coverage.dir/rule] Error 2
make[1]: Leaving directory '/home/martin/4Neuro'
Makefile:132: recipe for target 'coverage' failed
make: *** [coverage] Error 2

EDIT 2

I've modified my CMakeLists.txt like this

include(CTest)
enable_testing()
add_subdirectory(${SRC_DIR} ${PROJECT_BINARY_DIR})

# Adding Unit-tests
add_test(NAME constant_neuron_test COMMAND constant_neuron_test)
add_test(NAME binary_neuron_test COMMAND binary_neuron_test)
add_test(NAME logistic_neuron_test COMMAND logistic_neuron_test)
add_test(NAME connectionFunctionGeneral_test COMMAND connectionFunctionGeneral_test)
add_test(NAME connection_Function_identity_test COMMAND connection_Function_identity_test)
add_test(NAME neural_network_test COMMAND neural_network_test)
add_test(NAME dataset_test COMMAND dataset_test)
add_test(NAME particle_swarm_test COMMAND particle_swarm_test)
add_test(NAME particle_test COMMAND particle_test)
add_test(NAME NeuralNetworkSum_test COMMAND NeuralNetworkSum_test)
add_test(NAME errorfunction_test COMMAND errorfunction_test)
add_test(NAME DESolver_test COMMAND DESolver_test)

include(CodeCoverage.cmake)
APPEND_COVERAGE_COMPILER_FLAGS()
SETUP_TARGET_FOR_COVERAGE_LCOV(
        NAME coverage                 # New target name
        EXECUTABLE ctest -j ${n_cores} # Executable in PROJECT_BINARY_DIR
        DEPENDENCIES
            constant_neuron_test
            binary_neuron_test
            logistic_neuron_test
            connectionFunctionGeneral_test
            connection_Function_identity_test
            neural_network_test
            dataset_test
            particle_swarm_test
            particle_test
            NeuralNetworkSum_test
            errorfunction_test
            DESolver_test         # Dependencies to build first
)
set(COVERAGE_EXCLUDES 'external_dependencies/*')

But, unfortunately, the error persists

cd /home/martin/4Neuro/build && /usr/bin/lcov --gcov-tool /usr/bin/gcov -c -i -d . -o coverage.base
Capturing coverage data from .
Found gcov version: 7.3.0
Scanning . for .gcno files ...
geninfo: WARNING: no .gcno files found in . - skipping!
Finished .info-file creation
cd /home/martin/4Neuro/build && ctest -j 3
Test project /home/martin/4Neuro/build
No tests were found!!!
cd /home/martin/4Neuro/build && /usr/bin/lcov --gcov-tool /usr/bin/gcov --directory . --capture --output-file coverage.info
Capturing coverage data from .
Found gcov version: 7.3.0
Scanning . for .gcda files ...
geninfo: WARNING: no .gcda files found in . - skipping!
Finished .info-file creation
cd /home/martin/4Neuro/build && /usr/bin/lcov --gcov-tool /usr/bin/gcov -a coverage.base -a coverage.info --output-file coverage.total
Combining tracefiles.
Reading tracefile coverage.base
lcov: ERROR: no valid records found in tracefile coverage.base
CMakeFiles/coverage.dir/build.make:71: recipe for target 'CMakeFiles/coverage' failed
make[3]: *** [CMakeFiles/coverage] Error 255
make[3]: Leaving directory '/home/martin/4Neuro'
CMakeFiles/Makefile2:81: recipe for target 'CMakeFiles/coverage.dir/all' failed
make[2]: *** [CMakeFiles/coverage.dir/all] Error 2
make[2]: Leaving directory '/home/martin/4Neuro'
CMakeFiles/Makefile2:88: recipe for target 'CMakeFiles/coverage.dir/rule' failed
make[1]: *** [CMakeFiles/coverage.dir/rule] Error 2
make[1]: Leaving directory '/home/martin/4Neuro'
Makefile:132: recipe for target 'coverage' failed
make: *** [coverage] Error 2

OSE · Accepted Answer

I had this problem too and struggled for quite awhile to find a solution. For me the fix was to move this CMake code above any other targets that are defined:

include(CodeCoverage.cmake)
APPEND_COVERAGE_COMPILER_FLAGS()

It looks like if it the targets are defined first then the compiler flags are not updated.

jsantander · Answer

Ok, I tried to reproduce a minimal set of your setup.

I have:

.
├── CMakeLists.txt
├── CodeCoverage.cmake
├── test1.cpp
└── test2.cpp

Where CodeCoverage.cmake comes from https://github.com/bilke/cmake-modules/blob/master/CodeCoverage.cmake

CMakeLists.txt is:

cmake_minimum_required(VERSION 3.10)
find_package(Boost COMPONENTS system filesystem unit_test_framework REQUIRED)
include(CTest)
enable_testing()

add_executable(test1_test test1.cpp)
target_link_libraries(test1_test 
                      ${Boost_FILESYSTEM_LIBRARY}
                      ${Boost_SYSTEM_LIBRARY}
                      ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
add_executable(test2_test test2.cpp)
target_link_libraries(test2_test
                      ${Boost_FILESYSTEM_LIBRARY}
                      ${Boost_SYSTEM_LIBRARY}
                      ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})

add_test(NAME test1_test COMMAND test1_test)
add_test(NAME test2_test COMMAND test2_test)

include(CodeCoverage.cmake)
APPEND_COVERAGE_COMPILER_FLAGS()
SETUP_TARGET_FOR_COVERAGE_LCOV(
        NAME coverage                 
        EXECUTABLE ctest -j ${n_cores} # Executable in PROJECT_BINARY_DIR
        DEPENDENCIES
            test1_test
            test2_test)

And test1.cpp (and test2.cpp) are

#define BOOST_TEST_MODULE test1

#ifndef BOOST_TEST_DYN_LINK
#define BOOST_TEST_DYN_LINK
#endif

#define BOOST_TEST_MAIN

#include <boost/test/unit_test.hpp>
#include <boost/test/output_test_stream.hpp>

#define BOOST_TEST_MODULE test1


BOOST_AUTO_TEST_SUITE(test1)

    BOOST_AUTO_TEST_CASE(test1__test) {
        BOOST_CHECK_EQUAL(1, 1);
    }

BOOST_AUTO_TEST_SUITE_END()

Now I do:

mkdir build
cd build
cmake ..

the output is:

-- The C compiler identification is GNU 7.3.0
-- The CXX compiler identification is GNU 7.3.0
-- Check for working C compiler: /usr/bin/cc
-- Check for working C compiler: /usr/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Check for working CXX compiler: /usr/bin/c++
-- Check for working CXX compiler: /usr/bin/c++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Boost version: 1.65.1
-- Found the following Boost libraries:
--   system
--   filesystem
--   unit_test_framework
CMake Warning at CodeCoverage.cmake:116 (message):
  Code coverage results with an optimised (non-Debug) build may be misleading
Call Stack (most recent call first):
  CMakeLists.txt:20 (include)


-- Appending code coverage compiler flags: -g -O0 --coverage -fprofile-arcs -ftest-coverage
-- Configuring done
-- Generating done
-- Build files have been written to: /home/jsantand/t/build

Finally running make coverage:

Scanning dependencies of target test1_test
[ 20%] Building CXX object CMakeFiles/test1_test.dir/test1.cpp.o
[ 40%] Linking CXX executable test1_test
[ 40%] Built target test1_test
Scanning dependencies of target test2_test
[ 60%] Building CXX object CMakeFiles/test2_test.dir/test2.cpp.o
[ 80%] Linking CXX executable test2_test
[ 80%] Built target test2_test
Scanning dependencies of target coverage
[100%] Resetting code coverage counters to zero.
Processing code coverage counters and generating report.
Deleting all .da files in . and subdirectories
Done.
Capturing coverage data from .
Found gcov version: 7.3.0
Scanning . for .gcno files ...
Found 2 graph files in .
Processing test1_test.dir/test1.cpp.gcno
Processing test2_test.dir/test2.cpp.gcno
Finished .info-file creation
Test project /home/jsantand/t/build
    Start 1: test1_test
1/2 Test #1: test1_test .......................   Passed    0.01 sec
    Start 2: test2_test
2/2 Test #2: test2_test .......................   Passed    0.01 sec

100% tests passed, 0 tests failed out of 2

Total Test time (real) =   0.02 sec
Capturing coverage data from .
Found gcov version: 7.3.0
Scanning . for .gcda files ...
Found 2 data files in .
Processing test1_test.dir/test1.cpp.gcda
Processing test2_test.dir/test2.cpp.gcda
Finished .info-file creation
Combining tracefiles.
Reading tracefile coverage.base
Reading tracefile coverage.info
Writing data to coverage.total
Summary coverage rate:
  lines......: 65.4% (229 of 350 lines)
  functions..: 68.3% (110 of 161 functions)
  branches...: no data found
Reading tracefile coverage.total
Deleted 0 files
Writing data to /home/jsantand/t/build/coverage.info.cleaned
Summary coverage rate:
  lines......: 65.4% (229 of 350 lines)
  functions..: 68.3% (110 of 161 functions)
  branches...: no data found
Reading data file /home/jsantand/t/build/coverage.info.cleaned
Found 37 entries.
Found common filename prefix "/usr/include"
Writing .css and .png files.
Generating output.
Processing file /home/jsantand/t/test1.cpp
Processing file /home/jsantand/t/test2.cpp
Processing file boost/type_index.hpp
Processing file boost/function/function_base.hpp
Processing file boost/function/function_template.hpp
Processing file boost/smart_ptr/shared_ptr.hpp
Processing file boost/smart_ptr/detail/sp_counted_base_std_atomic.hpp
Processing file boost/smart_ptr/detail/shared_count.hpp
Processing file boost/test/unit_test_suite.hpp
Processing file boost/test/unit_test.hpp
Processing file boost/test/unit_test_log.hpp
Processing file boost/test/tools/assertion_result.hpp
Processing file boost/test/tools/detail/print_helper.hpp
Processing file boost/test/tools/detail/fwd.hpp
Processing file boost/test/tools/old/impl.hpp
Processing file boost/test/tree/observer.hpp
Processing file boost/test/tree/test_unit.hpp
Processing file boost/test/tree/fixture.hpp
Processing file boost/test/tree/decorator.hpp
Processing file boost/test/utils/lazy_ostream.hpp
Processing file boost/test/utils/class_properties.hpp
Processing file boost/test/utils/trivial_singleton.hpp
Processing file boost/test/utils/wrap_stringstream.hpp
Processing file boost/test/utils/basic_cstring/bcs_char_traits.hpp
Processing file boost/test/utils/basic_cstring/basic_cstring.hpp
Processing file boost/type_index/type_index_facade.hpp
Processing file boost/type_index/stl_type_index.hpp
Processing file boost/type_traits/integral_constant.hpp
Processing file c++/7/typeinfo
Processing file c++/7/bits/ios_base.h
Processing file c++/7/bits/move.h
Processing file c++/7/bits/alloc_traits.h
Processing file c++/7/bits/stl_construct.h
Processing file c++/7/bits/stl_vector.h
Processing file c++/7/bits/atomic_base.h
Processing file c++/7/bits/allocator.h
Processing file c++/7/ext/new_allocator.h
Writing directory view page.
Overall coverage rate:
  lines......: 65.4% (229 of 350 lines)
  functions..: 68.3% (110 of 161 functions)
Lcov code coverage info report saved in coverage.info.
Open ./coverage/index.html in your browser to view the coverage report.
[100%] Built target coverage

Code coverage analysis using CodeCoverage.cmake with multiple targets

Tags:

c++

unit-testing

code-coverage

cmake

EDIT

EDIT 2

Eenoku

2 Answers

OSE

jsantander

Recent Activity

Donate For Us

Code coverage analysis using CodeCoverage.cmake with multiple targets

Tags:

c++

unit-testing

code-coverage

cmake

EDIT

EDIT 2

Eenoku

2 Answers

OSE

jsantander

Related questions

Recent Activity

Donate For Us