Merge pull request #2098 from lammps/unittest

# - Find libyaml

# Find the native Yaml headers and libraries.

#

#  YAML_INCLUDE_DIRS - where to find yaml.h

#  YAML_LIBRARIES    - List of libraries when using libyaml

#  YAML_FOUND        - True if libyaml is found.

#

find_path(YAML_INCLUDE_DIR yaml.h PATH_SUFFIXES yaml)

find_library(YAML_LIBRARY NAMES yaml)

# handle the QUIET and REQUIRED arguments and

# set YAML_FOUND to TRUE if all variables are non-zero

include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(YAML DEFAULT_MSG YAML_LIBRARY YAML_INCLUDE_DIR)

# Copy the results to the output variables and target.

if(YAML_FOUND)

  set(YAML_LIBRARIES ${YAML_LIBRARY})

  set(YAML_INCLUDE_DIRS ${YAML_INCLUDE_DIR})

  if(NOT TARGET Yaml::Yaml)

    add_library(Yaml::Yaml UNKNOWN IMPORTED)

    set_target_properties(Yaml::Yaml PROPERTIES

      IMPORTED_LOCATION "${YAML_LIBRARY}"

      INTERFACE_INCLUDE_DIRECTORIES "${YAML_INCLUDE_DIR}")

  endif()

endif()

mark_as_advanced(YAML_INCLUDE_DIR YAML_LIBRARY)

message(STATUS "Downloading and building Google Test library")

if(CMAKE_BUILD_TYPE STREQUAL Debug)

  set(GTEST_LIB_POSTFIX d)

else()

  set(GTEST_LIB_POSTFIX)

endif()

include(ExternalProject)

ExternalProject_Add(googletest

                    GIT_REPOSITORY  https://github.com/google/googletest.git

                    GIT_TAG         release-1.10.0

                    SOURCE_DIR      "${CMAKE_BINARY_DIR}/gtest-src"

                    BINARY_DIR      "${CMAKE_BINARY_DIR}/gtest-build"

                    CMAKE_ARGS      ${CMAKE_REQUEST_PIC} ${CMAKE_EXTRA_GTEST_OPTS}

                                    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

                                    -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>

                                    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}

                                    -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}

                                    -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}

                    BUILD_BYPRODUCTS <BINARY_DIR>/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gtest${GTEST_LIB_POSTFIX}.a

                                     <BINARY_DIR>/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gmock${GTEST_LIB_POSTFIX}.a

                                     <BINARY_DIR>/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gtest_main${GTEST_LIB_POSTFIX}.a

                                     <BINARY_DIR>/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gmock_main${GTEST_LIB_POSTFIX}.a

                    LOG_DOWNLOAD ON

                    LOG_CONFIGURE ON

                    LOG_BUILD ON

                    INSTALL_COMMAND ""

                    TEST_COMMAND    "")

ExternalProject_Get_Property(googletest SOURCE_DIR)

set(GTEST_INCLUDE_DIR ${SOURCE_DIR}/googletest/include)

set(GMOCK_INCLUDE_DIR ${SOURCE_DIR}/googlemock/include)

# workaround for CMake 3.10 on ubuntu 18.04

file(MAKE_DIRECTORY ${GTEST_INCLUDE_DIR})

file(MAKE_DIRECTORY ${GMOCK_INCLUDE_DIR})

ExternalProject_Get_Property(googletest BINARY_DIR)

set(GTEST_LIBRARY_PATH ${BINARY_DIR}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gtest${GTEST_LIB_POSTFIX}.a)

set(GMOCK_LIBRARY_PATH ${BINARY_DIR}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gmock${GTEST_LIB_POSTFIX}.a)

set(GTEST_MAIN_LIBRARY_PATH ${BINARY_DIR}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gtest_main${GTEST_LIB_POSTFIX}.a)

set(GMOCK_MAIN_LIBRARY_PATH ${BINARY_DIR}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gmock_main${GTEST_LIB_POSTFIX}.a)

# Prevent GoogleTest from overriding our compiler/linker options

# when building with Visual Studio

set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)

find_package(Threads QUIET)

add_library(GTest::GTest UNKNOWN IMPORTED)

set_target_properties(GTest::GTest PROPERTIES

        IMPORTED_LOCATION ${GTEST_LIBRARY_PATH}

        INTERFACE_INCLUDE_DIRECTORIES ${GTEST_INCLUDE_DIR}

        IMPORTED_LINK_INTERFACE_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})

add_dependencies(GTest::GTest googletest)

add_library(GTest::GMock UNKNOWN IMPORTED)

set_target_properties(GTest::GMock PROPERTIES

        IMPORTED_LOCATION ${GMOCK_LIBRARY_PATH}

        INTERFACE_INCLUDE_DIRECTORIES ${GMOCK_INCLUDE_DIR}

        IMPORTED_LINK_INTERFACE_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})

add_dependencies(GTest::GMock googletest)

add_library(GTest::GTestMain UNKNOWN IMPORTED)

set_target_properties(GTest::GTestMain PROPERTIES

        IMPORTED_LOCATION ${GTEST_MAIN_LIBRARY_PATH}

        INTERFACE_INCLUDE_DIRECTORIES ${GTEST_INCLUDE_DIR}

        IMPORTED_LINK_INTERFACE_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})

add_dependencies(GTest::GTestMain googletest)

add_library(GTest::GMockMain UNKNOWN IMPORTED)

set_target_properties(GTest::GMockMain PROPERTIES

        IMPORTED_LOCATION ${GMOCK_MAIN_LIBRARY_PATH}

        INTERFACE_INCLUDE_DIRECTORIES ${GMOCK_INCLUDE_DIR}

        IMPORTED_LINK_INTERFACE_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})

add_dependencies(GTest::GMockMain googletest)

option(ENABLE_TESTING "Enable testing" OFF)

if(ENABLE_TESTING)

  enable_testing()

  option(LAMMPS_TESTING_SOURCE_DIR "Location of lammps-testing source directory" "")

  option(LAMMPS_TESTING_GIT_TAG    "Git tag of lammps-testing" "master")

  mark_as_advanced(LAMMPS_TESTING_SOURCE_DIR LAMMPS_TESTING_GIT_TAG)

  if (CMAKE_VERSION VERSION_GREATER "3.10.3" AND NOT LAMMPS_TESTING_SOURCE_DIR)

    include(FetchContent)

    FetchContent_Declare(lammps-testing

      GIT_REPOSITORY https://github.com/lammps/lammps-testing.git

      GIT_TAG ${LAMMPS_TESTING_GIT_TAG}

    )

    FetchContent_GetProperties(lammps-testing)

    if(NOT lammps-testing_POPULATED)

      message(STATUS "Downloading tests...")

      FetchContent_Populate(lammps-testing)

    endif()

    set(LAMMPS_TESTING_SOURCE_DIR ${lammps-testing_SOURCE_DIR})

  elseif(NOT LAMMPS_TESTING_SOURCE_DIR)

    message(WARNING "Full test-suite requires CMake >= 3.11 or copy of\n"

                    "https://github.com/lammps/lammps-testing in LAMMPS_TESTING_SOURCE_DIR")

  endif()

  add_test(NAME ShowHelp COMMAND $<TARGET_FILE:lmp> -help)

  if(EXISTS ${LAMMPS_TESTING_SOURCE_DIR})

    message(STATUS "Running test discovery...")

    file(GLOB_RECURSE TEST_SCRIPTS ${LAMMPS_TESTING_SOURCE_DIR}/tests/core/*/in.*)

    foreach(script_path ${TEST_SCRIPTS})

      get_filename_component(TEST_NAME ${script_path} EXT)

      get_filename_component(SCRIPT_NAME ${script_path} NAME)

      get_filename_component(PARENT_DIR ${script_path} DIRECTORY)

      string(SUBSTRING ${TEST_NAME} 1 -1 TEST_NAME)

      string(REPLACE "-" "_" TEST_NAME ${TEST_NAME})

      string(REPLACE "+" "_" TEST_NAME ${TEST_NAME})

      set(TEST_NAME "test_core_${TEST_NAME}_serial")

      add_test(NAME ${TEST_NAME} COMMAND $<TARGET_FILE:lmp> -in ${SCRIPT_NAME} WORKING_DIRECTORY ${PARENT_DIR})

    endforeach()

    list(LENGTH TEST_SCRIPTS NUM_TESTS)

    message(STATUS "Found ${NUM_TESTS} tests.")

  endif()

  get_filename_component(LAMMPS_UNITTEST_DIR ${LAMMPS_SOURCE_DIR}/../unittest ABSOLUTE)

  get_filename_component(LAMMPS_UNITTEST_BIN ${CMAKE_BINARY_DIR}/unittest ABSOLUTE)

  add_subdirectory(${LAMMPS_UNITTEST_DIR} ${LAMMPS_UNITTEST_BIN})

endif()

.. _testing:

Code Coverage and Testing

---------------------------------------

We do extensive regression testing of the LAMMPS code base on a continuous

basis. Some of the logic to do this has been added to the CMake build so

developers can run the tests directly on their workstation.

Code Coverage and Unit Testing

------------------------------

The LAMMPS code is subject to multiple levels of automated testing

during development: integration testing (i.e. whether the code compiles

on various platforms and with a variety of settings), unit testing

(i.e. whether certain individual parts of the code produce the expected

results for given inputs), run testing (whether selected complete input

decks run without crashing for multiple configurations), and regression

testing (i.e. whether selected input examples reproduce the same

results over a given number of steps and operations within a given

error margin).  The status of this automated testing can be viewed on

`https://ci.lammps.org <https://ci.lammps.org>`_.

The unit testing facility is integrated into the CMake build process

of the LAMMPS source code distribution itself.  It can be enabled by

setting ``-D ENABLE_TESTING=on`` during the CMake configuration step.

It requires the `YAML <http://pyyaml.org/>`_ library and development

headers to compile and will download and compile the

`Googletest <https://github.com/google/googletest/>`_ C++ test framework

for programming the tests.

After compilation is complete, the unit testing is started in the build

folder using the ``ctest`` command, which is part of the CMake software.

The output of this command will be looking something like this::

   [...]$ ctest

   Test project /home/akohlmey/compile/lammps/build-testing

         Start  1: MolPairStyle:hybrid-overlay

    1/26 Test  #1: MolPairStyle:hybrid-overlay .........   Passed    0.02 sec

         Start  2: MolPairStyle:hybrid

    2/26 Test  #2: MolPairStyle:hybrid .................   Passed    0.01 sec

         Start  3: MolPairStyle:lj_class2

    [...]

         Start 25: AngleStyle:harmonic

   25/26 Test #25: AngleStyle:harmonic .................   Passed    0.01 sec

         Start 26: AngleStyle:zero

   26/26 Test #26: AngleStyle:zero .....................   Passed    0.01 sec

   100% tests passed, 0 tests failed out of 26

   Total Test time (real) =   0.27 sec

.. note::

   this is incomplete and only represents a small subset of tests that we run

.. code-block:: bash

   -D ENABLE_TESTING=value               # enable simple run tests of LAMMPS, value = no (default) or yes

   -D LAMMPS_TESTING_SOURCE_DIR=path     # path to lammps-testing repository (option if in custom location)

   -D LAMMPS_TESTING_GIT_TAG=value       # version of lammps-testing repository that should be used, value = master (default) or custom git commit or tag

If you enable testing in the CMake build it will create an additional

target called "test". You can run them with:

.. code-block:: bash

   cmake --build . test

   This unit test framework is new and still under development.

   The coverage is only minimal and will be expanded over time.

   Tests styles of the same kind of style (e.g. pair styles or

   bond styles) are performed with the same executable using

   different input files in YAML format.  So to add a test for

   another pair style can be done by copying the YAML file and

   editing the style settings and then running the individual test

   program with a flag to update the computed reference data.

   Detailed documentation about how to add new test program and

   the contents of the YAML files for existing test programs

   will be provided in time as well.

The test cases used come from the lammps-testing repository. They are

derivatives of the examples folder with some modifications to make the

run faster.

------------

You can also collect code coverage metrics while running the tests by

enabling coverage support during building.

   -D ENABLE_COVERAGE=value  # enable coverage measurements, value = no (default) or yes

This will also add the following targets to generate coverage reports

after running the LAMMPS executable:

after running the LAMMPS executable or the unit tests:

.. code-block:: bash

   make test               # run tests first!

   make gen_coverage_html  # generate coverage report in HTML format

   make gen_coverage_xml   # generate coverage report in XML format

These reports require GCOVR to be installed. The easiest way to do this

to install it via pip:

These reports require `GCOVR <https://gcovr.com/>`_ to be installed. The easiest way

to do this to install it via pip:

.. code-block:: bash

include(GTest)

add_subdirectory(force-styles)